@midscene/core 1.10.2-beta-20260706032158.0 → 1.10.3

package/dist/types/agent/agent.d.ts

@@ -4,6 +4,7 @@ import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentOpt
 import type { AbstractInterface } from '../device';
 import type { TaskRunner } from '../task-runner';
 import { ModelConfigManager } from '@midscene/shared/env';
+import { type MidsceneUsageMetrics } from './metrics';
 import { type RunGherkinScenarioOptions } from './run-gherkin-scenario';
 import { TaskCache } from './task-cache';
 import { TaskExecutor } from './tasks';
@@ -29,6 +30,9 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     dryMode: boolean;
     onTaskStartTip?: OnTaskStartTip;
     taskCache?: TaskCache;
+    private readonly metricsCollector;
+    private usageCallCounter;
+    private readonly countedUsageKeys;
     private dumpUpdateListeners;
     private readonly progressBus;
     get onDumpUpdate(): ((dump: string, executionDump?: ExecutionDump) => void) | undefined;
@@ -80,6 +84,17 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     setAIActContext(prompt: string): Promise<void>;
     resetDump(): ReportActionDump;
     appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner): void;
+    /**
+     * Fold any not-yet-counted task usage from an execution dump into the
+     * instance metrics. Snapshots are re-emitted as tasks progress, so each
+     * usage value is keyed by `${taskId}:${field}` and counted at most once.
+     */
+    private collectUsageMetrics;
+    private consumeUsage;
+    /**
+     * Aggregated LLM usage accumulated by this agent since it was created.
+     */
+    get metrics(): MidsceneUsageMetrics;
     dumpDataString(opt?: {
         inlineScreenshots?: boolean;
     }): string;
@@ -102,7 +117,6 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
         value: string | number;
     } & {
         autoDismissKeyboard?: boolean;
-        keyboardTypeDelay?: number;
     } & {
         mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
     }): Promise<void>;
@@ -111,7 +125,6 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      */
     aiInput(value: string | number, locatePrompt: TUserPrompt, opt?: LocateOption & {
         autoDismissKeyboard?: boolean;
-        keyboardTypeDelay?: number;
     } & {
         mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
     }): Promise<void>;

package/dist/types/agent/index.d.ts

@@ -5,6 +5,7 @@ export { extractInsightParam, locateParamStr, paramStr, taskTitleStr, typeStr, }
 export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
 export { cacheFileExt } from './task-cache';
 export { TaskExecutor } from './tasks';
+export type { MidsceneUsageMetrics, UsageBucket } from './metrics';
 export type { GherkinStepKeyword, RunGherkinScenarioOptions, } from './run-gherkin-scenario';
 export type { AgentOpt } from '../types';
 export type { RecordToReportOptions, RecordToReportScreenshot } from '../types';

package/dist/types/agent/metrics.d.ts

@@ -0,0 +1,48 @@
+import type { AIUsageInfo } from '../types';
+/**
+ * Aggregated usage for a single grouping key (intent or model).
+ */
+export interface UsageBucket {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    calls: number;
+}
+/**
+ * Instance-level snapshot of LLM usage accumulated by an Agent.
+ *
+ * Designed for cost observability: read it back after a logical unit of work
+ * and push to tools like Langfuse. `byIntent` / `byModel` provide free
+ * breakdowns derived from the usage data Midscene already records per call.
+ */
+export interface MidsceneUsageMetrics {
+    totalPromptTokens: number;
+    totalCompletionTokens: number;
+    totalTokens: number;
+    totalCachedInput: number;
+    totalTimeCostMs: number;
+    calls: number;
+    byIntent: Record<string, UsageBucket>;
+    byModel: Record<string, UsageBucket>;
+}
+/**
+ * Pure accumulator for {@link AIUsageInfo}. Deduplication of calls is the
+ * caller's responsibility; every `add` counts as one call.
+ */
+export declare class MetricsCollector {
+    private totalPromptTokens;
+    private totalCompletionTokens;
+    private totalTokens;
+    private totalCachedInput;
+    private totalTimeCostMs;
+    private calls;
+    private byIntent;
+    private byModel;
+    add(usage: AIUsageInfo): void;
+    snapshot(): MidsceneUsageMetrics;
+    /**
+     * Clear all accumulated state. Not wired to a public Agent API yet; kept for
+     * a future per-unit reset (e.g. resetting metrics at the start of a spec).
+     */
+    reset(): void;
+}

package/dist/types/ai-model/model-adapter/types.d.ts

@@ -1,4 +1,5 @@
 import type { PixelBbox, PlanningAction } from '../../types';
+import type { AIUsageInfo } from '../../types';
 import type { IModelConfig, TIntent, TModelReasoningEnabled } from '@midscene/shared/env';
 import type OpenAI from 'openai';
 import type { JsonParser, JsonParserContext, JsonParserSource } from '../service-caller/json';
@@ -152,6 +153,13 @@ export interface ModelAdapter {
 export interface ModelRuntime {
     config: IModelConfig;
     adapter: ModelAdapter;
+    /**
+     * Optional callback fired after every underlying model call with the shaped
+     * usage info. Provides a single collection point for callers that want to
+     * aggregate usage across all model invocations (including auxiliary calls
+     * such as order-sensitive judging and deep-locate search-area calls).
+     */
+    onUsage?: (usage: AIUsageInfo) => void;
 }
 export interface ModelAdapterDefinition {
     jsonParser?: JsonParserPreset | JsonParser;

package/dist/types/ai-model/models/doubao.d.ts

@@ -1,8 +1,5 @@
-import type { JsonParserContext, JsonParserSource, ModelAdapterDefinition } from '../model-adapter/types';
+import type { ModelAdapterDefinition } from '../model-adapter/types';
 import { type LocateResultValue } from '../shared/model-locate-result';
-export declare function normalizeDoubaoJsonObject(obj: any, context?: Pick<JsonParserContext, 'preserveStringValueKeys'>): any;
-export declare function shouldRepairDoubaoLocateJson(source: JsonParserSource): source is "planning-action-param" | "locate" | "section-locator";
-export declare function preprocessDoubaoLocateJson(input: string): string;
 export declare function parseDoubaoRawLocateValue(input: unknown): LocateResultValue;
 export declare const doubaoAdapters: {
     'doubao-vision': ModelAdapterDefinition;

package/dist/types/ai-model/models/gemini.d.ts

@@ -14,7 +14,7 @@ export declare const extractGeminiContentAndReasoning: (message: GeminiContentSo
 export declare const geminiAdapters: {
     gemini: {
         chatCompletion: {
-            unsupportedUserConfig: ("reasoningEnabled" | "reasoningBudget")[];
+            unsupportedUserConfig: "reasoningBudget"[];
             buildChatCompletionParams: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
             messageExtraction: {
                 kind: "custom";

package/dist/types/ai-model/models/registry.d.ts

@@ -63,7 +63,7 @@ export declare const MODEL_ADAPTER_CONFIGS: {
     'vlm-ui-tars-doubao-1.5': ModelAdapterDefinition;
     gemini: {
         chatCompletion: {
-            unsupportedUserConfig: ("reasoningEnabled" | "reasoningBudget")[];
+            unsupportedUserConfig: "reasoningBudget"[];
             buildChatCompletionParams: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ChatCompletionParamsResult;
             messageExtraction: {
                 kind: "custom";

package/dist/types/ai-model/models/utils/intent.d.ts

@@ -0,0 +1,2 @@
+import type { TIntent } from '@midscene/shared/env';
+export declare function isLocateIntent(intent?: TIntent): boolean;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -13,12 +13,17 @@ export declare class AIResponseParseError extends Error {
 import { type IModelConfig, type TModelFamily } from '@midscene/shared/env';
 import OpenAI from 'openai';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
-import { type ModelRuntime } from '../models';
+import type { ModelRuntime } from '../models';
 import type { AIArgs } from '../types';
 import type { JsonParserSource } from './json';
 import { type OpenAIErrorResponseContext } from './openai-error';
-export { extractJSONFromCodeBlock, normalJsonParser, safeParseJson, } from './json';
+export { extractJSONFromCodeBlock, parseModelResponseJson, } from './json';
 export type { JsonParser } from './json';
+/**
+ * Internal field name stamped onto every AIUsageInfo shaped by callAI().
+ * Used for cross-path dedup when the provider does not return a request_id.
+ */
+export declare const INTERNAL_CALL_ID_FIELD = "_midscene_call_id";
 export declare function createChatClient({ modelConfig, }: {
     modelConfig: IModelConfig;
 }): Promise<{
@@ -41,7 +46,7 @@ export declare function callAI(messages: ChatCompletionMessageParam[], modelRunt
     usage?: AIUsageInfo;
     isStreamed: boolean;
 }>;
-export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], model: IModelConfig | ModelRuntime, options?: {
+export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], modelRuntime: ModelRuntime, options?: {
     abortSignal?: AbortSignal;
     jsonParserSource?: JsonParserSource;
 }): Promise<{

package/dist/types/ai-model/service-caller/json.d.ts

@@ -1,9 +1,26 @@
+/**
+ * Extract the JSON portion from a model response.
+ *
+ * This mainly handles fenced JSON like ```json { ... } ``` by removing the
+ * fence. If that does not produce a JSON object, it falls back to a greedy regex
+ * that extracts from the first "{" to the last "}". If that still fails, the
+ * original response is returned.
+ *
+ * Expected model responses are JSON objects, possibly wrapped in markdown
+ * fences. Natural language mixed with JSON, or arrays like
+ * [{"type":"Tap"}, {"type":"Hover"}], are outside the supported contract and
+ * are not reliably recoverable.
+ *
+ * This legacy extractor is also used by extraction responses that can be any
+ * JSON value. In those cases, arrays/strings/numbers have no object braces and
+ * pass through unchanged.
+ */
 export declare function extractJSONFromCodeBlock(response: string): string;
 export type JsonParserSource = 'generic-object' | 'planning-action-param' | 'locate' | 'section-locator';
 export interface JsonParserContext {
     source: JsonParserSource;
     preserveStringValueKeys?: string[];
+    requireObject?: boolean;
 }
 export type JsonParser = (raw: string, context?: JsonParserContext) => unknown;
-export declare function safeParseJson(raw: string, context?: JsonParserContext): any;
-export declare const normalJsonParser: JsonParser;
+export declare function parseModelResponseJson(raw: string, context?: JsonParserContext): any;

package/dist/types/device/device-options.d.ts

@@ -7,18 +7,6 @@ export type AndroidDeviceInputOpt = {
     autoDismissKeyboard?: boolean;
     /** Strategy for dismissing the keyboard: 'esc-first' tries ESC before BACK, 'back-first' tries BACK before ESC */
     keyboardDismissStrategy?: 'esc-first' | 'back-first';
-    /**
-     * Delay in milliseconds between keystrokes when typing text.
-     *
-     * When set, text is typed one character at a time with this delay between
-     * each character, instead of sending the whole string at once. This helps
-     * on devices or input fields that drop characters when input arrives too
-     * fast (e.g. WiFi password fields on automotive displays).
-     *
-     * Only applies to the `input text` path (non-yadb). When yadb is used, the
-     * entire string is committed atomically and this option is ignored.
-     */
-    keyboardTypeDelay?: number;
 };
 /**
  * Android device options
@@ -121,14 +109,6 @@ export type AndroidDeviceOpt = {
 export type IOSDeviceInputOpt = {
     /** Automatically dismiss the keyboard after input is completed */
     autoDismissKeyboard?: boolean;
-    /**
-     * Delay in milliseconds between keystrokes when typing text.
-     *
-     * When set, text is typed one character at a time with this delay between
-     * each character, instead of sending the whole string at once. This helps
-     * on devices or input fields that drop characters when input arrives too fast.
-     */
-    keyboardTypeDelay?: number;
 };
 /**
  * iOS device options
@@ -166,14 +146,6 @@ export type HarmonyDeviceInputOpt = {
     autoDismissKeyboard?: boolean;
     /** Strategy for dismissing the keyboard. Defaults to 'esc-first'. */
     keyboardDismissStrategy?: 'esc-first' | 'back-first';
-    /**
-     * Delay in milliseconds between keystrokes when typing text.
-     *
-     * When set, text is typed one character at a time with this delay between
-     * each character, instead of sending the whole string at once. This helps
-     * on devices or input fields that drop characters when input arrives too fast.
-     */
-    keyboardTypeDelay?: number;
 };
 /**
  * HarmonyOS device options

package/dist/types/device/index.d.ts

@@ -59,7 +59,6 @@ export interface KeyboardInputPrimitives {
     typeText(value: string, opts?: {
         autoDismissKeyboard?: boolean;
         keyboardDismissStrategy?: 'esc-first' | 'back-first';
-        keyboardTypeDelay?: number;
         target?: unknown;
         replace?: boolean;
         focusOnly?: boolean;
@@ -899,7 +898,6 @@ export declare const actionInputParamSchema: z.ZodObject<{
     }, z.ZodTypeAny, "passthrough">>>;
     mode: z.ZodDefault<z.ZodEnum<["replace", "clear", "typeOnly"]>>;
     autoDismissKeyboard: z.ZodOptional<z.ZodBoolean>;
-    keyboardTypeDelay: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     value: string;
     mode: "replace" | "clear" | "typeOnly";
@@ -941,7 +939,6 @@ export declare const actionInputParamSchema: z.ZodObject<{
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
     autoDismissKeyboard?: boolean | undefined;
-    keyboardTypeDelay?: number | undefined;
 }, {
     value: string | number;
     locate?: z.objectInputType<{
@@ -983,14 +980,12 @@ export declare const actionInputParamSchema: z.ZodObject<{
     }, z.ZodTypeAny, "passthrough"> | undefined;
     mode?: "replace" | "clear" | "typeOnly" | undefined;
     autoDismissKeyboard?: boolean | undefined;
-    keyboardTypeDelay?: number | undefined;
 }>;
 export type ActionInputParam = {
     value: string;
     locate?: LocateResultElement;
     mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
     autoDismissKeyboard?: boolean;
-    keyboardTypeDelay?: number;
 };
 export declare const defineActionInput: (keyboard: KeyboardInputPrimitives) => DeviceAction<ActionInputParam>;
 export declare const actionKeyboardPressParamSchema: z.ZodObject<{

package/dist/types/dump/html-utils.d.ts

@@ -1,5 +1,7 @@
+import type { IReportActionDump } from '../types';
 export declare const escapeContent: (html: string) => string;
 export declare const unescapeContent: (html: string) => string;
+export declare function generateAgentReportComment(report: IReportActionDump): string;
 /** Chunk size for streaming file operations (64KB) */
 export declare const STREAMING_CHUNK_SIZE: number;
 /**

package/dist/types/dump/screenshot-restoration.d.ts

@@ -2,7 +2,7 @@ import { type ScreenshotRef } from './screenshot-store';
 /**
  * Recursively restore image references in parsed data.
  * Replaces ScreenshotRef with lazy
- * { get base64() {...}, capturedAt } objects.
+ * { get base64() {...}, capturedAt, sourceRef } objects.
  * The resolver is only called when .base64 is first accessed.
  */
 export declare function restoreImageReferences<T>(data: T, resolveImage: (ref: ScreenshotRef) => string): T;

package/dist/types/element-describer.d.ts

@@ -5,13 +5,13 @@ export type DescribeElementCoordinateSpace = 'screenshot' | 'logical';
 export type LocatorVerifyFn = (input: {
     prompt: string;
     expectCenter: [number, number];
-    deepLocate: boolean;
     retryCount: number;
     verifyResult: LocateValidatorResult;
 }) => LocateValidatorResult | boolean;
 export type DescribeElementAtPointOptions = {
     verifyPrompt?: boolean;
     retryLimit?: number;
+    deepDescribe?: boolean;
     deepLocate?: boolean;
     locatorVerifyFn?: LocatorVerifyFn;
     screenshotBase64?: string;
@@ -20,6 +20,7 @@ export type DescribeElementAtPointOptions = {
     logicalSize?: Size;
     onProgress?: (progress: {
         prompt?: string;
+        deepDescribe?: boolean;
         deepLocate?: boolean;
         verifyResult?: LocateValidatorResult;
     }) => void;
@@ -31,16 +32,12 @@ type ScreenshotBoundContextOptions = {
     logicalSize?: Size;
 };
 export type VerifyElementDescriptionAtPointOptions = ScreenshotBoundContextOptions & LocatorValidatorOption;
-export type VerifyElementByServiceLocateOptions = VerifyElementDescriptionAtPointOptions & Pick<LocateOption, 'cacheable' | 'deepLocate' | 'xpath'> & {
-    abortSignal?: AbortSignal;
-};
 export type ElementDescriberRuntime = {
     service: Pick<Service, 'describe' | 'locate'>;
     describeModelRuntime: ModelRuntime;
     locateModelRuntime: ModelRuntime;
 };
-export declare function verifyLocator(runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>, prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption): Promise<LocateValidatorResult>;
+export declare function verifyLocator(runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>, prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption & Pick<LocateOption, 'deepLocate'>): Promise<LocateValidatorResult>;
 export declare function describeElementAtPoint(runtime: ElementDescriberRuntime, center: [number, number], opt?: DescribeElementAtPointOptions): Promise<AgentDescribeElementAtPointResult>;
 export declare function verifyElementDescriptionAtPoint(runtime: ElementDescriberRuntime, description: string, center: [number, number], opt?: VerifyElementDescriptionAtPointOptions): Promise<LocateValidatorResult>;
-export declare function verifyElementByServiceLocate(runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>, description: string, center: [number, number], opt?: VerifyElementByServiceLocateOptions): Promise<LocateValidatorResult>;
 export {};

package/dist/types/index.d.ts

@@ -10,8 +10,8 @@ export { z };
 export default Service;
 export { TaskRunner, Service, getVersion };
 export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlConfigResult, MidsceneYamlConfig, MidsceneYamlScriptWebEnv, MidsceneYamlScriptAndroidEnv, MidsceneYamlScriptIOSEnv, MidsceneYamlScriptEnv, LocateOption, DetailedLocateParam, } from './yaml';
-export { Agent, type AgentOpt, type AiActOptions, type GherkinStepKeyword, type RunGherkinScenarioOptions, createAgent, } from './agent';
-export { describeElementAtPoint, verifyElementDescriptionAtPoint, verifyElementByServiceLocate, verifyLocator, type DescribeElementAtPointOptions, type DescribeElementCoordinateSpace, type ElementDescriberRuntime, type VerifyElementDescriptionAtPointOptions, type VerifyElementByServiceLocateOptions, } from './element-describer';
+export { Agent, type AgentOpt, type AiActOptions, type GherkinStepKeyword, type MidsceneUsageMetrics, type RunGherkinScenarioOptions, type UsageBucket, createAgent, } from './agent';
+export { describeElementAtPoint, verifyElementDescriptionAtPoint, verifyLocator, type DescribeElementAtPointOptions, type DescribeElementCoordinateSpace, type ElementDescriberRuntime, type VerifyElementDescriptionAtPointOptions, } from './element-describer';
 export { restoreImageReferences, escapeContent, unescapeContent, parseImageScripts, parseDumpScript, parseDumpScriptAttributes, generateImageScriptTag, generateDumpScriptTag, deriveTaskStatus, deriveCaseStatus, } from './dump';
 export type { TaskStatusFields, DerivedTaskStatus } from './dump';
 export { getTaskSearchArea, getTaskServiceDump, } from './dump/task-service-dump';

package/dist/types/report-generator.d.ts

@@ -41,7 +41,11 @@ export declare class ReportGenerator implements IReportGenerator {
     private initialized;
     private lastExecution?;
     private lastReportMeta?;
+    private executionsByKey;
+    private executionCommentKeyByObject;
+    private executionCommentKeyIndex;
     private reportAttributes;
+    private agentCommentWritten;
     private writeQueue;
     private destroyed;
     constructor(options: {
@@ -83,6 +87,8 @@ export declare class ReportGenerator implements IReportGenerator {
      */
     private writeInlineExecution;
     private writeDirectoryExecution;
+    private appendAgentReportComment;
     private getExecutionLogKey;
+    private getExecutionCommentKey;
     private persistExecutionDumpToFile;
 }

package/dist/types/service/index.d.ts

@@ -21,7 +21,7 @@ export default class Service {
         abortSignal?: AbortSignal;
     }): Promise<ServiceExtractResult<T>>;
     describe(target: Rect | [number, number], modelRuntime: ModelRuntime, opt?: {
-        deepLocate?: boolean;
+        deepDescribe?: boolean;
         context?: UIContext;
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
 }

package/dist/types/service/utils.d.ts

@@ -1,4 +1,23 @@
-import type { AIDescribeElementResponse } from '../types';
-import type { PartialServiceDumpFromSDK, ServiceDump } from '../types';
+import type { AIDescribeElementResponse, PartialServiceDumpFromSDK, Rect, ServiceDump, Size } from '../types';
+export declare const DESCRIBE_POINT_MARKER_MAX_SIZE = 40;
+export declare const DESCRIBE_RECT_MARKER_BORDER_THICKNESS = 1;
+export declare const DESCRIBE_LARGE_RECT_MARKER_BORDER_THICKNESS = 2;
+export declare const DESCRIBE_WIDE_MARKER_INSET_MIN_WIDTH = 100;
+export declare const DESCRIBE_DEEP_CONTEXT_CONFIG: {
+    readonly resize: {
+        readonly cropMaxLongEdge: 1000;
+        readonly cropUpscaleMaxRatio: 2;
+    };
+};
+export type DescribeDeepContextArea = {
+    kind: 'focused';
+    rect: Rect;
+};
+export declare function clampRect(rect: Rect, size: Size): Rect;
+export declare function getDescribeDeepContextAreas(rect: Rect, screenSize: Size): DescribeDeepContextArea[];
+export declare function getRectInCrop(rect: Rect, cropRect: Rect, cropSize: Size): Rect;
+export declare function getDescribeMarkerRect(rect: Rect): Rect;
+export declare function getDescribeMarkerBorderThickness(rect: Rect): number;
+export declare function getDescribeDeepLocateResizeSize(size: Size): Size | undefined;
 export declare function createServiceDump(data: PartialServiceDumpFromSDK): ServiceDump;
 export declare function recoverDescribeResponseFromParseError(error: unknown): Pick<AIDescribeElementResponse, 'description'> | undefined;

package/dist/types/types.d.ts

@@ -84,6 +84,7 @@ export interface LocateValidatorResult {
 export interface AgentDescribeElementAtPointResult {
     prompt: string;
     deepLocate: boolean;
+    deepDescribe: boolean;
     verifyResult?: LocateValidatorResult;
     success: boolean;
     error?: string;
@@ -130,12 +131,12 @@ export type ThinkingLevel = 'off' | 'medium' | 'high';
 export type DeepThinkOption = 'unset' | true | false;
 export interface ServiceTaskInfo {
     durationMs: number;
-    formatResponse?: string;
+    formatResponse?: unknown;
     /**
      * Adapter-extracted content used by Midscene for parsing. This is not the
      * full provider response or choices[0].message.
      */
-    rawResponse?: string;
+    rawResponse?: unknown;
     rawChoiceMessage?: unknown;
     usage?: AIUsageInfo;
     searchArea?: Rect;
@@ -703,6 +704,15 @@ export interface AgentOpt {
      * ```
      */
     createOpenAIClient?: CreateOpenAIClientFn;
+    /**
+     * Called once per LLM call as soon as its usage is available, with the raw
+     * {@link AIUsageInfo} (token counts, model name, intent, request id, etc.).
+     *
+     * Use this for real-time, per-spec cost observability — e.g. push each call
+     * to Langfuse without waiting for the run to finish. For aggregated totals,
+     * read `agent.metrics` instead.
+     */
+    onLLMUsage?: (usage: AIUsageInfo) => void;
 }
 export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
 export interface ReportFileAttributes {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.10.2-beta-20260706032158.0",
+  "version": "1.10.3",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -100,14 +100,14 @@
     "dotenv": "^16.4.5",
     "fetch-socks": "^1.3.0",
     "js-yaml": "4.1.0",
-    "jsonrepair": "3.12.0",
+    "jsonrepair": "3.14.0",
     "mdast-util-from-markdown": "^2.0.2",
     "mdast-util-to-markdown": "^2.1.2",
     "openai": "6.3.0",
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "^3.25.1",
-    "@midscene/shared": "1.10.2-beta-20260706032158.0"
+    "@midscene/shared": "1.10.3"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",