@midscene/core 1.10.2 → 1.10.3

package/dist/types/agent/agent.d.ts

@@ -4,6 +4,7 @@ import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentOpt
 import type { AbstractInterface } from '../device';
 import type { TaskRunner } from '../task-runner';
 import { ModelConfigManager } from '@midscene/shared/env';
+import { type MidsceneUsageMetrics } from './metrics';
 import { type RunGherkinScenarioOptions } from './run-gherkin-scenario';
 import { TaskCache } from './task-cache';
 import { TaskExecutor } from './tasks';
@@ -29,6 +30,9 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     dryMode: boolean;
     onTaskStartTip?: OnTaskStartTip;
     taskCache?: TaskCache;
+    private readonly metricsCollector;
+    private usageCallCounter;
+    private readonly countedUsageKeys;
     private dumpUpdateListeners;
     private readonly progressBus;
     get onDumpUpdate(): ((dump: string, executionDump?: ExecutionDump) => void) | undefined;
@@ -80,6 +84,17 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     setAIActContext(prompt: string): Promise<void>;
     resetDump(): ReportActionDump;
     appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner): void;
+    /**
+     * Fold any not-yet-counted task usage from an execution dump into the
+     * instance metrics. Snapshots are re-emitted as tasks progress, so each
+     * usage value is keyed by `${taskId}:${field}` and counted at most once.
+     */
+    private collectUsageMetrics;
+    private consumeUsage;
+    /**
+     * Aggregated LLM usage accumulated by this agent since it was created.
+     */
+    get metrics(): MidsceneUsageMetrics;
     dumpDataString(opt?: {
         inlineScreenshots?: boolean;
     }): string;

package/dist/types/agent/index.d.ts

@@ -5,6 +5,7 @@ export { extractInsightParam, locateParamStr, paramStr, taskTitleStr, typeStr, }
 export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
 export { cacheFileExt } from './task-cache';
 export { TaskExecutor } from './tasks';
+export type { MidsceneUsageMetrics, UsageBucket } from './metrics';
 export type { GherkinStepKeyword, RunGherkinScenarioOptions, } from './run-gherkin-scenario';
 export type { AgentOpt } from '../types';
 export type { RecordToReportOptions, RecordToReportScreenshot } from '../types';

package/dist/types/agent/metrics.d.ts

@@ -0,0 +1,48 @@
+import type { AIUsageInfo } from '../types';
+/**
+ * Aggregated usage for a single grouping key (intent or model).
+ */
+export interface UsageBucket {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    calls: number;
+}
+/**
+ * Instance-level snapshot of LLM usage accumulated by an Agent.
+ *
+ * Designed for cost observability: read it back after a logical unit of work
+ * and push to tools like Langfuse. `byIntent` / `byModel` provide free
+ * breakdowns derived from the usage data Midscene already records per call.
+ */
+export interface MidsceneUsageMetrics {
+    totalPromptTokens: number;
+    totalCompletionTokens: number;
+    totalTokens: number;
+    totalCachedInput: number;
+    totalTimeCostMs: number;
+    calls: number;
+    byIntent: Record<string, UsageBucket>;
+    byModel: Record<string, UsageBucket>;
+}
+/**
+ * Pure accumulator for {@link AIUsageInfo}. Deduplication of calls is the
+ * caller's responsibility; every `add` counts as one call.
+ */
+export declare class MetricsCollector {
+    private totalPromptTokens;
+    private totalCompletionTokens;
+    private totalTokens;
+    private totalCachedInput;
+    private totalTimeCostMs;
+    private calls;
+    private byIntent;
+    private byModel;
+    add(usage: AIUsageInfo): void;
+    snapshot(): MidsceneUsageMetrics;
+    /**
+     * Clear all accumulated state. Not wired to a public Agent API yet; kept for
+     * a future per-unit reset (e.g. resetting metrics at the start of a spec).
+     */
+    reset(): void;
+}

package/dist/types/ai-model/model-adapter/types.d.ts

@@ -1,4 +1,5 @@
 import type { PixelBbox, PlanningAction } from '../../types';
+import type { AIUsageInfo } from '../../types';
 import type { IModelConfig, TIntent, TModelReasoningEnabled } from '@midscene/shared/env';
 import type OpenAI from 'openai';
 import type { JsonParser, JsonParserContext, JsonParserSource } from '../service-caller/json';
@@ -152,6 +153,13 @@ export interface ModelAdapter {
 export interface ModelRuntime {
     config: IModelConfig;
     adapter: ModelAdapter;
+    /**
+     * Optional callback fired after every underlying model call with the shaped
+     * usage info. Provides a single collection point for callers that want to
+     * aggregate usage across all model invocations (including auxiliary calls
+     * such as order-sensitive judging and deep-locate search-area calls).
+     */
+    onUsage?: (usage: AIUsageInfo) => void;
 }
 export interface ModelAdapterDefinition {
     jsonParser?: JsonParserPreset | JsonParser;

package/dist/types/ai-model/models/doubao.d.ts

@@ -1,8 +1,5 @@
-import type { JsonParserContext, JsonParserSource, ModelAdapterDefinition } from '../model-adapter/types';
+import type { ModelAdapterDefinition } from '../model-adapter/types';
 import { type LocateResultValue } from '../shared/model-locate-result';
-export declare function normalizeDoubaoJsonObject(obj: any, context?: Pick<JsonParserContext, 'preserveStringValueKeys'>): any;
-export declare function shouldRepairDoubaoLocateJson(source: JsonParserSource): source is "planning-action-param" | "locate" | "section-locator";
-export declare function preprocessDoubaoLocateJson(input: string): string;
 export declare function parseDoubaoRawLocateValue(input: unknown): LocateResultValue;
 export declare const doubaoAdapters: {
     'doubao-vision': ModelAdapterDefinition;

package/dist/types/ai-model/models/utils/intent.d.ts

@@ -0,0 +1,2 @@
+import type { TIntent } from '@midscene/shared/env';
+export declare function isLocateIntent(intent?: TIntent): boolean;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -13,12 +13,17 @@ export declare class AIResponseParseError extends Error {
 import { type IModelConfig, type TModelFamily } from '@midscene/shared/env';
 import OpenAI from 'openai';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
-import { type ModelRuntime } from '../models';
+import type { ModelRuntime } from '../models';
 import type { AIArgs } from '../types';
 import type { JsonParserSource } from './json';
 import { type OpenAIErrorResponseContext } from './openai-error';
-export { extractJSONFromCodeBlock, normalJsonParser, safeParseJson, } from './json';
+export { extractJSONFromCodeBlock, parseModelResponseJson, } from './json';
 export type { JsonParser } from './json';
+/**
+ * Internal field name stamped onto every AIUsageInfo shaped by callAI().
+ * Used for cross-path dedup when the provider does not return a request_id.
+ */
+export declare const INTERNAL_CALL_ID_FIELD = "_midscene_call_id";
 export declare function createChatClient({ modelConfig, }: {
     modelConfig: IModelConfig;
 }): Promise<{
@@ -41,7 +46,7 @@ export declare function callAI(messages: ChatCompletionMessageParam[], modelRunt
     usage?: AIUsageInfo;
     isStreamed: boolean;
 }>;
-export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], model: IModelConfig | ModelRuntime, options?: {
+export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], modelRuntime: ModelRuntime, options?: {
     abortSignal?: AbortSignal;
     jsonParserSource?: JsonParserSource;
 }): Promise<{

package/dist/types/ai-model/service-caller/json.d.ts

@@ -1,9 +1,26 @@
+/**
+ * Extract the JSON portion from a model response.
+ *
+ * This mainly handles fenced JSON like ```json { ... } ``` by removing the
+ * fence. If that does not produce a JSON object, it falls back to a greedy regex
+ * that extracts from the first "{" to the last "}". If that still fails, the
+ * original response is returned.
+ *
+ * Expected model responses are JSON objects, possibly wrapped in markdown
+ * fences. Natural language mixed with JSON, or arrays like
+ * [{"type":"Tap"}, {"type":"Hover"}], are outside the supported contract and
+ * are not reliably recoverable.
+ *
+ * This legacy extractor is also used by extraction responses that can be any
+ * JSON value. In those cases, arrays/strings/numbers have no object braces and
+ * pass through unchanged.
+ */
 export declare function extractJSONFromCodeBlock(response: string): string;
 export type JsonParserSource = 'generic-object' | 'planning-action-param' | 'locate' | 'section-locator';
 export interface JsonParserContext {
     source: JsonParserSource;
     preserveStringValueKeys?: string[];
+    requireObject?: boolean;
 }
 export type JsonParser = (raw: string, context?: JsonParserContext) => unknown;
-export declare function safeParseJson(raw: string, context?: JsonParserContext): any;
-export declare const normalJsonParser: JsonParser;
+export declare function parseModelResponseJson(raw: string, context?: JsonParserContext): any;

package/dist/types/dump/html-utils.d.ts

@@ -1,5 +1,7 @@
+import type { IReportActionDump } from '../types';
 export declare const escapeContent: (html: string) => string;
 export declare const unescapeContent: (html: string) => string;
+export declare function generateAgentReportComment(report: IReportActionDump): string;
 /** Chunk size for streaming file operations (64KB) */
 export declare const STREAMING_CHUNK_SIZE: number;
 /**

package/dist/types/dump/screenshot-restoration.d.ts

@@ -2,7 +2,7 @@ import { type ScreenshotRef } from './screenshot-store';
 /**
  * Recursively restore image references in parsed data.
  * Replaces ScreenshotRef with lazy
- * { get base64() {...}, capturedAt } objects.
+ * { get base64() {...}, capturedAt, sourceRef } objects.
  * The resolver is only called when .base64 is first accessed.
  */
 export declare function restoreImageReferences<T>(data: T, resolveImage: (ref: ScreenshotRef) => string): T;

package/dist/types/index.d.ts

@@ -10,7 +10,7 @@ export { z };
 export default Service;
 export { TaskRunner, Service, getVersion };
 export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlConfigResult, MidsceneYamlConfig, MidsceneYamlScriptWebEnv, MidsceneYamlScriptAndroidEnv, MidsceneYamlScriptIOSEnv, MidsceneYamlScriptEnv, LocateOption, DetailedLocateParam, } from './yaml';
-export { Agent, type AgentOpt, type AiActOptions, type GherkinStepKeyword, type RunGherkinScenarioOptions, createAgent, } from './agent';
+export { Agent, type AgentOpt, type AiActOptions, type GherkinStepKeyword, type MidsceneUsageMetrics, type RunGherkinScenarioOptions, type UsageBucket, createAgent, } from './agent';
 export { describeElementAtPoint, verifyElementDescriptionAtPoint, verifyLocator, type DescribeElementAtPointOptions, type DescribeElementCoordinateSpace, type ElementDescriberRuntime, type VerifyElementDescriptionAtPointOptions, } from './element-describer';
 export { restoreImageReferences, escapeContent, unescapeContent, parseImageScripts, parseDumpScript, parseDumpScriptAttributes, generateImageScriptTag, generateDumpScriptTag, deriveTaskStatus, deriveCaseStatus, } from './dump';
 export type { TaskStatusFields, DerivedTaskStatus } from './dump';

package/dist/types/report-generator.d.ts

@@ -41,7 +41,11 @@ export declare class ReportGenerator implements IReportGenerator {
     private initialized;
     private lastExecution?;
     private lastReportMeta?;
+    private executionsByKey;
+    private executionCommentKeyByObject;
+    private executionCommentKeyIndex;
     private reportAttributes;
+    private agentCommentWritten;
     private writeQueue;
     private destroyed;
     constructor(options: {
@@ -83,6 +87,8 @@ export declare class ReportGenerator implements IReportGenerator {
      */
     private writeInlineExecution;
     private writeDirectoryExecution;
+    private appendAgentReportComment;
     private getExecutionLogKey;
+    private getExecutionCommentKey;
     private persistExecutionDumpToFile;
 }

package/dist/types/types.d.ts

@@ -131,12 +131,12 @@ export type ThinkingLevel = 'off' | 'medium' | 'high';
 export type DeepThinkOption = 'unset' | true | false;
 export interface ServiceTaskInfo {
     durationMs: number;
-    formatResponse?: string;
+    formatResponse?: unknown;
     /**
      * Adapter-extracted content used by Midscene for parsing. This is not the
      * full provider response or choices[0].message.
      */
-    rawResponse?: string;
+    rawResponse?: unknown;
     rawChoiceMessage?: unknown;
     usage?: AIUsageInfo;
     searchArea?: Rect;
@@ -704,6 +704,15 @@ export interface AgentOpt {
      * ```
      */
     createOpenAIClient?: CreateOpenAIClientFn;
+    /**
+     * Called once per LLM call as soon as its usage is available, with the raw
+     * {@link AIUsageInfo} (token counts, model name, intent, request id, etc.).
+     *
+     * Use this for real-time, per-spec cost observability — e.g. push each call
+     * to Langfuse without waiting for the run to finish. For aggregated totals,
+     * read `agent.metrics` instead.
+     */
+    onLLMUsage?: (usage: AIUsageInfo) => void;
 }
 export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
 export interface ReportFileAttributes {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.10.2",
+  "version": "1.10.3",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -100,14 +100,14 @@
     "dotenv": "^16.4.5",
     "fetch-socks": "^1.3.0",
     "js-yaml": "4.1.0",
-    "jsonrepair": "3.12.0",
+    "jsonrepair": "3.14.0",
     "mdast-util-from-markdown": "^2.0.2",
     "mdast-util-to-markdown": "^2.1.2",
     "openai": "6.3.0",
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "^3.25.1",
-    "@midscene/shared": "1.10.2"
+    "@midscene/shared": "1.10.3"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",