@midscene/core 1.9.8 → 1.10.0

package/dist/types/element-describer.d.ts

@@ -0,0 +1,46 @@
+import type { ModelRuntime } from './ai-model/models';
+import type Service from './service';
+import type { AgentDescribeElementAtPointResult, LocateOption, LocateValidatorResult, LocatorValidatorOption, Size } from './types';
+export type DescribeElementCoordinateSpace = 'screenshot' | 'logical';
+export type LocatorVerifyFn = (input: {
+    prompt: string;
+    expectCenter: [number, number];
+    deepLocate: boolean;
+    retryCount: number;
+    verifyResult: LocateValidatorResult;
+}) => LocateValidatorResult | boolean;
+export type DescribeElementAtPointOptions = {
+    verifyPrompt?: boolean;
+    retryLimit?: number;
+    deepLocate?: boolean;
+    locatorVerifyFn?: LocatorVerifyFn;
+    screenshotBase64?: string;
+    screenshotSize?: Size;
+    coordinateSpace?: DescribeElementCoordinateSpace;
+    logicalSize?: Size;
+    onProgress?: (progress: {
+        prompt?: string;
+        deepLocate?: boolean;
+        verifyResult?: LocateValidatorResult;
+    }) => void;
+} & LocatorValidatorOption;
+type ScreenshotBoundContextOptions = {
+    screenshotBase64?: string;
+    screenshotSize?: Size;
+    coordinateSpace?: DescribeElementCoordinateSpace;
+    logicalSize?: Size;
+};
+export type VerifyElementDescriptionAtPointOptions = ScreenshotBoundContextOptions & LocatorValidatorOption;
+export type VerifyElementByServiceLocateOptions = VerifyElementDescriptionAtPointOptions & Pick<LocateOption, 'cacheable' | 'deepLocate' | 'xpath'> & {
+    abortSignal?: AbortSignal;
+};
+export type ElementDescriberRuntime = {
+    service: Pick<Service, 'describe' | 'locate'>;
+    describeModelRuntime: ModelRuntime;
+    locateModelRuntime: ModelRuntime;
+};
+export declare function verifyLocator(runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>, prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption): Promise<LocateValidatorResult>;
+export declare function describeElementAtPoint(runtime: ElementDescriberRuntime, center: [number, number], opt?: DescribeElementAtPointOptions): Promise<AgentDescribeElementAtPointResult>;
+export declare function verifyElementDescriptionAtPoint(runtime: ElementDescriberRuntime, description: string, center: [number, number], opt?: VerifyElementDescriptionAtPointOptions): Promise<LocateValidatorResult>;
+export declare function verifyElementByServiceLocate(runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>, description: string, center: [number, number], opt?: VerifyElementByServiceLocateOptions): Promise<LocateValidatorResult>;
+export {};

package/dist/types/index.d.ts

@@ -10,7 +10,8 @@ export { z };
 export default Service;
 export { TaskRunner, Service, getVersion };
 export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlConfigResult, MidsceneYamlConfig, MidsceneYamlScriptWebEnv, MidsceneYamlScriptAndroidEnv, MidsceneYamlScriptIOSEnv, MidsceneYamlScriptEnv, LocateOption, DetailedLocateParam, } from './yaml';
-export { Agent, type AgentOpt, type AiActOptions, createAgent, } from './agent';
+export { Agent, type AgentOpt, type AiActOptions, type GherkinStepKeyword, type RunGherkinScenarioOptions, createAgent, } from './agent';
+export { describeElementAtPoint, verifyElementDescriptionAtPoint, verifyElementByServiceLocate, verifyLocator, type DescribeElementAtPointOptions, type DescribeElementCoordinateSpace, type ElementDescriberRuntime, type VerifyElementDescriptionAtPointOptions, type VerifyElementByServiceLocateOptions, } from './element-describer';
 export { restoreImageReferences, escapeContent, unescapeContent, parseImageScripts, parseDumpScript, parseDumpScriptAttributes, generateImageScriptTag, generateDumpScriptTag, } from './dump';
 export { getTaskSearchArea, getTaskServiceDump, } from './dump/task-service-dump';
 export type { IReportGenerator } from './report-generator';

package/dist/types/service/index.d.ts

@@ -17,7 +17,9 @@ export default class Service {
     constructor(context: UIContext | (() => Promise<UIContext> | UIContext), opt?: ServiceOptions);
     locate(query: PlanningLocateParam, opt: LocateOpts, modelRuntime: ModelRuntime, abortSignal?: AbortSignal): Promise<LocateResultWithDump>;
     private resolveLocateSearchArea;
-    extract<T>(dataDemand: ServiceExtractParam, modelRuntime: ModelRuntime, opt?: ServiceExtractOption, pageDescription?: string, multimodalPrompt?: TMultimodalPrompt, context?: UIContext): Promise<ServiceExtractResult<T>>;
+    extract<T>(dataDemand: ServiceExtractParam, modelRuntime: ModelRuntime, opt?: ServiceExtractOption, pageDescription?: string, multimodalPrompt?: TMultimodalPrompt, context?: UIContext, executionOptions?: {
+        abortSignal?: AbortSignal;
+    }): Promise<ServiceExtractResult<T>>;
     describe(target: Rect | [number, number], modelRuntime: ModelRuntime, opt?: {
         deepLocate?: boolean;
         context?: UIContext;

package/dist/types/types.d.ts

@@ -85,6 +85,9 @@ export interface AgentDescribeElementAtPointResult {
     prompt: string;
     deepLocate: boolean;
     verifyResult?: LocateValidatorResult;
+    success: boolean;
+    error?: string;
+    failureStage?: 'describe' | 'verify';
 }
 /**
  * context
@@ -202,6 +205,8 @@ export interface AgentWaitForOpt extends ServiceExtractOption {
 }
 export interface AgentAssertOpt {
     keepRawResponse?: boolean;
+    context?: string;
+    abortSignal?: AbortSignal;
 }
 /**
  * planning

package/dist/types/yaml.d.ts

@@ -191,6 +191,9 @@ export interface MidsceneYamlFlowItemAIWaitFor extends ServiceExtractOption {
     aiWaitFor: string;
     timeout?: number;
 }
+export interface MidsceneYamlFlowItemRunGherkinScenario {
+    runGherkinScenario: string;
+}
 export interface MidsceneYamlFlowItemEvaluateJavaScript {
     javascript: string;
     name?: string;
@@ -203,7 +206,7 @@ export interface MidsceneYamlFlowItemLogScreenshot {
     recordToReport?: string;
     content?: string;
 }
-export type MidsceneYamlFlowItem = MidsceneYamlFlowItemAIAction | MidsceneYamlFlowItemAIAssert | MidsceneYamlFlowItemAIWaitFor | MidsceneYamlFlowItemEvaluateJavaScript | MidsceneYamlFlowItemSleep | MidsceneYamlFlowItemLogScreenshot;
+export type MidsceneYamlFlowItem = MidsceneYamlFlowItemAIAction | MidsceneYamlFlowItemAIAssert | MidsceneYamlFlowItemAIWaitFor | MidsceneYamlFlowItemRunGherkinScenario | MidsceneYamlFlowItemEvaluateJavaScript | MidsceneYamlFlowItemSleep | MidsceneYamlFlowItemLogScreenshot;
 export interface FreeFn {
     name: string;
     fn: () => void;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.9.8",
+  "version": "1.10.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -89,7 +89,7 @@
       "yaml": [
         "./dist/types/yaml.d.ts"
       ],
-      "mcp": [
+      "skill": [
         "./dist/types/skill/index.d.ts"
       ]
     }
@@ -107,7 +107,7 @@
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "^3.25.1",
-    "@midscene/shared": "1.9.8"
+    "@midscene/shared": "1.10.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",

package/dist/es/ai-model/models/chat-content.mjs

@@ -1,14 +0,0 @@
-function defaultExtractContentAndReasoning(message) {
-    if (!message) return {
-        content: '',
-        reasoning_content: ''
-    };
-    const messageReasoning = 'string' == typeof message.reasoning_content ? message.reasoning_content : '';
-    return {
-        content: 'string' == typeof message.content ? message.content : '',
-        reasoning_content: messageReasoning
-    };
-}
-export { defaultExtractContentAndReasoning };
-
-//# sourceMappingURL=chat-content.mjs.map

package/dist/es/ai-model/models/chat-content.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"ai-model/models/chat-content.mjs","sources":["../../../../src/ai-model/models/chat-content.ts"],"sourcesContent":["import type { ChatCompletionContentSource, ContentAndReasoning } from './types';\n\nexport function defaultExtractContentAndReasoning(\n  message: ChatCompletionContentSource | undefined,\n): ContentAndReasoning {\n  if (!message) {\n    return {\n      content: '',\n      reasoning_content: '',\n    };\n  }\n\n  const messageReasoning =\n    typeof message.reasoning_content === 'string'\n      ? message.reasoning_content\n      : '';\n\n  return {\n    content: typeof message.content === 'string' ? message.content : '',\n    reasoning_content: messageReasoning,\n  };\n}\n"],"names":["defaultExtractContentAndReasoning","message","messageReasoning"],"mappings":"AAEO,SAASA,kCACdC,OAAgD;IAEhD,IAAI,CAACA,SACH,OAAO;QACL,SAAS;QACT,mBAAmB;IACrB;IAGF,MAAMC,mBACJ,AAAqC,YAArC,OAAOD,QAAQ,iBAAiB,GAC5BA,QAAQ,iBAAiB,GACzB;IAEN,OAAO;QACL,SAAS,AAA2B,YAA3B,OAAOA,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;QACjE,mBAAmBC;IACrB;AACF"}

package/dist/es/ai-model/models/resolved.mjs

@@ -1,107 +0,0 @@
-import { normalJsonParser } from "../service-caller/json.mjs";
-import { createLocateResultAdapter } from "../shared/model-locate-result/factory.mjs";
-import { defaultExtractContentAndReasoning } from "./chat-content.mjs";
-function _define_property(obj, key, value) {
-    if (key in obj) Object.defineProperty(obj, key, {
-        value: value,
-        enumerable: true,
-        configurable: true,
-        writable: true
-    });
-    else obj[key] = value;
-    return obj;
-}
-const defaultReplanningCycleLimit = 20;
-const defaultImageDetail = (_input)=>void 0;
-const defaultChatCompletionParams = ({ midsceneDefaults, userConfig })=>({
-        config: {
-            temperature: userConfig.temperature ?? midsceneDefaults.temperature
-        }
-    });
-const midsceneChatCompletionDefaults = {
-    temperature: 0
-};
-const defaultLocateResultAdapterDefinition = {
-    coordinates: {
-        shape: 'bbox',
-        order: 'xy',
-        normalizedBy: 1000
-    }
-};
-function resolveJsonParser(jsonParser) {
-    if (!jsonParser || 'lenient-json' === jsonParser) return normalJsonParser;
-    if ('function' == typeof jsonParser) return jsonParser;
-    throw new Error(`Unknown json parser preset: ${jsonParser}`);
-}
-function resolveChatCompletion(chatCompletion) {
-    const buildChatCompletionParams = chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;
-    const resolveImageDetail = chatCompletion?.resolveImageDetail ?? defaultImageDetail;
-    const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];
-    const extractContentAndReasoning = chatCompletion?.extractContentAndReasoning ?? defaultExtractContentAndReasoning;
-    return {
-        unsupportedUserConfig,
-        buildChatCompletionParams: (input)=>{
-            const context = {
-                ...input,
-                userConfig: input.userConfig ?? {},
-                midsceneDefaults: midsceneChatCompletionDefaults
-            };
-            return buildChatCompletionParams(context);
-        },
-        resolveImageDetail: (input)=>resolveImageDetail({
-                ...input,
-                userConfig: input.userConfig ?? {},
-                midsceneDefaults: midsceneChatCompletionDefaults
-            }),
-        extractContentAndReasoning
-    };
-}
-function resolveImagePreprocess(imagePreprocess) {
-    return {
-        padBlockSize: imagePreprocess?.padBlockSize
-    };
-}
-function resolvePlanning(planning) {
-    if (planning?.kind === 'custom') return {
-        kind: 'custom',
-        cacheEnabled: planning.cacheEnabled ?? true,
-        defaultReplanningCycleLimit: planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
-        supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,
-        planFn: planning.planFn
-    };
-    return {
-        kind: 'standard',
-        cacheEnabled: planning?.cacheEnabled ?? true,
-        defaultReplanningCycleLimit: planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
-        supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true
-    };
-}
-function resolveLocate(locate) {
-    if (locate?.kind === 'custom') return {
-        kind: 'custom',
-        supportsSearchArea: locate.supportsSearchArea ?? false,
-        locateFn: locate.locateFn
-    };
-    return {
-        kind: 'standard',
-        supportsSearchArea: locate?.supportsSearchArea ?? true,
-        resultAdapter: createLocateResultAdapter(locate?.resultAdapter ?? defaultLocateResultAdapterDefinition)
-    };
-}
-class ResolvedModelAdapter {
-    constructor(config, modelFamily){
-        _define_property(this, "jsonParser", void 0);
-        _define_property(this, "chatCompletion", void 0);
-        _define_property(this, "imagePreprocess", void 0);
-        _define_property(this, "planning", void 0);
-        _define_property(this, "locate", void 0);
-        this.jsonParser = resolveJsonParser(config.jsonParser);
-        this.chatCompletion = resolveChatCompletion(config.chatCompletion);
-        this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);
-        this.planning = resolvePlanning(config.planning);
-        this.locate = resolveLocate(config.locate);
-    }
-}
-export { ResolvedModelAdapter };
-
-//# sourceMappingURL=resolved.mjs.map

package/dist/es/ai-model/models/resolved.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"ai-model/models/resolved.mjs","sources":["../../../../src/ai-model/models/resolved.ts"],"sourcesContent":["import { normalJsonParser } from '../service-caller/json';\nimport { createLocateResultAdapter } from '../shared/model-locate-result/factory';\nimport type { LocateResultAdapterDefinition } from '../shared/model-locate-result/types';\nimport { defaultExtractContentAndReasoning } from './chat-content';\nimport type {\n  ChatCompletionAdapter,\n  ChatCompletionCallContext,\n  ChatCompletionCallInput,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  MidsceneChatCompletionDefaults,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n} from './types';\n\nconst defaultReplanningCycleLimit = 20;\n\nconst defaultImageDetail = (_input: unknown) => undefined;\n\nconst defaultChatCompletionParams = ({\n  midsceneDefaults,\n  userConfig,\n}: ChatCompletionCallContext) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n  temperature: 0,\n};\n\nconst defaultLocateResultAdapterDefinition: LocateResultAdapterDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return normalJsonParser;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveChatCompletion(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n  const buildChatCompletionParams =\n    chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n  const resolveImageDetail =\n    chatCompletion?.resolveImageDetail ?? defaultImageDetail;\n  const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n  const extractContentAndReasoning =\n    chatCompletion?.extractContentAndReasoning ??\n    defaultExtractContentAndReasoning;\n\n  return {\n    unsupportedUserConfig,\n    buildChatCompletionParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      };\n      return buildChatCompletionParams(context);\n    },\n    resolveImageDetail: (input) =>\n      resolveImageDetail({\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      }),\n    extractContentAndReasoning,\n  };\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nfunction resolvePlanning(\n  planning: ModelAdapterDefinition['planning'],\n): PlanningAdapter {\n  if (planning?.kind === 'custom') {\n    return {\n      kind: 'custom',\n      cacheEnabled: planning.cacheEnabled ?? true,\n      defaultReplanningCycleLimit:\n        planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n      supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n      planFn: planning.planFn,\n    };\n  }\n\n  return {\n    kind: 'standard',\n    cacheEnabled: planning?.cacheEnabled ?? true,\n    defaultReplanningCycleLimit:\n      planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n    supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true,\n  };\n}\n\nfunction resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    return {\n      kind: 'custom',\n      supportsSearchArea: locate.supportsSearchArea ?? false,\n      locateFn: locate.locateFn,\n    };\n  }\n\n  return {\n    kind: 'standard',\n    supportsSearchArea: locate?.supportsSearchArea ?? true,\n    resultAdapter: createLocateResultAdapter(\n      locate?.resultAdapter ?? defaultLocateResultAdapterDefinition,\n    ),\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    this.planning = resolvePlanning(config.planning);\n    this.locate = resolveLocate(config.locate);\n  }\n}\n"],"names":["defaultReplanningCycleLimit","defaultImageDetail","_input","undefined","defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","defaultLocateResultAdapterDefinition","resolveJsonParser","jsonParser","normalJsonParser","Error","resolveChatCompletion","chatCompletion","buildChatCompletionParams","resolveImageDetail","unsupportedUserConfig","extractContentAndReasoning","defaultExtractContentAndReasoning","input","context","resolveImagePreprocess","imagePreprocess","resolvePlanning","planning","resolveLocate","locate","createLocateResultAdapter","ResolvedModelAdapter","config","modelFamily"],"mappings":";;;;;;;;;;;;;AAiBA,MAAMA,8BAA8B;AAEpC,MAAMC,qBAAqB,CAACC,SAAoBC;AAEhD,MAAMC,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEA,MAAMC,uCAAsE;IAC1E,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEA,SAASC,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,sBACPC,cAAwD;IAExD,MAAMC,4BACJD,gBAAgB,6BAA6BV;IAC/C,MAAMY,qBACJF,gBAAgB,sBAAsBb;IACxC,MAAMgB,wBAAwBH,gBAAgB,yBAAyB,EAAE;IACzE,MAAMI,6BACJJ,gBAAgB,8BAChBK;IAEF,OAAO;QACLF;QACA,2BAA2B,CAACG;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBb;YACpB;YACA,OAAOQ,0BAA0BM;QACnC;QACA,oBAAoB,CAACD,QACnBJ,mBAAmB;gBACjB,GAAGI,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBb;YACpB;QACFW;IACF;AACF;AAEA,SAASI,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEA,SAASC,gBACPC,QAA4C;IAE5C,IAAIA,UAAU,SAAS,UACrB,OAAO;QACL,MAAM;QACN,cAAcA,SAAS,YAAY,IAAI;QACvC,6BACEA,SAAS,2BAA2B,IAAIzB;QAC1C,0BAA0ByB,SAAS,wBAAwB,IAAI;QAC/D,QAAQA,SAAS,MAAM;IACzB;IAGF,OAAO;QACL,MAAM;QACN,cAAcA,UAAU,gBAAgB;QACxC,6BACEA,UAAU,+BAA+BzB;QAC3C,0BAA0ByB,UAAU,4BAA4B;IAClE;AACF;AAEA,SAASC,cACPC,MAAwC;IAExC,IAAIA,QAAQ,SAAS,UACnB,OAAO;QACL,MAAM;QACN,oBAAoBA,OAAO,kBAAkB,IAAI;QACjD,UAAUA,OAAO,QAAQ;IAC3B;IAGF,OAAO;QACL,MAAM;QACN,oBAAoBA,QAAQ,sBAAsB;QAClD,eAAeC,0BACbD,QAAQ,iBAAiBnB;IAE7B;AACF;AAEO,MAAMqB;IAOX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QANjE,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,UAAU,GAAGtB,kBAAkBqB,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGjB,sBAAsBiB,OAAO,cAAc;QACjE,IAAI,CAAC,eAAe,GAAGR,uBAAuBQ,OAAO,eAAe;QACpE,IAAI,CAAC,QAAQ,GAAGN,gBAAgBM,OAAO,QAAQ;QAC/C,IAAI,CAAC,MAAM,GAAGJ,cAAcI,OAAO,MAAM;IAC3C;AACF"}

package/dist/es/ai-model/prompt/recorder-ui-describer.mjs

@@ -1,56 +0,0 @@
-const RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = `You convert Studio preview recorder UI events into semantic replay instructions.
-
-The recorder works from screenshots and mapped real-device coordinates only. Infer stable UI intent from the highlighted BEFORE screenshot. The AFTER screenshot is contextual evidence for state changes and scroll destinations.
-
-Output JSON only:
-{
-  "elementDescription": "short stable target/region description",
-  "replayInstruction": "one executable natural-language replay step",
-  "actionSummary": "short timeline summary",
-  "scrollDestinationDescription": "for scroll only: concrete newly visible destination content or goal",
-  "confidence": "high" | "medium" | "low",
-  "error"?: "only if no useful visual description can be inferred"
-}
-
-Rules:
-- Do NOT output coordinates as the main description.
-- Do NOT mention "near coordinates", "nearby element", "near point", "red marker", highlighted box, highlighted element, or screenshot.
-- Prefer stable target descriptions in this order: exact stable control text > stable label > role + stable section/context > icon purpose > visual position.
-- Treat placeholder or hint text that can change by user, time, data, or context as dynamic. Do not use dynamic hint values as the primary target description; prefer role + stable region + intent.
-- For repeated collections, treat item identity text as dynamic unless the user is clearly verifying that exact item. This includes any list/grid/table/feed/menu or repeated record surface. Do not output descriptions like "<role> titled/named '<content>'"; prefer stable role + region + selection intent.
-- Keep quoted UI text in the original UI language when it is a stable control label.
-- Apply the platform guidance from the user event:
-  - Web: button, input, link, menu item, tab, dialog, aria-label, placeholder, form section.
-  - Mobile: tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, screen section.
-  - Desktop/computer: menu item, toolbar button, dialog field, sidebar item, window control, file row, application region.
-- Pointer action rules:
-  - Preserve event.actionType semantics. Tap, DoubleClick, LongPress, and RightClick must not all become Click.
-  - Tap replayInstruction format: Tap on the element described as "<elementDescription>".
-  - DoubleClick replayInstruction format: Double click on the element described as "<elementDescription>".
-  - LongPress replayInstruction format: Long press the element described as "<elementDescription>".
-  - RightClick replayInstruction format: Right click on the element described as "<elementDescription>".
-  - Click replayInstruction format: Click on the element described as "<elementDescription>".
-- Input-specific rules:
-  - The highlighted BEFORE screenshot marks the field that receives the text.
-  - The screenshot after the action may show the typed value; use it only to confirm the field, never as the field description.
-  - elementDescription must identify the field itself by stable field role, nearby label, region, section, or sequence intent.
-  - Never use "AI is analyzing element", the typed value, the page title alone, or a generic "input field" as elementDescription.
-  - For consecutive input events, distinguish fields by stable role, section, order, current focus, filled/empty state, and neighboring actions instead of reusing the same generic field description.
-  - Input replayInstruction format: Input "<value>" into the element described as "<elementDescription>".
-- Scroll target quality bar:
-  - elementDescription describes the scrollable page, panel, list, table, or section at the highlighted scroll point.
-  - When multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu. Do not generalize a panel/list scroll into the whole page.
-  - scrollDestinationDescription is required and describes what the scroll is trying to reveal or reach, using newly visible headings, section titles, list items, or stable content from the AFTER screenshot.
-  - Prefer descriptions like "Playwright integration documentation page, scrolling toward the API reference section" or "Android API documentation page, scrolling to the installation steps section".
-  - Do NOT write generic phrases like "more content", "the page", "current screen", or "main scrollable area".
-- Scroll replayInstruction format: Scroll the page/region with description "<elementDescription>" by value "<recorded value>" until "<scrollDestinationDescription>" is visible.
-- Scroll actionSummary format: Scroll <elementDescription> toward <scrollDestinationDescription>.
-- Drag/Swipe rules:
-  - Drag replayInstruction format: Drag through the area described as "<elementDescription>".
-  - Swipe replayInstruction format: Swipe through the area described as "<elementDescription>".
-  - Describe start/end regions or the dragged UI control; do not describe only the gesture path.
-- KeyboardPress replayInstruction format: Press "<value>" on the element described as "<elementDescription>".
-- If uncertain, provide the best concrete visible text/role/context description. Set confidence to "low"; do not fall back to coordinates.`;
-export { RECORDER_UI_DESCRIBER_SYSTEM_PROMPT };
-
-//# sourceMappingURL=recorder-ui-describer.mjs.map

package/dist/es/ai-model/prompt/recorder-ui-describer.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"ai-model/prompt/recorder-ui-describer.mjs","sources":["../../../../src/ai-model/prompt/recorder-ui-describer.ts"],"sourcesContent":["export const RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = `You convert Studio preview recorder UI events into semantic replay instructions.\n\nThe recorder works from screenshots and mapped real-device coordinates only. Infer stable UI intent from the highlighted BEFORE screenshot. The AFTER screenshot is contextual evidence for state changes and scroll destinations.\n\nOutput JSON only:\n{\n  \"elementDescription\": \"short stable target/region description\",\n  \"replayInstruction\": \"one executable natural-language replay step\",\n  \"actionSummary\": \"short timeline summary\",\n  \"scrollDestinationDescription\": \"for scroll only: concrete newly visible destination content or goal\",\n  \"confidence\": \"high\" | \"medium\" | \"low\",\n  \"error\"?: \"only if no useful visual description can be inferred\"\n}\n\nRules:\n- Do NOT output coordinates as the main description.\n- Do NOT mention \"near coordinates\", \"nearby element\", \"near point\", \"red marker\", highlighted box, highlighted element, or screenshot.\n- Prefer stable target descriptions in this order: exact stable control text > stable label > role + stable section/context > icon purpose > visual position.\n- Treat placeholder or hint text that can change by user, time, data, or context as dynamic. Do not use dynamic hint values as the primary target description; prefer role + stable region + intent.\n- For repeated collections, treat item identity text as dynamic unless the user is clearly verifying that exact item. This includes any list/grid/table/feed/menu or repeated record surface. Do not output descriptions like \"<role> titled/named '<content>'\"; prefer stable role + region + selection intent.\n- Keep quoted UI text in the original UI language when it is a stable control label.\n- Apply the platform guidance from the user event:\n  - Web: button, input, link, menu item, tab, dialog, aria-label, placeholder, form section.\n  - Mobile: tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, screen section.\n  - Desktop/computer: menu item, toolbar button, dialog field, sidebar item, window control, file row, application region.\n- Pointer action rules:\n  - Preserve event.actionType semantics. Tap, DoubleClick, LongPress, and RightClick must not all become Click.\n  - Tap replayInstruction format: Tap on the element described as \"<elementDescription>\".\n  - DoubleClick replayInstruction format: Double click on the element described as \"<elementDescription>\".\n  - LongPress replayInstruction format: Long press the element described as \"<elementDescription>\".\n  - RightClick replayInstruction format: Right click on the element described as \"<elementDescription>\".\n  - Click replayInstruction format: Click on the element described as \"<elementDescription>\".\n- Input-specific rules:\n  - The highlighted BEFORE screenshot marks the field that receives the text.\n  - The screenshot after the action may show the typed value; use it only to confirm the field, never as the field description.\n  - elementDescription must identify the field itself by stable field role, nearby label, region, section, or sequence intent.\n  - Never use \"AI is analyzing element\", the typed value, the page title alone, or a generic \"input field\" as elementDescription.\n  - For consecutive input events, distinguish fields by stable role, section, order, current focus, filled/empty state, and neighboring actions instead of reusing the same generic field description.\n  - Input replayInstruction format: Input \"<value>\" into the element described as \"<elementDescription>\".\n- Scroll target quality bar:\n  - elementDescription describes the scrollable page, panel, list, table, or section at the highlighted scroll point.\n  - When multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu. Do not generalize a panel/list scroll into the whole page.\n  - scrollDestinationDescription is required and describes what the scroll is trying to reveal or reach, using newly visible headings, section titles, list items, or stable content from the AFTER screenshot.\n  - Prefer descriptions like \"Playwright integration documentation page, scrolling toward the API reference section\" or \"Android API documentation page, scrolling to the installation steps section\".\n  - Do NOT write generic phrases like \"more content\", \"the page\", \"current screen\", or \"main scrollable area\".\n- Scroll replayInstruction format: Scroll the page/region with description \"<elementDescription>\" by value \"<recorded value>\" until \"<scrollDestinationDescription>\" is visible.\n- Scroll actionSummary format: Scroll <elementDescription> toward <scrollDestinationDescription>.\n- Drag/Swipe rules:\n  - Drag replayInstruction format: Drag through the area described as \"<elementDescription>\".\n  - Swipe replayInstruction format: Swipe through the area described as \"<elementDescription>\".\n  - Describe start/end regions or the dragged UI control; do not describe only the gesture path.\n- KeyboardPress replayInstruction format: Press \"<value>\" on the element described as \"<elementDescription>\".\n- If uncertain, provide the best concrete visible text/role/context description. Set confidence to \"low\"; do not fall back to coordinates.`;\n"],"names":["RECORDER_UI_DESCRIBER_SYSTEM_PROMPT"],"mappings":"AAAO,MAAMA,sCAAsC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;0IAoDsF,CAAC"}

package/dist/es/ai-model/prompt/ui-tars-planning.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"ai-model/prompt/ui-tars-planning.mjs","sources":["../../../../src/ai-model/prompt/ui-tars-planning.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport function getUiTarsPlanningPrompt(): string {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n\\`\\`\\`\nThought: ...\nAction: ...\n\\`\\`\\`\n\n## Action Space\n\nclick(start_box='[x1, y1, x2, y2]')\nleft_double(start_box='[x1, y1, x2, y2]')\nright_single(start_box='[x1, y1, x2, y2]')\ndrag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')\nhotkey(key='')\ntype(content='xxx') # Use escape characters \\\\', \\\\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\\\n at the end of content. \nscroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\\\', \\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use ${preferredLanguage} in \\`Thought\\` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in \\`Thought\\` part.\n\n## User Instruction\n`;\n}\n\nexport const getSummary = (prediction: string) =>\n  prediction\n    .replace(/Reflection:[\\s\\S]*?(?=Action_Summary:|Action:|$)/g, '')\n    .trim();\n"],"names":["getUiTarsPlanningPrompt","preferredLanguage","getPreferredLanguage","getSummary","prediction"],"mappings":";AAEO,SAASA;IACd,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;MAuBJ,EAAED,kBAAkB;;;;AAI1B,CAAC;AACD;AAEO,MAAME,aAAa,CAACC,aACzBA,WACG,OAAO,CAAC,qDAAqD,IAC7D,IAAI"}