@midscene/core 0.25.4-beta-20250807040242.0 → 0.25.4-beta-20250807062119.0

package/dist/lib/index.d.ts

@@ -1,7 +1,7 @@
-import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractParam, h as InsightExtractOption, T as TMultimodalPrompt, A as AIUsageInfo, i as TUserPrompt, j as InsightAssertionResponse, k as AIDescribeElementResponse } from './types-da4fb35b.js';
-export { y as AIAssertionResponse, w as AIDataExtractionResponse, u as AIElementCoordinatesResponse, t as AIElementLocatorResponse, v as AIElementResponse, p as AIResponseFormat, x as AISectionLocatorResponse, s as AISingleElementResponse, q as AISingleElementResponseById, r as AISingleElementResponseByPosition, V as AgentAssertOpt, C as AgentDescribeElementAtPointResult, S as AgentWaitForOpt, a9 as BaseAgentParserOpt, F as CallAIFn, aD as CodeGenerationChunk, a8 as Color, J as DumpMeta, O as ElementById, G as EnsureObject, ac as ExecutionRecorderItem, au as ExecutionTaskAction, at as ExecutionTaskActionApply, af as ExecutionTaskHitBy, as as ExecutionTaskInsightAssertion, ar as ExecutionTaskInsightAssertionApply, aq as ExecutionTaskInsightAssertionParam, aj as ExecutionTaskInsightDumpLog, al as ExecutionTaskInsightLocate, ak as ExecutionTaskInsightLocateApply, ai as ExecutionTaskInsightLocateOutput, ah as ExecutionTaskInsightLocateParam, ap as ExecutionTaskInsightQuery, ao as ExecutionTaskInsightQueryApply, an as ExecutionTaskInsightQueryOutput, am as ExecutionTaskInsightQueryParam, aw as ExecutionTaskLog, av as ExecutionTaskLogApply, ay as ExecutionTaskPlanning, ax as ExecutionTaskPlanningApply, ag as ExecutionTaskReturn, ad as ExecutionTaskType, ae as ExecutorContext, b1 as FreeFn, az as GroupedActionDump, K as InsightDump, N as LiteUISection, aF as LocateOption, H as LocateResultElement, B as LocateValidatorResult, z as LocatorValidatorOption, b4 as MidsceneYamlConfig, b5 as MidsceneYamlConfigOutput, o as MidsceneYamlConfigResult, m as MidsceneYamlFlowItem, aM as MidsceneYamlFlowItemAIAction, aR as MidsceneYamlFlowItemAIAsk, aN as MidsceneYamlFlowItemAIAssert, aS as MidsceneYamlFlowItemAIBoolean, aW as MidsceneYamlFlowItemAIHover, aX as MidsceneYamlFlowItemAIInput, aY as MidsceneYamlFlowItemAIKeyboardPress, aT as MidsceneYamlFlowItemAILocate, aP as MidsceneYamlFlowItemAINumber, aO as MidsceneYamlFlowItemAIQuery, n as MidsceneYamlFlowItemAIRightClick, aZ as MidsceneYamlFlowItemAIScroll, aQ as MidsceneYamlFlowItemAIString, aV as MidsceneYamlFlowItemAITap, aU as MidsceneYamlFlowItemAIWaitFor, a_ as MidsceneYamlFlowItemEvaluateJavaScript, b0 as MidsceneYamlFlowItemLogScreenshot, a$ as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aK as MidsceneYamlScriptAndroidEnv, aL as MidsceneYamlScriptEnv, aI as MidsceneYamlScriptEnvBase, aJ as MidsceneYamlScriptWebEnv, l as MidsceneYamlTask, Q as OnTaskStartTip, aA as PageType, P as PartialInsightDumpFromSDK, Y as PlanningAIResponse, X as PlanningAction, a6 as PlanningActionParamAndroidLongPress, a7 as PlanningActionParamAndroidPull, a2 as PlanningActionParamAssert, a4 as PlanningActionParamError, _ as PlanningActionParamHover, a0 as PlanningActionParamInputOrKeyPress, $ as PlanningActionParamRightClick, a1 as PlanningActionParamScroll, a3 as PlanningActionParamSleep, Z as PlanningActionParamTap, a5 as PlanningActionParamWaitFor, W as PlanningLocateParam, ab as PlaywrightParserOpt, aa as PuppeteerParserOpt, aG as ReferenceImage, R as ReportDumpWithAttributes, b3 as ScriptPlayerStatusValue, b2 as ScriptPlayerTaskStatus, aE as StreamingAIResponse, aC as StreamingCallback, aB as StreamingCodeGenerationOptions, aH as scrollParam } from './types-da4fb35b.js';
-import { c as callAiFn } from './llm-planning-45dd50cd.js';
-export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-45dd50cd.js';
+import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractParam, h as InsightExtractOption, T as TMultimodalPrompt, i as IModelPreferences, A as AIUsageInfo, j as TUserPrompt, k as InsightAssertionResponse, l as AIDescribeElementResponse } from './types-7435eba0.js';
+export { z as AIAssertionResponse, x as AIDataExtractionResponse, v as AIElementCoordinatesResponse, u as AIElementLocatorResponse, w as AIElementResponse, q as AIResponseFormat, y as AISectionLocatorResponse, t as AISingleElementResponse, r as AISingleElementResponseById, s as AISingleElementResponseByPosition, W as AgentAssertOpt, F as AgentDescribeElementAtPointResult, V as AgentWaitForOpt, aa as BaseAgentParserOpt, G as CallAIFn, aE as CodeGenerationChunk, a9 as Color, K as DumpMeta, Q as ElementById, H as EnsureObject, ad as ExecutionRecorderItem, av as ExecutionTaskAction, au as ExecutionTaskActionApply, ag as ExecutionTaskHitBy, at as ExecutionTaskInsightAssertion, as as ExecutionTaskInsightAssertionApply, ar as ExecutionTaskInsightAssertionParam, ak as ExecutionTaskInsightDumpLog, am as ExecutionTaskInsightLocate, al as ExecutionTaskInsightLocateApply, aj as ExecutionTaskInsightLocateOutput, ai as ExecutionTaskInsightLocateParam, aq as ExecutionTaskInsightQuery, ap as ExecutionTaskInsightQueryApply, ao as ExecutionTaskInsightQueryOutput, an as ExecutionTaskInsightQueryParam, ax as ExecutionTaskLog, aw as ExecutionTaskLogApply, az as ExecutionTaskPlanning, ay as ExecutionTaskPlanningApply, ah as ExecutionTaskReturn, ae as ExecutionTaskType, af as ExecutorContext, b2 as FreeFn, aA as GroupedActionDump, N as InsightDump, O as LiteUISection, aG as LocateOption, J as LocateResultElement, C as LocateValidatorResult, B as LocatorValidatorOption, b5 as MidsceneYamlConfig, b6 as MidsceneYamlConfigOutput, p as MidsceneYamlConfigResult, n as MidsceneYamlFlowItem, aN as MidsceneYamlFlowItemAIAction, aS as MidsceneYamlFlowItemAIAsk, aO as MidsceneYamlFlowItemAIAssert, aT as MidsceneYamlFlowItemAIBoolean, aX as MidsceneYamlFlowItemAIHover, aY as MidsceneYamlFlowItemAIInput, aZ as MidsceneYamlFlowItemAIKeyboardPress, aU as MidsceneYamlFlowItemAILocate, aQ as MidsceneYamlFlowItemAINumber, aP as MidsceneYamlFlowItemAIQuery, o as MidsceneYamlFlowItemAIRightClick, a_ as MidsceneYamlFlowItemAIScroll, aR as MidsceneYamlFlowItemAIString, aW as MidsceneYamlFlowItemAITap, aV as MidsceneYamlFlowItemAIWaitFor, a$ as MidsceneYamlFlowItemEvaluateJavaScript, b1 as MidsceneYamlFlowItemLogScreenshot, b0 as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aL as MidsceneYamlScriptAndroidEnv, aM as MidsceneYamlScriptEnv, aJ as MidsceneYamlScriptEnvBase, aK as MidsceneYamlScriptWebEnv, m as MidsceneYamlTask, S as OnTaskStartTip, aB as PageType, P as PartialInsightDumpFromSDK, Z as PlanningAIResponse, Y as PlanningAction, a7 as PlanningActionParamAndroidLongPress, a8 as PlanningActionParamAndroidPull, a3 as PlanningActionParamAssert, a5 as PlanningActionParamError, $ as PlanningActionParamHover, a1 as PlanningActionParamInputOrKeyPress, a0 as PlanningActionParamRightClick, a2 as PlanningActionParamScroll, a4 as PlanningActionParamSleep, _ as PlanningActionParamTap, a6 as PlanningActionParamWaitFor, X as PlanningLocateParam, ac as PlaywrightParserOpt, ab as PuppeteerParserOpt, aH as ReferenceImage, R as ReportDumpWithAttributes, b4 as ScriptPlayerStatusValue, b3 as ScriptPlayerTaskStatus, aF as StreamingAIResponse, aD as StreamingCallback, aC as StreamingCodeGenerationOptions, aI as scrollParam } from './types-7435eba0.js';
+import { c as callAiFn } from './llm-planning-f449f3b8.js';
+export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-f449f3b8.js';
 import { BaseElement, Rect } from '@midscene/shared/types';
 export { BaseElement, ElementTreeNode, Point, Rect, Size } from '@midscene/shared/types';
 export { getVersion } from './utils.js';
@@ -39,7 +39,7 @@ declare class Insight<ElementType extends BaseElement = BaseElement, ContextType
     taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
     constructor(context: ContextType | ((action: InsightAction) => Promise<ContextType> | ContextType), opt?: InsightOptions);
     locate(query: DetailedLocateParam, opt?: LocateOpts): Promise<LocateResult>;
-    extract<T>(dataDemand: InsightExtractParam, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<{
+    extract<T>(dataDemand: InsightExtractParam, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt, modelPreferences?: IModelPreferences): Promise<{
         data: T;
         thought?: string;
         usage?: AIUsageInfo;
@@ -50,4 +50,4 @@ declare class Insight<ElementType extends BaseElement = BaseElement, ContextType
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
 }
 
-export { AIDescribeElementResponse, AIUsageInfo, DetailedLocateParam, DumpSubscriber, ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, Executor, Insight, InsightAction, InsightAssertionResponse, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, TMultimodalPrompt, TUserPrompt, UIContext, Insight as default };
+export { AIDescribeElementResponse, AIUsageInfo, DetailedLocateParam, DumpSubscriber, ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, Executor, IModelPreferences, Insight, InsightAction, InsightAssertionResponse, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, TMultimodalPrompt, TUserPrompt, UIContext, Insight as default };

package/dist/lib/index.js

@@ -1,6 +1,6 @@
 "use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
 
-var _chunkEK3JQ4ZVjs = require('./chunk-EK3JQ4ZV.js');
+var _chunkJH54OF4Ejs = require('./chunk-JH54OF4E.js');
 
 
 
@@ -11,7 +11,7 @@ var _chunkEK3JQ4ZVjs = require('./chunk-EK3JQ4ZV.js');
 
 
 
-var _chunkI5LBWOQAjs = require('./chunk-I5LBWOQA.js');
+var _chunkG2JTYWI6js = require('./chunk-G2JTYWI6.js');
 
 // src/ai-model/action-executor.ts
 
@@ -176,7 +176,7 @@ ${_optionalChain([this, 'access', _7 => _7.latestErrorTask, 'call', _8 => _8(),
       }
     }
     const dumpData = {
-      sdkVersion: _chunkEK3JQ4ZVjs.getVersion.call(void 0, ),
+      sdkVersion: _chunkJH54OF4Ejs.getVersion.call(void 0, ),
       model_name: _env.getAIConfig.call(void 0, _env.MIDSCENE_MODEL_NAME) || "",
       model_description: modelDescription,
       logTime: Date.now(),
@@ -248,7 +248,7 @@ var _logger = require('@midscene/shared/logger');
 
 function emitInsightDump(data, dumpSubscriber) {
   const baseData = {
-    sdkVersion: _chunkEK3JQ4ZVjs.getVersion.call(void 0, ),
+    sdkVersion: _chunkJH54OF4Ejs.getVersion.call(void 0, ),
     logTime: Date.now(),
     model_name: _env.getAIConfig.call(void 0, _env.MIDSCENE_MODEL_NAME) || ""
   };
@@ -264,7 +264,7 @@ function emitInsightDump(data, dumpSubscriber) {
 var debug = _logger.getDebug.call(void 0, "ai:insight");
 var Insight = class {
   constructor(context, opt) {
-    this.aiVendorFn = _chunkI5LBWOQAjs.callAiFn;
+    this.aiVendorFn = _chunkG2JTYWI6js.callAiFn;
     _utils.assert.call(void 0, context, "context is required for Insight");
     if (typeof context === "function") {
       this.contextRetrieverFn = context;
@@ -307,7 +307,7 @@ var Insight = class {
     let searchAreaUsage = void 0;
     let searchAreaResponse = void 0;
     if (searchAreaPrompt) {
-      searchAreaResponse = await _chunkI5LBWOQAjs.AiLocateSection.call(void 0, {
+      searchAreaResponse = await _chunkG2JTYWI6js.AiLocateSection.call(void 0, {
         context,
         sectionDescription: searchAreaPrompt
       });
@@ -327,7 +327,7 @@ var Insight = class {
       rawResponse,
       usage,
       isOrderSensitive
-    } = await _chunkI5LBWOQAjs.AiLocateElement.call(void 0, {
+    } = await _chunkG2JTYWI6js.AiLocateElement.call(void 0, {
       callAI: callAI || this.aiVendorFn,
       context,
       targetElementDescription: queryPrompt,
@@ -407,7 +407,7 @@ ${parseResult.errors.join("\n")}`;
       rect
     };
   }
-  async extract(dataDemand, opt, multimodalPrompt) {
+  async extract(dataDemand, opt, multimodalPrompt, modelPreferences) {
     _utils.assert.call(void 0, 
       typeof dataDemand === "object" || typeof dataDemand === "string",
       `dataDemand should be object or string, but get ${typeof dataDemand}`
@@ -416,11 +416,12 @@ ${parseResult.errors.join("\n")}`;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn("extract");
     const startTime = Date.now();
-    const { parseResult, usage } = await _chunkI5LBWOQAjs.AiExtractElementInfo.call(void 0, {
+    const { parseResult, usage } = await _chunkG2JTYWI6js.AiExtractElementInfo.call(void 0, {
       context,
       dataQuery: dataDemand,
       multimodalPrompt,
-      extractOption: opt
+      extractOption: opt,
+      modelPreferences
     });
     const timeCost = Date.now() - startTime;
     const taskInfo = {
@@ -465,7 +466,7 @@ ${parseResult.errors.join("\n")}`;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn("assert");
     const startTime = Date.now();
-    const assertResult = await _chunkI5LBWOQAjs.AiAssert.call(void 0, {
+    const assertResult = await _chunkG2JTYWI6js.AiAssert.call(void 0, {
       assertion,
       context
     });
@@ -519,7 +520,7 @@ ${parseResult.errors.join("\n")}`;
       borderThickness: 3
     });
     if (_optionalChain([opt, 'optionalAccess', _22 => _22.deepThink])) {
-      const searchArea = _chunkI5LBWOQAjs.expandSearchArea.call(void 0, targetRect, context.size);
+      const searchArea = _chunkG2JTYWI6js.expandSearchArea.call(void 0, targetRect, context.size);
       debug("describe: set searchArea", searchArea);
       imagePayload = await _img.cropByRect.call(void 0, 
         imagePayload,
@@ -542,7 +543,7 @@ ${parseResult.errors.join("\n")}`;
         ]
       }
     ];
-    const callAIFn = this.aiVendorFn || _chunkI5LBWOQAjs.callToGetJSONObject;
+    const callAIFn = this.aiVendorFn || _chunkG2JTYWI6js.callToGetJSONObject;
     const res = await callAIFn(msgs, 4 /* DESCRIBE_ELEMENT */);
     const { content } = res;
     _utils.assert.call(void 0, !content.error, `describe failed: ${content.error}`);
@@ -565,6 +566,6 @@ var src_default = Insight;
 
 
 
-exports.AiAssert = _chunkI5LBWOQAjs.AiAssert; exports.AiLocateElement = _chunkI5LBWOQAjs.AiLocateElement; exports.Executor = Executor; exports.Insight = Insight; exports.MIDSCENE_MODEL_NAME = _env.MIDSCENE_MODEL_NAME; exports.default = src_default; exports.describeUserPage = _chunkI5LBWOQAjs.describeUserPage; exports.getAIConfig = _env.getAIConfig; exports.getVersion = _chunkEK3JQ4ZVjs.getVersion; exports.plan = _chunkI5LBWOQAjs.plan;
+exports.AiAssert = _chunkG2JTYWI6js.AiAssert; exports.AiLocateElement = _chunkG2JTYWI6js.AiLocateElement; exports.Executor = Executor; exports.Insight = Insight; exports.MIDSCENE_MODEL_NAME = _env.MIDSCENE_MODEL_NAME; exports.default = src_default; exports.describeUserPage = _chunkG2JTYWI6js.describeUserPage; exports.getAIConfig = _env.getAIConfig; exports.getVersion = _chunkJH54OF4Ejs.getVersion; exports.plan = _chunkG2JTYWI6js.plan;
 
 //# sourceMappingURL=index.js.map

package/dist/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"mappings":";;;;;;;;;;;;;;;;AAUA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,cAAc;AAEhB,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,SACH,SAAS,SAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,SAAS,SAAS,SAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,SAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AAC3E;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,QAAyD,KAAK,gBAAgB,GAAG,KAAK;AAAA,EAAK,KAAK,gBAAgB,GAAG,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAgE;AACpE,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,oBAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY,YACjB,KAAK,YAAY,aACjB,KAAK,YAAY,YACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,aACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,QAAQ;AACb,aAAK,eACH,GAAG,YAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,YAAM,EAAE,SAAS,OAAO,IAAI,KAAK,MAAM,WAAW;AAClD,aAAO;AAAA,QACL;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,QAAI,mBAAmB;AAEvB,QAAI,aAAa,GAAG;AAClB,YAAM,iBAAiB,mBAAmB;AAC1C,UAAI,gBAAgB;AAClB,2BAAmB,WAAW,cAAc;AAAA,MAC9C,OAAO;AACL,2BAAmB,GAAG,aAAa,CAAC;AAAA,MACtC;AAAA,IACF;AACA,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,mBAAmB;AAAA,MACnB,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;AClOA,SAAS,4BAA4B;AAE9B,IAAM,8BAA8B,MAAM;AAC/C,SAAO;AAAA,4EACmE,qBAAqB,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAuClG;;;ACVA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA,gBAAAA;AAAA,OACK;AACP,SAAS,yBAAyB,kBAAkB;AACpD,SAAS,gBAAgB;AACzB,SAAS,UAAAC,eAAc;;;AClCvB,SAAS,uBAAAC,sBAAqB,eAAAC,oBAAmB;AACjD,SAAS,YAAY;AAEd,SAAS,gBACd,MACA,gBACA;AACA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAYA,aAAYD,oBAAmB,KAAK;AAAA,EAClD;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO,KAAK;AAAA,IACZ,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mBAAiB,SAAS;AAC5B;;;AD2BA,IAAM,QAAQ,SAAS,YAAY;AACnC,IAAqB,UAArB,MAGE;AAAA,EAWA,YACE,SAGA,KACA;AAXF,sBAAoD;AAYlD,IAAAD,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,QAAI,OAAO,KAAK,eAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,OAAO,KAAK,aAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,OACJ,OACA,KACuB;AACvB,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,UAAM,cAAc,OAAO,UAAU,WAAW,QAAQ,MAAM;AAC9D,IAAAA,QAAO,aAAa,8BAA8B;AAClD,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,IAAAA,QAAO,OAAO,UAAU,UAAU,sCAAsC;AAExE,UAAM,wBAAwB;AAAA,MAC5B;AAAA,IACF;AACA,QAAI,uBAAuB;AACzB,YAAM,yBAAyB,qBAAqB;AAAA,IACtD;AACA,QAAI;AACJ,QAAI,MAAM,aAAa,uBAAuB;AAC5C,yBAAmB,MAAM;AAAA,IAC3B;AAEA,QAAI,oBAAoB,CAACD,cAAa,GAAG;AACvC,cAAQ;AAAA,QACN;AAAA,MACF;AACA,yBAAmB;AAAA,IACrB;AAEA,UAAM,UAAU,KAAK,WAAY,MAAM,KAAK,mBAAmB,QAAQ;AAEvE,QAAI,aAA+B;AACnC,QAAI,wBAA4C;AAChD,QAAI,kBAA2C;AAC/C,QAAI,qBAEY;AAChB,QAAI,kBAAkB;AACpB,2BAAqB,MAAM,gBAAgB;AAAA,QACzC;AAAA,QACA,oBAAoB;AAAA,MACtB,CAAC;AACD,MAAAC;AAAA,QACE,mBAAmB;AAAA,QACnB,gCAAgC,gBAAgB,IAC9C,mBAAmB,QAAQ,KAAK,mBAAmB,KAAK,KAAK,EAC/D;AAAA,MACF;AACA,8BAAwB,mBAAmB;AAC3C,wBAAkB,mBAAmB;AACrC,mBAAa,mBAAmB;AAAA,IAClC;AAEA,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,IAAI,MAAM,gBAAgB;AAAA,MACxB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,cAAc;AAAA,IAChB,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAgC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC1E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,aAAa;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,CAAC,CAAC;AAAA,MACb,OAAO;AAAA,IACT;AAEA,UAAM,WAA0B,CAAC;AACjC,KAAC,YAAY,YAAY,CAAC,GAAG,QAAQ,CAAC,SAAS;AAC7C,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,MAAM,EAAE;AAEpC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,IAAAA;AAAA,MACE,SAAS,UAAU;AAAA,MACnB,6CAA6C,SAAS,MAAM;AAAA,IAC9D;AAEA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,UACP,IAAI,SAAS,CAAC,EAAG;AAAA,UACjB,SAAS,SAAS,CAAC,EAAG;AAAA,UACtB,QAAQ,SAAS,CAAC,EAAG;AAAA,UACrB,MAAM,SAAS,CAAC,EAAG;AAAA,UACnB,QAAQ,SAAS,CAAC,EAAG,UAAU,CAAC;AAAA,UAChC,YAAY,SAAS,CAAC,EAAG;AAAA,UACzB;AAAA,QACF;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,QACJ,YACA,KACA,kBAKC;AACD,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,MACX;AAAA,MACA,eAAe;AAAA,IACjB,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,EAAE,MAAM,QAAQ,IAAI,eAAe,CAAC;AAG1C;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,QAAI,YAAY,CAAC,MAAM;AACrB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAA2D;AACtE,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,cAAc;AAExC,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AAAA,EACA,MAAM,SACJ,QACA,KAGyD;AACzD,IAAAA,QAAO,QAAQ,yCAAyC;AACxD,UAAM,UAAU,MAAM,KAAK,mBAAmB,UAAU;AACxD,UAAM,EAAE,kBAAkB,KAAK,IAAI;AACnC,IAAAA,QAAO,kBAAkB,6CAA6C;AAEtE,UAAM,eAAe,4BAA4B;AAGjD,UAAM,kBAAkB;AACxB,UAAM,aAAmB,MAAM,QAAQ,MAAM,IACzC;AAAA,MACE,MAAM,KAAK,MAAM,OAAO,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAChD,KAAK,KAAK,MAAM,OAAO,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAC/C,OAAO;AAAA,MACP,QAAQ;AAAA,IACV,IACA;AAEJ,QAAI,eAAe,MAAM,wBAAwB;AAAA,MAC/C,gBAAgB;AAAA,MAChB;AAAA,MACA,sBAAsB;AAAA,QACpB;AAAA,UACE,MAAM;AAAA,QACR;AAAA,MACF;AAAA,MACA,iBAAiB;AAAA,IACnB,CAAC;AAED,QAAI,KAAK,WAAW;AAClB,YAAM,aAAa,iBAAiB,YAAY,QAAQ,IAAI;AAC5D,YAAM,4BAA4B,UAAU;AAC5C,qBAAe,MAAM;AAAA,QACnB;AAAA,QACA;AAAA,QACA,qBAAqB,oBAAoB;AAAA,MAC3C;AAAA,IACF;AAEA,UAAM,OAAe;AAAA,MACnB,EAAE,MAAM,UAAU,SAAS,aAAa;AAAA,MACxC;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP;AAAA,YACE,MAAM;AAAA,YACN,WAAW;AAAA,cACT,KAAK;AAAA,cACL,QAAQ;AAAA,YACV;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,UAAM,WACJ,KAAK,cAAc;AAErB,UAAM,MAAM,MAAM,SAAS,8BAAmC;AAE9D,UAAM,EAAE,QAAQ,IAAI;AACpB,IAAAA,QAAO,CAAC,QAAQ,OAAO,oBAAoB,QAAQ,KAAK,EAAE;AAC1D,IAAAA,QAAO,QAAQ,aAAa,gCAAgC;AAC5D,WAAO;AAAA,EACT;AACF;;;AE1ZA,SAAS,eAAAE,cAAa,uBAAAD,4BAA2B;AAGjD,IAAO,cAAQ","names":["vlLocateMode","assert","MIDSCENE_MODEL_NAME","getAIConfig"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/ai-model/prompt/describe.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport {\n  MIDSCENE_MODEL_NAME,\n  getAIConfig,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    let modelDescription = '';\n\n    if (vlLocateMode()) {\n      const uiTarsModelVer = uiTarsModelVersion();\n      if (uiTarsModelVer) {\n        modelDescription = `UI-TARS=${uiTarsModelVer}`;\n      } else {\n        modelDescription = `${vlLocateMode()} mode`;\n      }\n    }\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      model_description: modelDescription,\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport const elementDescriberInstruction = () => {\n  return `\nDescribe the element in the red rectangle for precise identification. Use ${getPreferredLanguage()}.\n\nCRITICAL REQUIREMENTS:\n1. UNIQUENESS: The description must uniquely identify this element on the current page\n2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts\n3. PRECISION: Be specific enough to distinguish from similar elements\n\nDESCRIPTION STRUCTURE:\n1. Element type (button, input, link, div, etc.)\n2. Primary identifier (in order of preference):\n   - Unique text content: \"with text 'Login'\"\n   - Unique attribute: \"with aria-label 'Search'\"\n   - Unique class/ID: \"with class 'primary-button'\"\n   - Unique position: \"in header navigation\"\n3. Secondary identifiers (if needed for uniqueness):\n   - Visual features: \"blue background\", \"with icon\"\n   - Relative position: \"below search bar\", \"in sidebar\"\n   - Parent context: \"in login form\", \"in main menu\"\n\nGUIDELINES:\n- Keep description under 25 words\n- Prioritize semantic identifiers over visual ones\n- Use consistent terminology across similar elements\n- Avoid page-specific or temporary content\n- Don't mention the red rectangle or selection box\n- Focus on stable, reusable characteristics\n\nEXAMPLES:\n- \"Login button with text 'Sign In'\"\n- \"Search input with placeholder 'Enter keywords'\"\n- \"Navigation link with text 'Home' in header\"\n- \"Submit button in contact form\"\n- \"Menu icon with aria-label 'Open menu'\"\n\nReturn JSON:\n{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n};\n","import {\n  AIActionType,\n  type AIArgs,\n  callAiFn,\n  expandSearchArea,\n} from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callToGetJSONObject,\n} from '@/ai-model/index';\nimport { AiAssert, AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  TMultimodalPrompt,\n  TUserPrompt,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_FORCE_DEEP_THINK,\n  MIDSCENE_USE_QWEN_VL,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n  callAI?: typeof callAiFn<AIElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt?: LocateOpts,\n  ): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = getAIConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    if (searchAreaPrompt && !vlLocateMode()) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn('locate'));\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      elementById,\n      rawResponse,\n      usage,\n      isOrderSensitive,\n    } = await AiLocateElement({\n      callAI: callAI || this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n          isOrderSensitive,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<{\n    data: T;\n    thought?: string;\n    usage?: AIUsageInfo;\n  }> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n    };\n  }\n\n  async assert(assertion: TUserPrompt): Promise<InsightAssertionResponse> {\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      userQuery: {\n        assertion,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n  async describe(\n    target: Rect | [number, number],\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn('describe');\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size);\n      debug('describe: set searchArea', searchArea);\n      imagePayload = await cropByRect(\n        imagePayload,\n        searchArea,\n        getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL),\n      );\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn =\n      this.aiVendorFn || callToGetJSONObject<AIDescribeElementResponse>;\n\n    const res = await callAIFn(msgs, AIActionType.DESCRIBE_ELEMENT);\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n","import type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  AiAssert,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlFlowItemAIRightClick,\n  MidsceneYamlConfigResult,\n} from './yaml';\n"]}
+{"version":3,"mappings":";;;;;;;;;;;;;;;;AAUA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,cAAc;AAEhB,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,SACH,SAAS,SAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,SAAS,SAAS,SAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,SAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AAC3E;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,QAAyD,KAAK,gBAAgB,GAAG,KAAK;AAAA,EAAK,KAAK,gBAAgB,GAAG,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAgE;AACpE,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,oBAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY,YACjB,KAAK,YAAY,aACjB,KAAK,YAAY,YACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,aACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,QAAQ;AACb,aAAK,eACH,GAAG,YAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,YAAM,EAAE,SAAS,OAAO,IAAI,KAAK,MAAM,WAAW;AAClD,aAAO;AAAA,QACL;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,QAAI,mBAAmB;AAEvB,QAAI,aAAa,GAAG;AAClB,YAAM,iBAAiB,mBAAmB;AAC1C,UAAI,gBAAgB;AAClB,2BAAmB,WAAW,cAAc;AAAA,MAC9C,OAAO;AACL,2BAAmB,GAAG,aAAa,CAAC;AAAA,MACtC;AAAA,IACF;AACA,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,mBAAmB;AAAA,MACnB,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;AClOA,SAAS,4BAA4B;AAE9B,IAAM,8BAA8B,MAAM;AAC/C,SAAO;AAAA,4EACmE,qBAAqB,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAuClG;;;ACTA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA,gBAAAA;AAAA,OACK;AACP,SAAS,yBAAyB,kBAAkB;AACpD,SAAS,gBAAgB;AACzB,SAAS,UAAAC,eAAc;;;ACnCvB,SAAS,uBAAAC,sBAAqB,eAAAC,oBAAmB;AACjD,SAAS,YAAY;AAEd,SAAS,gBACd,MACA,gBACA;AACA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAYA,aAAYD,oBAAmB,KAAK;AAAA,EAClD;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO,KAAK;AAAA,IACZ,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mBAAiB,SAAS;AAC5B;;;AD4BA,IAAM,QAAQ,SAAS,YAAY;AACnC,IAAqB,UAArB,MAGE;AAAA,EAWA,YACE,SAGA,KACA;AAXF,sBAAoD;AAYlD,IAAAD,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,QAAI,OAAO,KAAK,eAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,OAAO,KAAK,aAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,OACJ,OACA,KACuB;AACvB,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,UAAM,cAAc,OAAO,UAAU,WAAW,QAAQ,MAAM;AAC9D,IAAAA,QAAO,aAAa,8BAA8B;AAClD,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,IAAAA,QAAO,OAAO,UAAU,UAAU,sCAAsC;AAExE,UAAM,wBAAwB;AAAA,MAC5B;AAAA,IACF;AACA,QAAI,uBAAuB;AACzB,YAAM,yBAAyB,qBAAqB;AAAA,IACtD;AACA,QAAI;AACJ,QAAI,MAAM,aAAa,uBAAuB;AAC5C,yBAAmB,MAAM;AAAA,IAC3B;AAEA,QAAI,oBAAoB,CAACD,cAAa,GAAG;AACvC,cAAQ;AAAA,QACN;AAAA,MACF;AACA,yBAAmB;AAAA,IACrB;AAEA,UAAM,UAAU,KAAK,WAAY,MAAM,KAAK,mBAAmB,QAAQ;AAEvE,QAAI,aAA+B;AACnC,QAAI,wBAA4C;AAChD,QAAI,kBAA2C;AAC/C,QAAI,qBAEY;AAChB,QAAI,kBAAkB;AACpB,2BAAqB,MAAM,gBAAgB;AAAA,QACzC;AAAA,QACA,oBAAoB;AAAA,MACtB,CAAC;AACD,MAAAC;AAAA,QACE,mBAAmB;AAAA,QACnB,gCAAgC,gBAAgB,IAC9C,mBAAmB,QAAQ,KAAK,mBAAmB,KAAK,KAAK,EAC/D;AAAA,MACF;AACA,8BAAwB,mBAAmB;AAC3C,wBAAkB,mBAAmB;AACrC,mBAAa,mBAAmB;AAAA,IAClC;AAEA,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,IAAI,MAAM,gBAAgB;AAAA,MACxB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,cAAc;AAAA,IAChB,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAgC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC1E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,aAAa;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,CAAC,CAAC;AAAA,MACb,OAAO;AAAA,IACT;AAEA,UAAM,WAA0B,CAAC;AACjC,KAAC,YAAY,YAAY,CAAC,GAAG,QAAQ,CAAC,SAAS;AAC7C,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,MAAM,EAAE;AAEpC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,IAAAA;AAAA,MACE,SAAS,UAAU;AAAA,MACnB,6CAA6C,SAAS,MAAM;AAAA,IAC9D;AAEA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,UACP,IAAI,SAAS,CAAC,EAAG;AAAA,UACjB,SAAS,SAAS,CAAC,EAAG;AAAA,UACtB,QAAQ,SAAS,CAAC,EAAG;AAAA,UACrB,MAAM,SAAS,CAAC,EAAG;AAAA,UACnB,QAAQ,SAAS,CAAC,EAAG,UAAU,CAAC;AAAA,UAChC,YAAY,SAAS,CAAC,EAAG;AAAA,UACzB;AAAA,QACF;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,QACJ,YACA,KACA,kBACA,kBAKC;AACD,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,MACX;AAAA,MACA,eAAe;AAAA,MACf;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,EAAE,MAAM,QAAQ,IAAI,eAAe,CAAC;AAG1C;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,QAAI,YAAY,CAAC,MAAM;AACrB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAA2D;AACtE,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,cAAc;AAExC,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AAAA,EACA,MAAM,SACJ,QACA,KAGyD;AACzD,IAAAA,QAAO,QAAQ,yCAAyC;AACxD,UAAM,UAAU,MAAM,KAAK,mBAAmB,UAAU;AACxD,UAAM,EAAE,kBAAkB,KAAK,IAAI;AACnC,IAAAA,QAAO,kBAAkB,6CAA6C;AAEtE,UAAM,eAAe,4BAA4B;AAGjD,UAAM,kBAAkB;AACxB,UAAM,aAAmB,MAAM,QAAQ,MAAM,IACzC;AAAA,MACE,MAAM,KAAK,MAAM,OAAO,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAChD,KAAK,KAAK,MAAM,OAAO,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAC/C,OAAO;AAAA,MACP,QAAQ;AAAA,IACV,IACA;AAEJ,QAAI,eAAe,MAAM,wBAAwB;AAAA,MAC/C,gBAAgB;AAAA,MAChB;AAAA,MACA,sBAAsB;AAAA,QACpB;AAAA,UACE,MAAM;AAAA,QACR;AAAA,MACF;AAAA,MACA,iBAAiB;AAAA,IACnB,CAAC;AAED,QAAI,KAAK,WAAW;AAClB,YAAM,aAAa,iBAAiB,YAAY,QAAQ,IAAI;AAC5D,YAAM,4BAA4B,UAAU;AAC5C,qBAAe,MAAM;AAAA,QACnB;AAAA,QACA;AAAA,QACA,qBAAqB,oBAAoB;AAAA,MAC3C;AAAA,IACF;AAEA,UAAM,OAAe;AAAA,MACnB,EAAE,MAAM,UAAU,SAAS,aAAa;AAAA,MACxC;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP;AAAA,YACE,MAAM;AAAA,YACN,WAAW;AAAA,cACT,KAAK;AAAA,cACL,QAAQ;AAAA,YACV;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,UAAM,WACJ,KAAK,cAAc;AAErB,UAAM,MAAM,MAAM,SAAS,8BAAmC;AAE9D,UAAM,EAAE,QAAQ,IAAI;AACpB,IAAAA,QAAO,CAAC,QAAQ,OAAO,oBAAoB,QAAQ,KAAK,EAAE;AAC1D,IAAAA,QAAO,QAAQ,aAAa,gCAAgC;AAC5D,WAAO;AAAA,EACT;AACF;;;AE7ZA,SAAS,eAAAE,cAAa,uBAAAD,4BAA2B;AAGjD,IAAO,cAAQ","names":["vlLocateMode","assert","MIDSCENE_MODEL_NAME","getAIConfig"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/ai-model/prompt/describe.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport {\n  MIDSCENE_MODEL_NAME,\n  getAIConfig,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    let modelDescription = '';\n\n    if (vlLocateMode()) {\n      const uiTarsModelVer = uiTarsModelVersion();\n      if (uiTarsModelVer) {\n        modelDescription = `UI-TARS=${uiTarsModelVer}`;\n      } else {\n        modelDescription = `${vlLocateMode()} mode`;\n      }\n    }\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      model_description: modelDescription,\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport const elementDescriberInstruction = () => {\n  return `\nDescribe the element in the red rectangle for precise identification. Use ${getPreferredLanguage()}.\n\nCRITICAL REQUIREMENTS:\n1. UNIQUENESS: The description must uniquely identify this element on the current page\n2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts\n3. PRECISION: Be specific enough to distinguish from similar elements\n\nDESCRIPTION STRUCTURE:\n1. Element type (button, input, link, div, etc.)\n2. Primary identifier (in order of preference):\n   - Unique text content: \"with text 'Login'\"\n   - Unique attribute: \"with aria-label 'Search'\"\n   - Unique class/ID: \"with class 'primary-button'\"\n   - Unique position: \"in header navigation\"\n3. Secondary identifiers (if needed for uniqueness):\n   - Visual features: \"blue background\", \"with icon\"\n   - Relative position: \"below search bar\", \"in sidebar\"\n   - Parent context: \"in login form\", \"in main menu\"\n\nGUIDELINES:\n- Keep description under 25 words\n- Prioritize semantic identifiers over visual ones\n- Use consistent terminology across similar elements\n- Avoid page-specific or temporary content\n- Don't mention the red rectangle or selection box\n- Focus on stable, reusable characteristics\n\nEXAMPLES:\n- \"Login button with text 'Sign In'\"\n- \"Search input with placeholder 'Enter keywords'\"\n- \"Navigation link with text 'Home' in header\"\n- \"Submit button in contact form\"\n- \"Menu icon with aria-label 'Open menu'\"\n\nReturn JSON:\n{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n};\n","import {\n  AIActionType,\n  type AIArgs,\n  callAiFn,\n  expandSearchArea,\n} from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callToGetJSONObject,\n} from '@/ai-model/index';\nimport { AiAssert, AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  IModelPreferences,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  TMultimodalPrompt,\n  TUserPrompt,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_FORCE_DEEP_THINK,\n  MIDSCENE_USE_QWEN_VL,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n  callAI?: typeof callAiFn<AIElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt?: LocateOpts,\n  ): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = getAIConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    if (searchAreaPrompt && !vlLocateMode()) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn('locate'));\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      elementById,\n      rawResponse,\n      usage,\n      isOrderSensitive,\n    } = await AiLocateElement({\n      callAI: callAI || this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n          isOrderSensitive,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    modelPreferences?: IModelPreferences,\n  ): Promise<{\n    data: T;\n    thought?: string;\n    usage?: AIUsageInfo;\n  }> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n      modelPreferences,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n    };\n  }\n\n  async assert(assertion: TUserPrompt): Promise<InsightAssertionResponse> {\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      userQuery: {\n        assertion,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n  async describe(\n    target: Rect | [number, number],\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn('describe');\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size);\n      debug('describe: set searchArea', searchArea);\n      imagePayload = await cropByRect(\n        imagePayload,\n        searchArea,\n        getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL),\n      );\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn =\n      this.aiVendorFn || callToGetJSONObject<AIDescribeElementResponse>;\n\n    const res = await callAIFn(msgs, AIActionType.DESCRIBE_ELEMENT);\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n","import type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  AiAssert,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlFlowItemAIRightClick,\n  MidsceneYamlConfigResult,\n} from './yaml';\n"]}

package/dist/lib/{llm-planning-45dd50cd.d.ts → llm-planning-f449f3b8.d.ts}

@@ -1,4 +1,4 @@
-import { A as AIUsageInfo, U as UIContext, i as TUserPrompt, aG as ReferenceImage, t as AIElementLocatorResponse, O as ElementById, T as TMultimodalPrompt, h as InsightExtractOption, w as AIDataExtractionResponse, y as AIAssertionResponse, aA as PageType, Y as PlanningAIResponse } from './types-da4fb35b.js';
+import { i as IModelPreferences, A as AIUsageInfo, U as UIContext, j as TUserPrompt, aH as ReferenceImage, u as AIElementLocatorResponse, Q as ElementById, T as TMultimodalPrompt, h as InsightExtractOption, x as AIDataExtractionResponse, z as AIAssertionResponse, aB as PageType, Z as PlanningAIResponse } from './types-7435eba0.js';
 import { Rect, ElementTreeNode, BaseElement } from '@midscene/shared/types';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 
@@ -13,7 +13,7 @@ declare enum AIActionType {
     PLAN = 3,
     DESCRIBE_ELEMENT = 4
 }
-declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
+declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType, modelPreferences?: IModelPreferences): Promise<{
     content: T;
     usage?: AIUsageInfo;
 }>;
@@ -81,6 +81,7 @@ declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseE
     multimodalPrompt?: TMultimodalPrompt;
     context: UIContext<ElementType>;
     extractOption?: InsightExtractOption;
+    modelPreferences?: IModelPreferences;
 }): Promise<{
     parseResult: AIDataExtractionResponse<T>;
     elementById: (idOrIndexId: string) => ElementType;

package/dist/{types/types-da4fb35b.d.ts → lib/types-7435eba0.d.ts}

@@ -563,5 +563,12 @@ type TMultimodalPrompt = {
 type TUserPrompt = string | ({
     prompt: string;
 } & Partial<TMultimodalPrompt>);
+interface IModelPreferences {
+    /**
+     * - VQA: Visual Question Answering
+     * - grounding：short for Visual Grounding
+     */
+    intent: 'VQA' | 'planning' | 'grounding';
+}
 
-export { type PlanningActionParamRightClick as $, type AIUsageInfo as A, type LocateValidatorResult as B, type AgentDescribeElementAtPointResult as C, type DumpSubscriber as D, type ExecutionTask as E, type CallAIFn as F, type EnsureObject as G, type LocateResultElement as H, type InsightAction as I, type DumpMeta as J, type InsightDump as K, type LocateResult as L, type MidsceneYamlScript as M, type LiteUISection as N, type ElementById as O, type PartialInsightDumpFromSDK as P, type OnTaskStartTip as Q, type ReportDumpWithAttributes as R, type AgentWaitForOpt as S, type TMultimodalPrompt as T, UIContext as U, type AgentAssertOpt as V, type PlanningLocateParam as W, type PlanningAction as X, type PlanningAIResponse as Y, type PlanningActionParamTap as Z, type PlanningActionParamHover as _, type ExecutionTaskProgressOptions as a, type MidsceneYamlFlowItemSleep as a$, type PlanningActionParamInputOrKeyPress as a0, type PlanningActionParamScroll as a1, type PlanningActionParamAssert as a2, type PlanningActionParamSleep as a3, type PlanningActionParamError as a4, type PlanningActionParamWaitFor as a5, type PlanningActionParamAndroidLongPress as a6, type PlanningActionParamAndroidPull as a7, type Color as a8, type BaseAgentParserOpt as a9, type PageType as aA, type StreamingCodeGenerationOptions as aB, type StreamingCallback as aC, type CodeGenerationChunk as aD, type StreamingAIResponse as aE, type LocateOption as aF, type ReferenceImage as aG, type scrollParam as aH, type MidsceneYamlScriptEnvBase as aI, type MidsceneYamlScriptWebEnv as aJ, type MidsceneYamlScriptAndroidEnv as aK, type MidsceneYamlScriptEnv as aL, type MidsceneYamlFlowItemAIAction as aM, type MidsceneYamlFlowItemAIAssert as aN, type MidsceneYamlFlowItemAIQuery as aO, type MidsceneYamlFlowItemAINumber as aP, type MidsceneYamlFlowItemAIString as aQ, type MidsceneYamlFlowItemAIAsk as aR, type MidsceneYamlFlowItemAIBoolean as aS, type MidsceneYamlFlowItemAILocate as aT, type MidsceneYamlFlowItemAIWaitFor as aU, type MidsceneYamlFlowItemAITap as aV, type MidsceneYamlFlowItemAIHover as aW, type MidsceneYamlFlowItemAIInput as aX, type MidsceneYamlFlowItemAIKeyboardPress as aY, type MidsceneYamlFlowItemAIScroll as aZ, type MidsceneYamlFlowItemEvaluateJavaScript as a_, type PuppeteerParserOpt as aa, type PlaywrightParserOpt as ab, type ExecutionRecorderItem as ac, type ExecutionTaskType as ad, type ExecutorContext as ae, type ExecutionTaskHitBy as af, type ExecutionTaskReturn as ag, type ExecutionTaskInsightLocateParam as ah, type ExecutionTaskInsightLocateOutput as ai, type ExecutionTaskInsightDumpLog as aj, type ExecutionTaskInsightLocateApply as ak, type ExecutionTaskInsightLocate as al, type ExecutionTaskInsightQueryParam as am, type ExecutionTaskInsightQueryOutput as an, type ExecutionTaskInsightQueryApply as ao, type ExecutionTaskInsightQuery as ap, type ExecutionTaskInsightAssertionParam as aq, type ExecutionTaskInsightAssertionApply as ar, type ExecutionTaskInsightAssertion as as, type ExecutionTaskActionApply as at, type ExecutionTaskAction as au, type ExecutionTaskLogApply as av, type ExecutionTaskLog as aw, type ExecutionTaskPlanningApply as ax, type ExecutionTaskPlanning as ay, type GroupedActionDump as az, type ExecutionTaskApply as b, type MidsceneYamlFlowItemLogScreenshot as b0, type FreeFn as b1, type ScriptPlayerTaskStatus as b2, type ScriptPlayerStatusValue as b3, type MidsceneYamlConfig as b4, type MidsceneYamlConfigOutput as b5, type ExecutionDump as c, type InsightTaskInfo as d, type InsightOptions as e, type DetailedLocateParam as f, type InsightExtractParam as g, type InsightExtractOption as h, type TUserPrompt as i, type InsightAssertionResponse as j, type AIDescribeElementResponse as k, type MidsceneYamlTask as l, type MidsceneYamlFlowItem as m, type MidsceneYamlFlowItemAIRightClick as n, type MidsceneYamlConfigResult as o, AIResponseFormat as p, type AISingleElementResponseById as q, type AISingleElementResponseByPosition as r, type AISingleElementResponse as s, type AIElementLocatorResponse as t, type AIElementCoordinatesResponse as u, type AIElementResponse as v, type AIDataExtractionResponse as w, type AISectionLocatorResponse as x, type AIAssertionResponse as y, type LocatorValidatorOption as z };
+export { type PlanningActionParamHover as $, type AIUsageInfo as A, type LocatorValidatorOption as B, type LocateValidatorResult as C, type DumpSubscriber as D, type ExecutionTask as E, type AgentDescribeElementAtPointResult as F, type CallAIFn as G, type EnsureObject as H, type InsightAction as I, type LocateResultElement as J, type DumpMeta as K, type LocateResult as L, type MidsceneYamlScript as M, type InsightDump as N, type LiteUISection as O, type PartialInsightDumpFromSDK as P, type ElementById as Q, type ReportDumpWithAttributes as R, type OnTaskStartTip as S, type TMultimodalPrompt as T, UIContext as U, type AgentWaitForOpt as V, type AgentAssertOpt as W, type PlanningLocateParam as X, type PlanningAction as Y, type PlanningAIResponse as Z, type PlanningActionParamTap as _, type ExecutionTaskProgressOptions as a, type MidsceneYamlFlowItemEvaluateJavaScript as a$, type PlanningActionParamRightClick as a0, type PlanningActionParamInputOrKeyPress as a1, type PlanningActionParamScroll as a2, type PlanningActionParamAssert as a3, type PlanningActionParamSleep as a4, type PlanningActionParamError as a5, type PlanningActionParamWaitFor as a6, type PlanningActionParamAndroidLongPress as a7, type PlanningActionParamAndroidPull as a8, type Color as a9, type GroupedActionDump as aA, type PageType as aB, type StreamingCodeGenerationOptions as aC, type StreamingCallback as aD, type CodeGenerationChunk as aE, type StreamingAIResponse as aF, type LocateOption as aG, type ReferenceImage as aH, type scrollParam as aI, type MidsceneYamlScriptEnvBase as aJ, type MidsceneYamlScriptWebEnv as aK, type MidsceneYamlScriptAndroidEnv as aL, type MidsceneYamlScriptEnv as aM, type MidsceneYamlFlowItemAIAction as aN, type MidsceneYamlFlowItemAIAssert as aO, type MidsceneYamlFlowItemAIQuery as aP, type MidsceneYamlFlowItemAINumber as aQ, type MidsceneYamlFlowItemAIString as aR, type MidsceneYamlFlowItemAIAsk as aS, type MidsceneYamlFlowItemAIBoolean as aT, type MidsceneYamlFlowItemAILocate as aU, type MidsceneYamlFlowItemAIWaitFor as aV, type MidsceneYamlFlowItemAITap as aW, type MidsceneYamlFlowItemAIHover as aX, type MidsceneYamlFlowItemAIInput as aY, type MidsceneYamlFlowItemAIKeyboardPress as aZ, type MidsceneYamlFlowItemAIScroll as a_, type BaseAgentParserOpt as aa, type PuppeteerParserOpt as ab, type PlaywrightParserOpt as ac, type ExecutionRecorderItem as ad, type ExecutionTaskType as ae, type ExecutorContext as af, type ExecutionTaskHitBy as ag, type ExecutionTaskReturn as ah, type ExecutionTaskInsightLocateParam as ai, type ExecutionTaskInsightLocateOutput as aj, type ExecutionTaskInsightDumpLog as ak, type ExecutionTaskInsightLocateApply as al, type ExecutionTaskInsightLocate as am, type ExecutionTaskInsightQueryParam as an, type ExecutionTaskInsightQueryOutput as ao, type ExecutionTaskInsightQueryApply as ap, type ExecutionTaskInsightQuery as aq, type ExecutionTaskInsightAssertionParam as ar, type ExecutionTaskInsightAssertionApply as as, type ExecutionTaskInsightAssertion as at, type ExecutionTaskActionApply as au, type ExecutionTaskAction as av, type ExecutionTaskLogApply as aw, type ExecutionTaskLog as ax, type ExecutionTaskPlanningApply as ay, type ExecutionTaskPlanning as az, type ExecutionTaskApply as b, type MidsceneYamlFlowItemSleep as b0, type MidsceneYamlFlowItemLogScreenshot as b1, type FreeFn as b2, type ScriptPlayerTaskStatus as b3, type ScriptPlayerStatusValue as b4, type MidsceneYamlConfig as b5, type MidsceneYamlConfigOutput as b6, type ExecutionDump as c, type InsightTaskInfo as d, type InsightOptions as e, type DetailedLocateParam as f, type InsightExtractParam as g, type InsightExtractOption as h, type IModelPreferences as i, type TUserPrompt as j, type InsightAssertionResponse as k, type AIDescribeElementResponse as l, type MidsceneYamlTask as m, type MidsceneYamlFlowItem as n, type MidsceneYamlFlowItemAIRightClick as o, type MidsceneYamlConfigResult as p, AIResponseFormat as q, type AISingleElementResponseById as r, type AISingleElementResponseByPosition as s, type AISingleElementResponse as t, type AIElementLocatorResponse as u, type AIElementCoordinatesResponse as v, type AIElementResponse as w, type AIDataExtractionResponse as x, type AISectionLocatorResponse as y, type AIAssertionResponse as z };

package/dist/lib/utils.d.ts

@@ -1,4 +1,4 @@
-import { R as ReportDumpWithAttributes } from './types-da4fb35b.js';
+import { R as ReportDumpWithAttributes } from './types-7435eba0.js';
 import { Rect } from '@midscene/shared/types';
 import '@midscene/shared/constants';
 import 'openai/resources';

package/dist/lib/utils.js

@@ -12,7 +12,7 @@
 
 
 
-var _chunkEK3JQ4ZVjs = require('./chunk-EK3JQ4ZV.js');
+var _chunkJH54OF4Ejs = require('./chunk-JH54OF4E.js');
 
 
 
@@ -27,4 +27,4 @@ var _chunkEK3JQ4ZVjs = require('./chunk-EK3JQ4ZV.js');
 
 
 
-exports.getTmpDir = _chunkEK3JQ4ZVjs.getTmpDir; exports.getTmpFile = _chunkEK3JQ4ZVjs.getTmpFile; exports.getVersion = _chunkEK3JQ4ZVjs.getVersion; exports.groupedActionDumpFileExt = _chunkEK3JQ4ZVjs.groupedActionDumpFileExt; exports.insertScriptBeforeClosingHtml = _chunkEK3JQ4ZVjs.insertScriptBeforeClosingHtml; exports.overlapped = _chunkEK3JQ4ZVjs.overlapped; exports.replacerForPageObject = _chunkEK3JQ4ZVjs.replacerForPageObject; exports.reportHTMLContent = _chunkEK3JQ4ZVjs.reportHTMLContent; exports.sleep = _chunkEK3JQ4ZVjs.sleep; exports.stringifyDumpData = _chunkEK3JQ4ZVjs.stringifyDumpData; exports.uploadTestInfoToServer = _chunkEK3JQ4ZVjs.uploadTestInfoToServer; exports.writeDumpReport = _chunkEK3JQ4ZVjs.writeDumpReport; exports.writeLogFile = _chunkEK3JQ4ZVjs.writeLogFile;
+exports.getTmpDir = _chunkJH54OF4Ejs.getTmpDir; exports.getTmpFile = _chunkJH54OF4Ejs.getTmpFile; exports.getVersion = _chunkJH54OF4Ejs.getVersion; exports.groupedActionDumpFileExt = _chunkJH54OF4Ejs.groupedActionDumpFileExt; exports.insertScriptBeforeClosingHtml = _chunkJH54OF4Ejs.insertScriptBeforeClosingHtml; exports.overlapped = _chunkJH54OF4Ejs.overlapped; exports.replacerForPageObject = _chunkJH54OF4Ejs.replacerForPageObject; exports.reportHTMLContent = _chunkJH54OF4Ejs.reportHTMLContent; exports.sleep = _chunkJH54OF4Ejs.sleep; exports.stringifyDumpData = _chunkJH54OF4Ejs.stringifyDumpData; exports.uploadTestInfoToServer = _chunkJH54OF4Ejs.uploadTestInfoToServer; exports.writeDumpReport = _chunkJH54OF4Ejs.writeDumpReport; exports.writeLogFile = _chunkJH54OF4Ejs.writeLogFile;

package/dist/types/ai-model.d.ts

@@ -1,23 +1,22 @@
-import { aC as StreamingCallback, A as AIUsageInfo, aB as StreamingCodeGenerationOptions, aE as StreamingAIResponse, X as PlanningAction, m as MidsceneYamlFlowItem } from './types-da4fb35b.js';
-import OpenAI from 'openai';
+import { aD as StreamingCallback, i as IModelPreferences, A as AIUsageInfo, aC as StreamingCodeGenerationOptions, aF as StreamingAIResponse, Y as PlanningAction, n as MidsceneYamlFlowItem } from './types-7435eba0.js';
 import { ChatCompletionMessageParam } from 'openai/resources';
 export { ChatCompletionMessageParam } from 'openai/resources';
-import { b as AIActionType, e as AIArgs } from './llm-planning-45dd50cd.js';
-export { a as AiAssert, g as AiExtractElementInfo, A as AiLocateElement, h as AiLocateSection, i as adaptBboxToRect, c as callAiFn, d as describeUserPage, f as elementByPositionWithElementInfo, p as plan } from './llm-planning-45dd50cd.js';
+import { b as AIActionType, e as AIArgs } from './llm-planning-f449f3b8.js';
+export { a as AiAssert, g as AiExtractElementInfo, A as AiLocateElement, h as AiLocateSection, i as adaptBboxToRect, c as callAiFn, d as describeUserPage, f as elementByPositionWithElementInfo, p as plan } from './llm-planning-f449f3b8.js';
 import { vlLocateMode } from '@midscene/shared/env';
 import { actionParser } from '@ui-tars/action-parser';
 import { Size } from '@midscene/shared/types';
 import '@midscene/shared/constants';
 
-declare function call(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, responseFormat?: OpenAI.ChatCompletionCreateParams['response_format'] | OpenAI.ResponseFormatJSONObject, options?: {
+declare function call(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, options?: {
     stream?: boolean;
     onChunk?: StreamingCallback;
-}): Promise<{
+}, modelPreferences?: IModelPreferences): Promise<{
     content: string;
     usage?: AIUsageInfo;
     isStreamed: boolean;
 }>;
-declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType): Promise<{
+declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelPreferences?: IModelPreferences): Promise<{
     content: T;
     usage?: AIUsageInfo;
 }>;

package/dist/types/index.d.ts

@@ -1,7 +1,7 @@
-import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractParam, h as InsightExtractOption, T as TMultimodalPrompt, A as AIUsageInfo, i as TUserPrompt, j as InsightAssertionResponse, k as AIDescribeElementResponse } from './types-da4fb35b.js';
-export { y as AIAssertionResponse, w as AIDataExtractionResponse, u as AIElementCoordinatesResponse, t as AIElementLocatorResponse, v as AIElementResponse, p as AIResponseFormat, x as AISectionLocatorResponse, s as AISingleElementResponse, q as AISingleElementResponseById, r as AISingleElementResponseByPosition, V as AgentAssertOpt, C as AgentDescribeElementAtPointResult, S as AgentWaitForOpt, a9 as BaseAgentParserOpt, F as CallAIFn, aD as CodeGenerationChunk, a8 as Color, J as DumpMeta, O as ElementById, G as EnsureObject, ac as ExecutionRecorderItem, au as ExecutionTaskAction, at as ExecutionTaskActionApply, af as ExecutionTaskHitBy, as as ExecutionTaskInsightAssertion, ar as ExecutionTaskInsightAssertionApply, aq as ExecutionTaskInsightAssertionParam, aj as ExecutionTaskInsightDumpLog, al as ExecutionTaskInsightLocate, ak as ExecutionTaskInsightLocateApply, ai as ExecutionTaskInsightLocateOutput, ah as ExecutionTaskInsightLocateParam, ap as ExecutionTaskInsightQuery, ao as ExecutionTaskInsightQueryApply, an as ExecutionTaskInsightQueryOutput, am as ExecutionTaskInsightQueryParam, aw as ExecutionTaskLog, av as ExecutionTaskLogApply, ay as ExecutionTaskPlanning, ax as ExecutionTaskPlanningApply, ag as ExecutionTaskReturn, ad as ExecutionTaskType, ae as ExecutorContext, b1 as FreeFn, az as GroupedActionDump, K as InsightDump, N as LiteUISection, aF as LocateOption, H as LocateResultElement, B as LocateValidatorResult, z as LocatorValidatorOption, b4 as MidsceneYamlConfig, b5 as MidsceneYamlConfigOutput, o as MidsceneYamlConfigResult, m as MidsceneYamlFlowItem, aM as MidsceneYamlFlowItemAIAction, aR as MidsceneYamlFlowItemAIAsk, aN as MidsceneYamlFlowItemAIAssert, aS as MidsceneYamlFlowItemAIBoolean, aW as MidsceneYamlFlowItemAIHover, aX as MidsceneYamlFlowItemAIInput, aY as MidsceneYamlFlowItemAIKeyboardPress, aT as MidsceneYamlFlowItemAILocate, aP as MidsceneYamlFlowItemAINumber, aO as MidsceneYamlFlowItemAIQuery, n as MidsceneYamlFlowItemAIRightClick, aZ as MidsceneYamlFlowItemAIScroll, aQ as MidsceneYamlFlowItemAIString, aV as MidsceneYamlFlowItemAITap, aU as MidsceneYamlFlowItemAIWaitFor, a_ as MidsceneYamlFlowItemEvaluateJavaScript, b0 as MidsceneYamlFlowItemLogScreenshot, a$ as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aK as MidsceneYamlScriptAndroidEnv, aL as MidsceneYamlScriptEnv, aI as MidsceneYamlScriptEnvBase, aJ as MidsceneYamlScriptWebEnv, l as MidsceneYamlTask, Q as OnTaskStartTip, aA as PageType, P as PartialInsightDumpFromSDK, Y as PlanningAIResponse, X as PlanningAction, a6 as PlanningActionParamAndroidLongPress, a7 as PlanningActionParamAndroidPull, a2 as PlanningActionParamAssert, a4 as PlanningActionParamError, _ as PlanningActionParamHover, a0 as PlanningActionParamInputOrKeyPress, $ as PlanningActionParamRightClick, a1 as PlanningActionParamScroll, a3 as PlanningActionParamSleep, Z as PlanningActionParamTap, a5 as PlanningActionParamWaitFor, W as PlanningLocateParam, ab as PlaywrightParserOpt, aa as PuppeteerParserOpt, aG as ReferenceImage, R as ReportDumpWithAttributes, b3 as ScriptPlayerStatusValue, b2 as ScriptPlayerTaskStatus, aE as StreamingAIResponse, aC as StreamingCallback, aB as StreamingCodeGenerationOptions, aH as scrollParam } from './types-da4fb35b.js';
-import { c as callAiFn } from './llm-planning-45dd50cd.js';
-export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-45dd50cd.js';
+import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractParam, h as InsightExtractOption, T as TMultimodalPrompt, i as IModelPreferences, A as AIUsageInfo, j as TUserPrompt, k as InsightAssertionResponse, l as AIDescribeElementResponse } from './types-7435eba0.js';
+export { z as AIAssertionResponse, x as AIDataExtractionResponse, v as AIElementCoordinatesResponse, u as AIElementLocatorResponse, w as AIElementResponse, q as AIResponseFormat, y as AISectionLocatorResponse, t as AISingleElementResponse, r as AISingleElementResponseById, s as AISingleElementResponseByPosition, W as AgentAssertOpt, F as AgentDescribeElementAtPointResult, V as AgentWaitForOpt, aa as BaseAgentParserOpt, G as CallAIFn, aE as CodeGenerationChunk, a9 as Color, K as DumpMeta, Q as ElementById, H as EnsureObject, ad as ExecutionRecorderItem, av as ExecutionTaskAction, au as ExecutionTaskActionApply, ag as ExecutionTaskHitBy, at as ExecutionTaskInsightAssertion, as as ExecutionTaskInsightAssertionApply, ar as ExecutionTaskInsightAssertionParam, ak as ExecutionTaskInsightDumpLog, am as ExecutionTaskInsightLocate, al as ExecutionTaskInsightLocateApply, aj as ExecutionTaskInsightLocateOutput, ai as ExecutionTaskInsightLocateParam, aq as ExecutionTaskInsightQuery, ap as ExecutionTaskInsightQueryApply, ao as ExecutionTaskInsightQueryOutput, an as ExecutionTaskInsightQueryParam, ax as ExecutionTaskLog, aw as ExecutionTaskLogApply, az as ExecutionTaskPlanning, ay as ExecutionTaskPlanningApply, ah as ExecutionTaskReturn, ae as ExecutionTaskType, af as ExecutorContext, b2 as FreeFn, aA as GroupedActionDump, N as InsightDump, O as LiteUISection, aG as LocateOption, J as LocateResultElement, C as LocateValidatorResult, B as LocatorValidatorOption, b5 as MidsceneYamlConfig, b6 as MidsceneYamlConfigOutput, p as MidsceneYamlConfigResult, n as MidsceneYamlFlowItem, aN as MidsceneYamlFlowItemAIAction, aS as MidsceneYamlFlowItemAIAsk, aO as MidsceneYamlFlowItemAIAssert, aT as MidsceneYamlFlowItemAIBoolean, aX as MidsceneYamlFlowItemAIHover, aY as MidsceneYamlFlowItemAIInput, aZ as MidsceneYamlFlowItemAIKeyboardPress, aU as MidsceneYamlFlowItemAILocate, aQ as MidsceneYamlFlowItemAINumber, aP as MidsceneYamlFlowItemAIQuery, o as MidsceneYamlFlowItemAIRightClick, a_ as MidsceneYamlFlowItemAIScroll, aR as MidsceneYamlFlowItemAIString, aW as MidsceneYamlFlowItemAITap, aV as MidsceneYamlFlowItemAIWaitFor, a$ as MidsceneYamlFlowItemEvaluateJavaScript, b1 as MidsceneYamlFlowItemLogScreenshot, b0 as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aL as MidsceneYamlScriptAndroidEnv, aM as MidsceneYamlScriptEnv, aJ as MidsceneYamlScriptEnvBase, aK as MidsceneYamlScriptWebEnv, m as MidsceneYamlTask, S as OnTaskStartTip, aB as PageType, P as PartialInsightDumpFromSDK, Z as PlanningAIResponse, Y as PlanningAction, a7 as PlanningActionParamAndroidLongPress, a8 as PlanningActionParamAndroidPull, a3 as PlanningActionParamAssert, a5 as PlanningActionParamError, $ as PlanningActionParamHover, a1 as PlanningActionParamInputOrKeyPress, a0 as PlanningActionParamRightClick, a2 as PlanningActionParamScroll, a4 as PlanningActionParamSleep, _ as PlanningActionParamTap, a6 as PlanningActionParamWaitFor, X as PlanningLocateParam, ac as PlaywrightParserOpt, ab as PuppeteerParserOpt, aH as ReferenceImage, R as ReportDumpWithAttributes, b4 as ScriptPlayerStatusValue, b3 as ScriptPlayerTaskStatus, aF as StreamingAIResponse, aD as StreamingCallback, aC as StreamingCodeGenerationOptions, aI as scrollParam } from './types-7435eba0.js';
+import { c as callAiFn } from './llm-planning-f449f3b8.js';
+export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-f449f3b8.js';
 import { BaseElement, Rect } from '@midscene/shared/types';
 export { BaseElement, ElementTreeNode, Point, Rect, Size } from '@midscene/shared/types';
 export { getVersion } from './utils.js';
@@ -39,7 +39,7 @@ declare class Insight<ElementType extends BaseElement = BaseElement, ContextType
     taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
     constructor(context: ContextType | ((action: InsightAction) => Promise<ContextType> | ContextType), opt?: InsightOptions);
     locate(query: DetailedLocateParam, opt?: LocateOpts): Promise<LocateResult>;
-    extract<T>(dataDemand: InsightExtractParam, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<{
+    extract<T>(dataDemand: InsightExtractParam, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt, modelPreferences?: IModelPreferences): Promise<{
         data: T;
         thought?: string;
         usage?: AIUsageInfo;
@@ -50,4 +50,4 @@ declare class Insight<ElementType extends BaseElement = BaseElement, ContextType
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
 }
 
-export { AIDescribeElementResponse, AIUsageInfo, DetailedLocateParam, DumpSubscriber, ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, Executor, Insight, InsightAction, InsightAssertionResponse, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, TMultimodalPrompt, TUserPrompt, UIContext, Insight as default };
+export { AIDescribeElementResponse, AIUsageInfo, DetailedLocateParam, DumpSubscriber, ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, Executor, IModelPreferences, Insight, InsightAction, InsightAssertionResponse, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, TMultimodalPrompt, TUserPrompt, UIContext, Insight as default };

package/dist/types/{llm-planning-45dd50cd.d.ts → llm-planning-f449f3b8.d.ts}

@@ -1,4 +1,4 @@
-import { A as AIUsageInfo, U as UIContext, i as TUserPrompt, aG as ReferenceImage, t as AIElementLocatorResponse, O as ElementById, T as TMultimodalPrompt, h as InsightExtractOption, w as AIDataExtractionResponse, y as AIAssertionResponse, aA as PageType, Y as PlanningAIResponse } from './types-da4fb35b.js';
+import { i as IModelPreferences, A as AIUsageInfo, U as UIContext, j as TUserPrompt, aH as ReferenceImage, u as AIElementLocatorResponse, Q as ElementById, T as TMultimodalPrompt, h as InsightExtractOption, x as AIDataExtractionResponse, z as AIAssertionResponse, aB as PageType, Z as PlanningAIResponse } from './types-7435eba0.js';
 import { Rect, ElementTreeNode, BaseElement } from '@midscene/shared/types';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 
@@ -13,7 +13,7 @@ declare enum AIActionType {
     PLAN = 3,
     DESCRIBE_ELEMENT = 4
 }
-declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
+declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType, modelPreferences?: IModelPreferences): Promise<{
     content: T;
     usage?: AIUsageInfo;
 }>;
@@ -81,6 +81,7 @@ declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseE
     multimodalPrompt?: TMultimodalPrompt;
     context: UIContext<ElementType>;
     extractOption?: InsightExtractOption;
+    modelPreferences?: IModelPreferences;
 }): Promise<{
     parseResult: AIDataExtractionResponse<T>;
     elementById: (idOrIndexId: string) => ElementType;

package/dist/{lib/types-da4fb35b.d.ts → types/types-7435eba0.d.ts}

@@ -563,5 +563,12 @@ type TMultimodalPrompt = {
 type TUserPrompt = string | ({
     prompt: string;
 } & Partial<TMultimodalPrompt>);
+interface IModelPreferences {
+    /**
+     * - VQA: Visual Question Answering
+     * - grounding：short for Visual Grounding
+     */
+    intent: 'VQA' | 'planning' | 'grounding';
+}
 
-export { type PlanningActionParamRightClick as $, type AIUsageInfo as A, type LocateValidatorResult as B, type AgentDescribeElementAtPointResult as C, type DumpSubscriber as D, type ExecutionTask as E, type CallAIFn as F, type EnsureObject as G, type LocateResultElement as H, type InsightAction as I, type DumpMeta as J, type InsightDump as K, type LocateResult as L, type MidsceneYamlScript as M, type LiteUISection as N, type ElementById as O, type PartialInsightDumpFromSDK as P, type OnTaskStartTip as Q, type ReportDumpWithAttributes as R, type AgentWaitForOpt as S, type TMultimodalPrompt as T, UIContext as U, type AgentAssertOpt as V, type PlanningLocateParam as W, type PlanningAction as X, type PlanningAIResponse as Y, type PlanningActionParamTap as Z, type PlanningActionParamHover as _, type ExecutionTaskProgressOptions as a, type MidsceneYamlFlowItemSleep as a$, type PlanningActionParamInputOrKeyPress as a0, type PlanningActionParamScroll as a1, type PlanningActionParamAssert as a2, type PlanningActionParamSleep as a3, type PlanningActionParamError as a4, type PlanningActionParamWaitFor as a5, type PlanningActionParamAndroidLongPress as a6, type PlanningActionParamAndroidPull as a7, type Color as a8, type BaseAgentParserOpt as a9, type PageType as aA, type StreamingCodeGenerationOptions as aB, type StreamingCallback as aC, type CodeGenerationChunk as aD, type StreamingAIResponse as aE, type LocateOption as aF, type ReferenceImage as aG, type scrollParam as aH, type MidsceneYamlScriptEnvBase as aI, type MidsceneYamlScriptWebEnv as aJ, type MidsceneYamlScriptAndroidEnv as aK, type MidsceneYamlScriptEnv as aL, type MidsceneYamlFlowItemAIAction as aM, type MidsceneYamlFlowItemAIAssert as aN, type MidsceneYamlFlowItemAIQuery as aO, type MidsceneYamlFlowItemAINumber as aP, type MidsceneYamlFlowItemAIString as aQ, type MidsceneYamlFlowItemAIAsk as aR, type MidsceneYamlFlowItemAIBoolean as aS, type MidsceneYamlFlowItemAILocate as aT, type MidsceneYamlFlowItemAIWaitFor as aU, type MidsceneYamlFlowItemAITap as aV, type MidsceneYamlFlowItemAIHover as aW, type MidsceneYamlFlowItemAIInput as aX, type MidsceneYamlFlowItemAIKeyboardPress as aY, type MidsceneYamlFlowItemAIScroll as aZ, type MidsceneYamlFlowItemEvaluateJavaScript as a_, type PuppeteerParserOpt as aa, type PlaywrightParserOpt as ab, type ExecutionRecorderItem as ac, type ExecutionTaskType as ad, type ExecutorContext as ae, type ExecutionTaskHitBy as af, type ExecutionTaskReturn as ag, type ExecutionTaskInsightLocateParam as ah, type ExecutionTaskInsightLocateOutput as ai, type ExecutionTaskInsightDumpLog as aj, type ExecutionTaskInsightLocateApply as ak, type ExecutionTaskInsightLocate as al, type ExecutionTaskInsightQueryParam as am, type ExecutionTaskInsightQueryOutput as an, type ExecutionTaskInsightQueryApply as ao, type ExecutionTaskInsightQuery as ap, type ExecutionTaskInsightAssertionParam as aq, type ExecutionTaskInsightAssertionApply as ar, type ExecutionTaskInsightAssertion as as, type ExecutionTaskActionApply as at, type ExecutionTaskAction as au, type ExecutionTaskLogApply as av, type ExecutionTaskLog as aw, type ExecutionTaskPlanningApply as ax, type ExecutionTaskPlanning as ay, type GroupedActionDump as az, type ExecutionTaskApply as b, type MidsceneYamlFlowItemLogScreenshot as b0, type FreeFn as b1, type ScriptPlayerTaskStatus as b2, type ScriptPlayerStatusValue as b3, type MidsceneYamlConfig as b4, type MidsceneYamlConfigOutput as b5, type ExecutionDump as c, type InsightTaskInfo as d, type InsightOptions as e, type DetailedLocateParam as f, type InsightExtractParam as g, type InsightExtractOption as h, type TUserPrompt as i, type InsightAssertionResponse as j, type AIDescribeElementResponse as k, type MidsceneYamlTask as l, type MidsceneYamlFlowItem as m, type MidsceneYamlFlowItemAIRightClick as n, type MidsceneYamlConfigResult as o, AIResponseFormat as p, type AISingleElementResponseById as q, type AISingleElementResponseByPosition as r, type AISingleElementResponse as s, type AIElementLocatorResponse as t, type AIElementCoordinatesResponse as u, type AIElementResponse as v, type AIDataExtractionResponse as w, type AISectionLocatorResponse as x, type AIAssertionResponse as y, type LocatorValidatorOption as z };
+export { type PlanningActionParamHover as $, type AIUsageInfo as A, type LocatorValidatorOption as B, type LocateValidatorResult as C, type DumpSubscriber as D, type ExecutionTask as E, type AgentDescribeElementAtPointResult as F, type CallAIFn as G, type EnsureObject as H, type InsightAction as I, type LocateResultElement as J, type DumpMeta as K, type LocateResult as L, type MidsceneYamlScript as M, type InsightDump as N, type LiteUISection as O, type PartialInsightDumpFromSDK as P, type ElementById as Q, type ReportDumpWithAttributes as R, type OnTaskStartTip as S, type TMultimodalPrompt as T, UIContext as U, type AgentWaitForOpt as V, type AgentAssertOpt as W, type PlanningLocateParam as X, type PlanningAction as Y, type PlanningAIResponse as Z, type PlanningActionParamTap as _, type ExecutionTaskProgressOptions as a, type MidsceneYamlFlowItemEvaluateJavaScript as a$, type PlanningActionParamRightClick as a0, type PlanningActionParamInputOrKeyPress as a1, type PlanningActionParamScroll as a2, type PlanningActionParamAssert as a3, type PlanningActionParamSleep as a4, type PlanningActionParamError as a5, type PlanningActionParamWaitFor as a6, type PlanningActionParamAndroidLongPress as a7, type PlanningActionParamAndroidPull as a8, type Color as a9, type GroupedActionDump as aA, type PageType as aB, type StreamingCodeGenerationOptions as aC, type StreamingCallback as aD, type CodeGenerationChunk as aE, type StreamingAIResponse as aF, type LocateOption as aG, type ReferenceImage as aH, type scrollParam as aI, type MidsceneYamlScriptEnvBase as aJ, type MidsceneYamlScriptWebEnv as aK, type MidsceneYamlScriptAndroidEnv as aL, type MidsceneYamlScriptEnv as aM, type MidsceneYamlFlowItemAIAction as aN, type MidsceneYamlFlowItemAIAssert as aO, type MidsceneYamlFlowItemAIQuery as aP, type MidsceneYamlFlowItemAINumber as aQ, type MidsceneYamlFlowItemAIString as aR, type MidsceneYamlFlowItemAIAsk as aS, type MidsceneYamlFlowItemAIBoolean as aT, type MidsceneYamlFlowItemAILocate as aU, type MidsceneYamlFlowItemAIWaitFor as aV, type MidsceneYamlFlowItemAITap as aW, type MidsceneYamlFlowItemAIHover as aX, type MidsceneYamlFlowItemAIInput as aY, type MidsceneYamlFlowItemAIKeyboardPress as aZ, type MidsceneYamlFlowItemAIScroll as a_, type BaseAgentParserOpt as aa, type PuppeteerParserOpt as ab, type PlaywrightParserOpt as ac, type ExecutionRecorderItem as ad, type ExecutionTaskType as ae, type ExecutorContext as af, type ExecutionTaskHitBy as ag, type ExecutionTaskReturn as ah, type ExecutionTaskInsightLocateParam as ai, type ExecutionTaskInsightLocateOutput as aj, type ExecutionTaskInsightDumpLog as ak, type ExecutionTaskInsightLocateApply as al, type ExecutionTaskInsightLocate as am, type ExecutionTaskInsightQueryParam as an, type ExecutionTaskInsightQueryOutput as ao, type ExecutionTaskInsightQueryApply as ap, type ExecutionTaskInsightQuery as aq, type ExecutionTaskInsightAssertionParam as ar, type ExecutionTaskInsightAssertionApply as as, type ExecutionTaskInsightAssertion as at, type ExecutionTaskActionApply as au, type ExecutionTaskAction as av, type ExecutionTaskLogApply as aw, type ExecutionTaskLog as ax, type ExecutionTaskPlanningApply as ay, type ExecutionTaskPlanning as az, type ExecutionTaskApply as b, type MidsceneYamlFlowItemSleep as b0, type MidsceneYamlFlowItemLogScreenshot as b1, type FreeFn as b2, type ScriptPlayerTaskStatus as b3, type ScriptPlayerStatusValue as b4, type MidsceneYamlConfig as b5, type MidsceneYamlConfigOutput as b6, type ExecutionDump as c, type InsightTaskInfo as d, type InsightOptions as e, type DetailedLocateParam as f, type InsightExtractParam as g, type InsightExtractOption as h, type IModelPreferences as i, type TUserPrompt as j, type InsightAssertionResponse as k, type AIDescribeElementResponse as l, type MidsceneYamlTask as m, type MidsceneYamlFlowItem as n, type MidsceneYamlFlowItemAIRightClick as o, type MidsceneYamlConfigResult as p, AIResponseFormat as q, type AISingleElementResponseById as r, type AISingleElementResponseByPosition as s, type AISingleElementResponse as t, type AIElementLocatorResponse as u, type AIElementCoordinatesResponse as v, type AIElementResponse as w, type AIDataExtractionResponse as x, type AISectionLocatorResponse as y, type AIAssertionResponse as z };

package/dist/types/utils.d.ts

@@ -1,4 +1,4 @@
-import { R as ReportDumpWithAttributes } from './types-da4fb35b.js';
+import { R as ReportDumpWithAttributes } from './types-7435eba0.js';
 import { Rect } from '@midscene/shared/types';
 import '@midscene/shared/constants';
 import 'openai/resources';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.25.4-beta-20250807040242.0",
+  "version": "0.25.4-beta-20250807062119.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
@@ -44,8 +44,8 @@
     "langsmith": "0.3.7",
     "openai": "4.81.0",
     "socks-proxy-agent": "8.0.4",
-    "@midscene/recorder": "0.25.4-beta-20250807040242.0",
-    "@midscene/shared": "0.25.4-beta-20250807040242.0"
+    "@midscene/recorder": "0.25.4-beta-20250807062119.0",
+    "@midscene/shared": "0.25.4-beta-20250807062119.0"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.60.6",