@midscene/core 0.16.10 → 0.16.11-beta-20250512134434.0

package/dist/lib/index.d.ts

@@ -1,7 +1,7 @@
-import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, B as BaseElement, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightAssertionResponse, A as AISingleElementResponse } from './types-8c83481c.js';
-export { t as AIAssertionResponse, r as AIDataExtractionResponse, p as AIElementCoordinatesResponse, o as AIElementLocatorResponse, q as AIElementResponse, l as AIResponseFormat, s as AISectionLocatorResponse, m as AISingleElementResponseById, n as AISingleElementResponseByPosition, k as AIUsageInfo, K as AgentAssertOpt, J as AgentWaitForOpt, a2 as BaseAgentParserOpt, C as CallAIFn, a1 as Color, x as DumpMeta, H as ElementById, j as ElementTreeNode, u as EnsureObject, a5 as ExecutionRecorderItem, an as ExecutionTaskAction, am as ExecutionTaskActionApply, al as ExecutionTaskInsightAssertion, ak as ExecutionTaskInsightAssertionApply, aj as ExecutionTaskInsightAssertionParam, ac as ExecutionTaskInsightDumpLog, ae as ExecutionTaskInsightLocate, ad as ExecutionTaskInsightLocateApply, ab as ExecutionTaskInsightLocateOutput, aa as ExecutionTaskInsightLocateParam, ai as ExecutionTaskInsightQuery, ah as ExecutionTaskInsightQueryApply, ag as ExecutionTaskInsightQueryOutput, af as ExecutionTaskInsightQueryParam, ap as ExecutionTaskPlanning, ao as ExecutionTaskPlanningApply, a9 as ExecutionTaskReturn, a6 as ExecutionTaskType, a7 as ExecutorContext, aN as FreeFn, aq as GroupedActionDump, z as InsightDump, v as InsightExtractParam, G as LiteUISection, as as LocateOption, w as LocateResultElement, i as MidsceneYamlFlowItem, ay as MidsceneYamlFlowItemAIAction, az as MidsceneYamlFlowItemAIAssert, aD as MidsceneYamlFlowItemAIBoolean, aH as MidsceneYamlFlowItemAIHover, aI as MidsceneYamlFlowItemAIInput, aJ as MidsceneYamlFlowItemAIKeyboardPress, aE as MidsceneYamlFlowItemAILocate, aC as MidsceneYamlFlowItemAINString, aB as MidsceneYamlFlowItemAINumber, aA as MidsceneYamlFlowItemAIQuery, aK as MidsceneYamlFlowItemAIScroll, aG as MidsceneYamlFlowItemAITap, aF as MidsceneYamlFlowItemAIWaitFor, aL as MidsceneYamlFlowItemEvaluateJavaScript, aM as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aw as MidsceneYamlScriptAndroidEnv, ax as MidsceneYamlScriptEnv, au as MidsceneYamlScriptEnvBase, av as MidsceneYamlScriptWebEnv, h as MidsceneYamlTask, O as OnTaskStartTip, ar as PageType, F as PartialInsightDumpFromSDK, T as PlanningAIResponse, Q as PlanningAction, Z as PlanningActionParamAssert, $ as PlanningActionParamError, W as PlanningActionParamHover, X as PlanningActionParamInputOrKeyPress, Y as PlanningActionParamScroll, _ as PlanningActionParamSleep, V as PlanningActionParamTap, a0 as PlanningActionParamWaitFor, N as PlanningLocateParam, a4 as PlaywrightParserOpt, P as Point, a3 as PuppeteerParserOpt, R as Rect, y as ReportDumpWithAttributes, aP as ScriptPlayerStatusValue, aO as ScriptPlayerTaskStatus, S as Size, a8 as TaskCacheInfo, at as scrollParam } from './types-8c83481c.js';
-import { c as callAiFn } from './llm-planning-e16d1814.js';
-export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-e16d1814.js';
+import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, B as BaseElement, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightAssertionResponse, A as AISingleElementResponse } from './types-81664b3b.js';
+export { t as AIAssertionResponse, r as AIDataExtractionResponse, p as AIElementCoordinatesResponse, o as AIElementLocatorResponse, q as AIElementResponse, l as AIResponseFormat, s as AISectionLocatorResponse, m as AISingleElementResponseById, n as AISingleElementResponseByPosition, k as AIUsageInfo, K as AgentAssertOpt, J as AgentWaitForOpt, a2 as BaseAgentParserOpt, C as CallAIFn, a1 as Color, x as DumpMeta, H as ElementById, j as ElementTreeNode, u as EnsureObject, a5 as ExecutionRecorderItem, an as ExecutionTaskAction, am as ExecutionTaskActionApply, al as ExecutionTaskInsightAssertion, ak as ExecutionTaskInsightAssertionApply, aj as ExecutionTaskInsightAssertionParam, ac as ExecutionTaskInsightDumpLog, ae as ExecutionTaskInsightLocate, ad as ExecutionTaskInsightLocateApply, ab as ExecutionTaskInsightLocateOutput, aa as ExecutionTaskInsightLocateParam, ai as ExecutionTaskInsightQuery, ah as ExecutionTaskInsightQueryApply, ag as ExecutionTaskInsightQueryOutput, af as ExecutionTaskInsightQueryParam, ap as ExecutionTaskPlanning, ao as ExecutionTaskPlanningApply, a9 as ExecutionTaskReturn, a6 as ExecutionTaskType, a7 as ExecutorContext, aN as FreeFn, aq as GroupedActionDump, z as InsightDump, v as InsightExtractParam, G as LiteUISection, as as LocateOption, w as LocateResultElement, i as MidsceneYamlFlowItem, ay as MidsceneYamlFlowItemAIAction, az as MidsceneYamlFlowItemAIAssert, aD as MidsceneYamlFlowItemAIBoolean, aH as MidsceneYamlFlowItemAIHover, aI as MidsceneYamlFlowItemAIInput, aJ as MidsceneYamlFlowItemAIKeyboardPress, aE as MidsceneYamlFlowItemAILocate, aC as MidsceneYamlFlowItemAINString, aB as MidsceneYamlFlowItemAINumber, aA as MidsceneYamlFlowItemAIQuery, aK as MidsceneYamlFlowItemAIScroll, aG as MidsceneYamlFlowItemAITap, aF as MidsceneYamlFlowItemAIWaitFor, aL as MidsceneYamlFlowItemEvaluateJavaScript, aM as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aw as MidsceneYamlScriptAndroidEnv, ax as MidsceneYamlScriptEnv, au as MidsceneYamlScriptEnvBase, av as MidsceneYamlScriptWebEnv, h as MidsceneYamlTask, O as OnTaskStartTip, ar as PageType, F as PartialInsightDumpFromSDK, T as PlanningAIResponse, Q as PlanningAction, Z as PlanningActionParamAssert, $ as PlanningActionParamError, W as PlanningActionParamHover, X as PlanningActionParamInputOrKeyPress, Y as PlanningActionParamScroll, _ as PlanningActionParamSleep, V as PlanningActionParamTap, a0 as PlanningActionParamWaitFor, N as PlanningLocateParam, a4 as PlaywrightParserOpt, P as Point, a3 as PuppeteerParserOpt, R as Rect, y as ReportDumpWithAttributes, aP as ScriptPlayerStatusValue, aO as ScriptPlayerTaskStatus, S as Size, a8 as TaskCacheInfo, at as scrollParam } from './types-81664b3b.js';
+import { c as callAiFn } from './llm-planning-0867ed8d.js';
+export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-0867ed8d.js';
 export { getVersion } from './utils.js';
 export { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';
 import '@midscene/shared/constants';

package/dist/lib/index.js

@@ -1,6 +1,6 @@
 "use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
 
-var _chunkQIM376LAjs = require('./chunk-QIM376LA.js');
+var _chunkSI2LIZ4Fjs = require('./chunk-SI2LIZ4F.js');
 
 
 
@@ -9,7 +9,7 @@ var _chunkQIM376LAjs = require('./chunk-QIM376LA.js');
 
 
 
-var _chunkFYD4T7QDjs = require('./chunk-FYD4T7QD.js');
+var _chunk35CPO7N5js = require('./chunk-35CPO7N5.js');
 
 // src/ai-model/action-executor.ts
 var _env = require('@midscene/shared/env');
@@ -156,7 +156,7 @@ ${_optionalChain([this, 'access', _7 => _7.latestErrorTask, 'call', _8 => _8(),
   }
   dump() {
     const dumpData = {
-      sdkVersion: _chunkQIM376LAjs.getVersion.call(void 0, ),
+      sdkVersion: _chunkSI2LIZ4Fjs.getVersion.call(void 0, ),
       model_name: _env.getAIConfig.call(void 0, _env.MIDSCENE_MODEL_NAME) || "",
       logTime: Date.now(),
       name: this.name,
@@ -194,7 +194,7 @@ function emitInsightDump(data, dumpSubscriber) {
     }
   }
   const baseData = {
-    sdkVersion: _chunkQIM376LAjs.getVersion.call(void 0, ),
+    sdkVersion: _chunkSI2LIZ4Fjs.getVersion.call(void 0, ),
     logTime: Date.now(),
     model_name: _env.getAIConfig.call(void 0, _env.MIDSCENE_MODEL_NAME) || "",
     model_description: modelDescription
@@ -211,7 +211,7 @@ function emitInsightDump(data, dumpSubscriber) {
 var debug = _logger.getDebug.call(void 0, "ai:insight");
 var Insight = class {
   constructor(context, opt) {
-    this.aiVendorFn = _chunkFYD4T7QDjs.callAiFn;
+    this.aiVendorFn = _chunk35CPO7N5js.callAiFn;
     _utils.assert.call(void 0, context, "context is required for Insight");
     if (typeof context === "function") {
       this.contextRetrieverFn = context;
@@ -257,7 +257,7 @@ var Insight = class {
     let searchAreaUsage = void 0;
     let searchAreaResponse = void 0;
     if (searchAreaPrompt) {
-      searchAreaResponse = await _chunkFYD4T7QDjs.AiLocateSection.call(void 0, {
+      searchAreaResponse = await _chunk35CPO7N5js.AiLocateSection.call(void 0, {
         context,
         sectionDescription: searchAreaPrompt
       });
@@ -270,7 +270,7 @@ var Insight = class {
       searchArea = searchAreaResponse.rect;
     }
     const startTime = Date.now();
-    const { parseResult, rect, elementById, rawResponse, usage } = await _chunkFYD4T7QDjs.AiLocateElement.call(void 0, {
+    const { parseResult, rect, elementById, rawResponse, usage } = await _chunk35CPO7N5js.AiLocateElement.call(void 0, {
       callAI: callAI || this.aiVendorFn,
       context,
       targetElementDescription: queryPrompt,
@@ -309,7 +309,7 @@ ${parseResult.errors.join("\n")}`;
     const elements = [];
     (parseResult.elements || []).forEach((item) => {
       if ("id" in item) {
-        const element = elementById(item.id);
+        const element = elementById(_optionalChain([item, 'optionalAccess', _22 => _22.id]));
         if (!element) {
           console.warn(
             `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`
@@ -339,7 +339,9 @@ ${parseResult.errors.join("\n")}`;
           id: elements[0].id,
           indexId: elements[0].indexId,
           center: elements[0].center,
-          rect: elements[0].rect
+          rect: elements[0].rect,
+          xpaths: elements[0].xpaths || [],
+          attributes: elements[0].attributes
         },
         rect
       };
@@ -358,7 +360,7 @@ ${parseResult.errors.join("\n")}`;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn("extract");
     const startTime = Date.now();
-    const { parseResult, usage } = await _chunkFYD4T7QDjs.AiExtractElementInfo.call(void 0, {
+    const { parseResult, usage } = await _chunk35CPO7N5js.AiExtractElementInfo.call(void 0, {
       context,
       dataQuery: dataDemand
     });
@@ -369,7 +371,7 @@ ${parseResult.errors.join("\n")}`;
       rawResponse: JSON.stringify(parseResult)
     };
     let errorLog;
-    if (_optionalChain([parseResult, 'access', _22 => _22.errors, 'optionalAccess', _23 => _23.length])) {
+    if (_optionalChain([parseResult, 'access', _23 => _23.errors, 'optionalAccess', _24 => _24.length])) {
       errorLog = `AI response error: 
 ${parseResult.errors.join("\n")}`;
     }
@@ -409,7 +411,7 @@ ${parseResult.errors.join("\n")}`;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn("assert");
     const startTime = Date.now();
-    const assertResult = await _chunkFYD4T7QDjs.AiAssert.call(void 0, {
+    const assertResult = await _chunk35CPO7N5js.AiAssert.call(void 0, {
       assertion,
       context
     });
@@ -455,6 +457,6 @@ var src_default = Insight;
 
 
 
-exports.AiAssert = _chunkFYD4T7QDjs.AiAssert; exports.AiLocateElement = _chunkFYD4T7QDjs.AiLocateElement; exports.Executor = Executor; exports.Insight = Insight; exports.MIDSCENE_MODEL_NAME = _env.MIDSCENE_MODEL_NAME; exports.default = src_default; exports.describeUserPage = _chunkFYD4T7QDjs.describeUserPage; exports.getAIConfig = _env.getAIConfig; exports.getVersion = _chunkQIM376LAjs.getVersion; exports.plan = _chunkFYD4T7QDjs.plan;
+exports.AiAssert = _chunk35CPO7N5js.AiAssert; exports.AiLocateElement = _chunk35CPO7N5js.AiLocateElement; exports.Executor = Executor; exports.Insight = Insight; exports.MIDSCENE_MODEL_NAME = _env.MIDSCENE_MODEL_NAME; exports.default = src_default; exports.describeUserPage = _chunk35CPO7N5js.describeUserPage; exports.getAIConfig = _env.getAIConfig; exports.getVersion = _chunkSI2LIZ4Fjs.getVersion; exports.plan = _chunk35CPO7N5js.plan;
 
 //# sourceMappingURL=index.js.map

package/dist/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"mappings":";;;;;;;;;;;;;;AAUA,SAAS,qBAAqB,mBAAmB;AACjD,SAAS,cAAc;AAEhB,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,SACH,SAAS,SAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,SAAS,SAAS,SAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,SAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AAC3E;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,QAAyD,KAAK,gBAAgB,GAAG,KAAK;AAAA,EAAK,KAAK,gBAAgB,GAAG,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAsB;AAC1B,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,oBAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY,YACjB,KAAK,YAAY,aACjB,KAAK,YAAY,YACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,aACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD,aAAK,OAAO,SAAU,aAAqB,UAAU;AACrD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,QACH,GAAG,YAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,aAAO,KAAK,MAAM,WAAW,EAAE;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;AC1LA;AAAA,EACE;AAAA,EACA;AAAA,EACA,gBAAAA;AAAA,OACK;AACP,SAAS,gBAAgB;AACzB,SAAS,UAAAC,eAAc;;;ACnBvB;AAAA,EACE,uBAAAC;AAAA,EACA,eAAAC;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY;AAEd,SAAS,gBACd,MACA,gBACA;AACA,MAAI,mBAAmB;AAEvB,MAAI,aAAa,GAAG;AAClB,UAAM,iBAAiB,mBAAmB;AAC1C,QAAI,gBAAgB;AAClB,yBAAmB,WAAW,cAAc;AAAA,IAC9C,OAAO;AACL,yBAAmB,GAAG,aAAa,CAAC;AAAA,IACtC;AAAA,EACF;AAEA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAYA,aAAYD,oBAAmB,KAAK;AAAA,IAChD,mBAAmB;AAAA,EACrB;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO,KAAK;AAAA,IACZ,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mBAAiB,SAAS;AAC5B;;;ADLA,IAAM,QAAQ,SAAS,YAAY;AACnC,IAAqB,UAArB,MAGE;AAAA,EAWA,YACE,SAGA,KACA;AAXF,sBAAoD;AAYlD,IAAAD,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,QAAI,OAAO,KAAK,eAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,OAAO,KAAK,aAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,OACJ,OACA,KACuB;AACvB,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,UAAM,cAAc,OAAO,UAAU,WAAW,QAAQ,MAAM;AAC9D,IAAAA;AAAA,MACE,eAAe,KAAK;AAAA,MACpB;AAAA,IACF;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,IAAAA,QAAO,OAAO,UAAU,UAAU,sCAAsC;AAExE,UAAM,wBAAwB;AAAA,MAC5B;AAAA,IACF;AACA,QAAI,uBAAuB;AACzB,YAAM,yBAAyB,qBAAqB;AAAA,IACtD;AACA,QAAI;AACJ,QAAI,MAAM,aAAa,uBAAuB;AAC5C,yBAAmB,MAAM;AAAA,IAC3B;AAEA,QAAI,oBAAoB,CAACD,cAAa,GAAG;AACvC,cAAQ;AAAA,QACN;AAAA,MACF;AACA,yBAAmB;AAAA,IACrB;AAEA,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AAEtD,QAAI,aAA+B;AACnC,QAAI,wBAA4C;AAChD,QAAI,kBAA2C;AAC/C,QAAI,qBAEY;AAChB,QAAI,kBAAkB;AACpB,2BAAqB,MAAM,gBAAgB;AAAA,QACzC;AAAA,QACA,oBAAoB;AAAA,MACtB,CAAC;AACD,MAAAC;AAAA,QACE,mBAAmB;AAAA,QACnB,gCAAgC,gBAAgB,IAC9C,mBAAmB,QAAQ,KAAK,mBAAmB,KAAK,KAAK,EAC/D;AAAA,MACF;AACA,8BAAwB,mBAAmB;AAC3C,wBAAkB,mBAAmB;AACrC,mBAAa,mBAAmB;AAAA,IAClC;AAEA,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,aAAa,aAAa,MAAM,IACzD,MAAM,gBAAgB;AAAA,MACpB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,aAAa,KAAK;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAEH,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAgC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC1E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,gBAAgB,CAAC;AAAA,MACjB,aAAa;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,CAAC,CAAC;AAAA,MACb,OAAO;AAAA,IACT;AAEA,UAAM,WAA0B,CAAC;AACjC,KAAC,YAAY,YAAY,CAAC,GAAG,QAAQ,CAAC,SAAS;AAC7C,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,KAAK,EAAE;AAEnC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,IAAAA;AAAA,MACE,SAAS,UAAU;AAAA,MACnB,6CAA6C,SAAS,MAAM;AAAA,IAC9D;AAEA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,UACP,IAAI,SAAS,CAAC,EAAG;AAAA,UACjB,SAAS,SAAS,CAAC,EAAG;AAAA,UACtB,QAAQ,SAAS,CAAC,EAAG;AAAA,UACrB,MAAM,SAAS,CAAC,EAAG;AAAA,QACrB;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAQA,MAAM,QAAW,YAA+C;AAC9D,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,IACb,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,EAAE,KAAK,IAAI,eAAe,CAAC;AAGjC;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,QAAI,YAAY,CAAC,MAAM;AACrB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAAsD;AACjE,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,cAAc;AAExC,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AACF;;;AEjUA,SAAS,eAAAE,cAAa,uBAAAD,4BAA2B;AAGjD,IAAO,cAAQ","names":["vlLocateMode","assert","MIDSCENE_MODEL_NAME","getAIConfig"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<any> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        task.timing.aiCost = (returnValue as any)?.aiCost || 0;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      return this.tasks[outputIndex].output;\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import { callAiFn } from '@/ai-model/common';\nimport { AiExtractElementInfo, AiLocateElement } from '@/ai-model/index';\nimport { AiAssert, AiLocateSection } from '@/ai-model/inspect';\nimport type {\n  AIElementResponse,\n  AISingleElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_FORCE_DEEP_THINK,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  callAI?: typeof callAiFn<AIElementResponse>;\n  quickAnswer?: Partial<AISingleElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt?: LocateOpts,\n  ): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(\n      queryPrompt || opt?.quickAnswer,\n      'query or quickAnswer is required for locate',\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = getAIConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    if (searchAreaPrompt && !vlLocateMode()) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = await this.contextRetrieverFn('locate');\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, elementById, rawResponse, usage } =\n      await AiLocateElement({\n        callAI: callAI || this.aiVendorFn,\n        context,\n        targetElementDescription: queryPrompt,\n        quickAnswer: opt?.quickAnswer,\n        searchConfig: searchAreaResponse,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      quickAnswer: opt?.quickAnswer,\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T = any>(input: string): Promise<T>;\n  async extract<T extends Record<string, string>>(\n    input: T,\n  ): Promise<Record<keyof T, any>>;\n  async extract<T extends object>(input: Record<keyof T, string>): Promise<T>;\n\n  async extract<T>(dataDemand: InsightExtractParam): Promise<any> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      usage,\n    };\n  }\n\n  async assert(assertion: string): Promise<InsightAssertionResponse> {\n    if (typeof assertion !== 'string') {\n      throw new Error(\n        'This is the assert method for Midscene, the first argument should be a string. If you want to use the assert method from Node.js, please import it from the Node.js assert module.',\n      );\n    }\n\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      userQuery: {\n        assertion,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n}\n","import type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport {\n  MIDSCENE_MODEL_NAME,\n  getAIConfig,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  let modelDescription = '';\n\n  if (vlLocateMode()) {\n    const uiTarsModelVer = uiTarsModelVersion();\n    if (uiTarsModelVer) {\n      modelDescription = `UI-TARS=${uiTarsModelVer}`;\n    } else {\n      modelDescription = `${vlLocateMode()} mode`;\n    }\n  }\n\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n    model_description: modelDescription,\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  AiAssert,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n} from './yaml';\n"]}
+{"version":3,"mappings":";;;;;;;;;;;;;;AAUA,SAAS,qBAAqB,mBAAmB;AACjD,SAAS,cAAc;AAEhB,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,SACH,SAAS,SAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,SAAS,SAAS,SAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,SAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AAC3E;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,QAAyD,KAAK,gBAAgB,GAAG,KAAK;AAAA,EAAK,KAAK,gBAAgB,GAAG,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAsB;AAC1B,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,oBAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY,YACjB,KAAK,YAAY,aACjB,KAAK,YAAY,YACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,aACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD,aAAK,OAAO,SAAU,aAAqB,UAAU;AACrD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,QACH,GAAG,YAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,aAAO,KAAK,MAAM,WAAW,EAAE;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;AC1LA;AAAA,EACE;AAAA,EACA;AAAA,EACA,gBAAAA;AAAA,OACK;AACP,SAAS,gBAAgB;AACzB,SAAS,UAAAC,eAAc;;;ACnBvB;AAAA,EACE,uBAAAC;AAAA,EACA,eAAAC;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY;AAEd,SAAS,gBACd,MACA,gBACA;AACA,MAAI,mBAAmB;AAEvB,MAAI,aAAa,GAAG;AAClB,UAAM,iBAAiB,mBAAmB;AAC1C,QAAI,gBAAgB;AAClB,yBAAmB,WAAW,cAAc;AAAA,IAC9C,OAAO;AACL,yBAAmB,GAAG,aAAa,CAAC;AAAA,IACtC;AAAA,EACF;AAEA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAYA,aAAYD,oBAAmB,KAAK;AAAA,IAChD,mBAAmB;AAAA,EACrB;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO,KAAK;AAAA,IACZ,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mBAAiB,SAAS;AAC5B;;;ADLA,IAAM,QAAQ,SAAS,YAAY;AACnC,IAAqB,UAArB,MAGE;AAAA,EAWA,YACE,SAGA,KACA;AAXF,sBAAoD;AAYlD,IAAAD,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,QAAI,OAAO,KAAK,eAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,OAAO,KAAK,aAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,OACJ,OACA,KACuB;AACvB,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,UAAM,cAAc,OAAO,UAAU,WAAW,QAAQ,MAAM;AAC9D,IAAAA;AAAA,MACE,eAAe,KAAK;AAAA,MACpB;AAAA,IACF;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,IAAAA,QAAO,OAAO,UAAU,UAAU,sCAAsC;AAExE,UAAM,wBAAwB;AAAA,MAC5B;AAAA,IACF;AACA,QAAI,uBAAuB;AACzB,YAAM,yBAAyB,qBAAqB;AAAA,IACtD;AACA,QAAI;AACJ,QAAI,MAAM,aAAa,uBAAuB;AAC5C,yBAAmB,MAAM;AAAA,IAC3B;AAEA,QAAI,oBAAoB,CAACD,cAAa,GAAG;AACvC,cAAQ;AAAA,QACN;AAAA,MACF;AACA,yBAAmB;AAAA,IACrB;AAEA,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AAEtD,QAAI,aAA+B;AACnC,QAAI,wBAA4C;AAChD,QAAI,kBAA2C;AAC/C,QAAI,qBAEY;AAChB,QAAI,kBAAkB;AACpB,2BAAqB,MAAM,gBAAgB;AAAA,QACzC;AAAA,QACA,oBAAoB;AAAA,MACtB,CAAC;AACD,MAAAC;AAAA,QACE,mBAAmB;AAAA,QACnB,gCAAgC,gBAAgB,IAC9C,mBAAmB,QAAQ,KAAK,mBAAmB,KAAK,KAAK,EAC/D;AAAA,MACF;AACA,8BAAwB,mBAAmB;AAC3C,wBAAkB,mBAAmB;AACrC,mBAAa,mBAAmB;AAAA,IAClC;AAEA,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,aAAa,aAAa,MAAM,IACzD,MAAM,gBAAgB;AAAA,MACpB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,aAAa,KAAK;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAEH,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAgC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC1E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,gBAAgB,CAAC;AAAA,MACjB,aAAa;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,CAAC,CAAC;AAAA,MACb,OAAO;AAAA,IACT;AAEA,UAAM,WAA0B,CAAC;AACjC,KAAC,YAAY,YAAY,CAAC,GAAG,QAAQ,CAAC,SAAS;AAC7C,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,MAAM,EAAE;AAEpC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,IAAAA;AAAA,MACE,SAAS,UAAU;AAAA,MACnB,6CAA6C,SAAS,MAAM;AAAA,IAC9D;AAEA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,UACP,IAAI,SAAS,CAAC,EAAG;AAAA,UACjB,SAAS,SAAS,CAAC,EAAG;AAAA,UACtB,QAAQ,SAAS,CAAC,EAAG;AAAA,UACrB,MAAM,SAAS,CAAC,EAAG;AAAA,UACnB,QAAQ,SAAS,CAAC,EAAG,UAAU,CAAC;AAAA,UAChC,YAAY,SAAS,CAAC,EAAG;AAAA,QAC3B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAQA,MAAM,QAAW,YAA+C;AAC9D,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,IACb,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,EAAE,KAAK,IAAI,eAAe,CAAC;AAGjC;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,QAAI,YAAY,CAAC,MAAM;AACrB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAAsD;AACjE,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,cAAc;AAExC,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AACF;;;AEnUA,SAAS,eAAAE,cAAa,uBAAAD,4BAA2B;AAGjD,IAAO,cAAQ","names":["vlLocateMode","assert","MIDSCENE_MODEL_NAME","getAIConfig"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<any> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        task.timing.aiCost = (returnValue as any)?.aiCost || 0;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      return this.tasks[outputIndex].output;\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import { callAiFn } from '@/ai-model/common';\nimport { AiExtractElementInfo, AiLocateElement } from '@/ai-model/index';\nimport { AiAssert, AiLocateSection } from '@/ai-model/inspect';\nimport type {\n  AIElementResponse,\n  AISingleElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_FORCE_DEEP_THINK,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  callAI?: typeof callAiFn<AIElementResponse>;\n  quickAnswer?: Partial<AISingleElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt?: LocateOpts,\n  ): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(\n      queryPrompt || opt?.quickAnswer,\n      'query or quickAnswer is required for locate',\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = getAIConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    if (searchAreaPrompt && !vlLocateMode()) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = await this.contextRetrieverFn('locate');\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, elementById, rawResponse, usage } =\n      await AiLocateElement({\n        callAI: callAI || this.aiVendorFn,\n        context,\n        targetElementDescription: queryPrompt,\n        quickAnswer: opt?.quickAnswer,\n        searchConfig: searchAreaResponse,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      quickAnswer: opt?.quickAnswer,\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T = any>(input: string): Promise<T>;\n  async extract<T extends Record<string, string>>(\n    input: T,\n  ): Promise<Record<keyof T, any>>;\n  async extract<T extends object>(input: Record<keyof T, string>): Promise<T>;\n\n  async extract<T>(dataDemand: InsightExtractParam): Promise<any> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      usage,\n    };\n  }\n\n  async assert(assertion: string): Promise<InsightAssertionResponse> {\n    if (typeof assertion !== 'string') {\n      throw new Error(\n        'This is the assert method for Midscene, the first argument should be a string. If you want to use the assert method from Node.js, please import it from the Node.js assert module.',\n      );\n    }\n\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      userQuery: {\n        assertion,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n}\n","import type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport {\n  MIDSCENE_MODEL_NAME,\n  getAIConfig,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  let modelDescription = '';\n\n  if (vlLocateMode()) {\n    const uiTarsModelVer = uiTarsModelVersion();\n    if (uiTarsModelVer) {\n      modelDescription = `UI-TARS=${uiTarsModelVer}`;\n    } else {\n      modelDescription = `${vlLocateMode()} mode`;\n    }\n  }\n\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n    model_description: modelDescription,\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  AiAssert,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n} from './yaml';\n"]}

package/dist/lib/{llm-planning-e16d1814.d.ts → llm-planning-0867ed8d.d.ts}

@@ -1,4 +1,4 @@
-import { k as AIUsageInfo, R as Rect, B as BaseElement, U as UIContext, A as AISingleElementResponse, n as AISingleElementResponseByPosition, o as AIElementLocatorResponse, H as ElementById, r as AIDataExtractionResponse, t as AIAssertionResponse, ar as PageType, T as PlanningAIResponse } from './types-8c83481c.js';
+import { k as AIUsageInfo, R as Rect, j as ElementTreeNode, B as BaseElement, U as UIContext, A as AISingleElementResponse, n as AISingleElementResponseByPosition, o as AIElementLocatorResponse, H as ElementById, r as AIDataExtractionResponse, t as AIAssertionResponse, ar as PageType, T as PlanningAIResponse } from './types-81664b3b.js';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 
 type AIArgs = [
@@ -17,6 +17,10 @@ declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Pro
 }>;
 declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number): Rect;
 
+declare function elementByPositionWithElementInfo(treeRoot: ElementTreeNode<BaseElement>, position: {
+    x: number;
+    y: number;
+}, strict?: boolean): BaseElement | undefined;
 declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, opt?: {
     truncateTextLength?: number;
     filterNonTextContent?: boolean;
@@ -88,4 +92,4 @@ declare function plan(userInstruction: string, opts: {
     actionContext?: string;
 }): Promise<PlanningAIResponse>;
 
-export { AiLocateElement as A, AiAssert as a, AIActionType as b, callAiFn as c, describeUserPage as d, AiExtractElementInfo as e, AiLocateSection as f, adaptBboxToRect as g, plan as p };
+export { AiLocateElement as A, AiAssert as a, AIActionType as b, callAiFn as c, describeUserPage as d, elementByPositionWithElementInfo as e, AiExtractElementInfo as f, AiLocateSection as g, adaptBboxToRect as h, plan as p };

package/dist/lib/tree.d.ts

@@ -1,5 +1,5 @@
 import * as _midscene_shared_constants from '@midscene/shared/constants';
-import { B as BaseElement, j as ElementTreeNode } from './types-8c83481c.js';
+import { B as BaseElement, j as ElementTreeNode } from './types-81664b3b.js';
 import 'openai/resources';
 
 declare function truncateText(text: string | number | object | undefined, maxLength?: number): string;

package/dist/{types/types-8c83481c.d.ts → lib/types-81664b3b.d.ts}

@@ -143,6 +143,7 @@ declare abstract class BaseElement {
     abstract rect: Rect;
     abstract center: [number, number];
     abstract locator?: string;
+    abstract xpaths?: string[];
 }
 interface ElementTreeNode<ElementType extends BaseElement = BaseElement> {
     node: ElementType | null;
@@ -165,6 +166,7 @@ type AISingleElementResponseById = {
     id: string;
     reason?: string;
     text?: string;
+    xpaths?: string[];
 };
 type AISingleElementResponseByPosition = {
     position?: {
@@ -181,6 +183,7 @@ interface AIElementLocatorResponse {
         id: string;
         reason?: string;
         text?: string;
+        xpaths?: string[];
     }[];
     bbox?: [number, number, number, number];
     errors?: string[];
@@ -230,6 +233,11 @@ type LocateResultElement = {
     indexId?: number;
     center: [number, number];
     rect: Rect;
+    xpaths: string[];
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
 };
 interface LocateResult {
     element: LocateResultElement | null;

package/dist/lib/utils.d.ts

@@ -1,4 +1,4 @@
-import { y as ReportDumpWithAttributes, R as Rect } from './types-8c83481c.js';
+import { y as ReportDumpWithAttributes, R as Rect } from './types-81664b3b.js';
 import '@midscene/shared/constants';
 import 'openai/resources';
 

package/dist/lib/utils.js

@@ -13,7 +13,7 @@
 
 
 
-var _chunkQIM376LAjs = require('./chunk-QIM376LA.js');
+var _chunkSI2LIZ4Fjs = require('./chunk-SI2LIZ4F.js');
 
 
 
@@ -29,4 +29,4 @@ var _chunkQIM376LAjs = require('./chunk-QIM376LA.js');
 
 
 
-exports.getLogDir = _chunkQIM376LAjs.getLogDir; exports.getTmpDir = _chunkQIM376LAjs.getTmpDir; exports.getTmpFile = _chunkQIM376LAjs.getTmpFile; exports.getVersion = _chunkQIM376LAjs.getVersion; exports.groupedActionDumpFileExt = _chunkQIM376LAjs.groupedActionDumpFileExt; exports.overlapped = _chunkQIM376LAjs.overlapped; exports.replaceStringWithFirstAppearance = _chunkQIM376LAjs.replaceStringWithFirstAppearance; exports.replacerForPageObject = _chunkQIM376LAjs.replacerForPageObject; exports.reportHTMLContent = _chunkQIM376LAjs.reportHTMLContent; exports.sleep = _chunkQIM376LAjs.sleep; exports.stringifyDumpData = _chunkQIM376LAjs.stringifyDumpData; exports.uploadTestInfoToServer = _chunkQIM376LAjs.uploadTestInfoToServer; exports.writeDumpReport = _chunkQIM376LAjs.writeDumpReport; exports.writeLogFile = _chunkQIM376LAjs.writeLogFile;
+exports.getLogDir = _chunkSI2LIZ4Fjs.getLogDir; exports.getTmpDir = _chunkSI2LIZ4Fjs.getTmpDir; exports.getTmpFile = _chunkSI2LIZ4Fjs.getTmpFile; exports.getVersion = _chunkSI2LIZ4Fjs.getVersion; exports.groupedActionDumpFileExt = _chunkSI2LIZ4Fjs.groupedActionDumpFileExt; exports.overlapped = _chunkSI2LIZ4Fjs.overlapped; exports.replaceStringWithFirstAppearance = _chunkSI2LIZ4Fjs.replaceStringWithFirstAppearance; exports.replacerForPageObject = _chunkSI2LIZ4Fjs.replacerForPageObject; exports.reportHTMLContent = _chunkSI2LIZ4Fjs.reportHTMLContent; exports.sleep = _chunkSI2LIZ4Fjs.sleep; exports.stringifyDumpData = _chunkSI2LIZ4Fjs.stringifyDumpData; exports.uploadTestInfoToServer = _chunkSI2LIZ4Fjs.uploadTestInfoToServer; exports.writeDumpReport = _chunkSI2LIZ4Fjs.writeDumpReport; exports.writeLogFile = _chunkSI2LIZ4Fjs.writeLogFile;

package/dist/types/ai-model.d.ts

@@ -1,8 +1,8 @@
-import { k as AIUsageInfo, Q as PlanningAction } from './types-8c83481c.js';
+import { k as AIUsageInfo, Q as PlanningAction } from './types-81664b3b.js';
 import { ChatCompletionMessageParam } from 'openai/resources';
 export { ChatCompletionMessageParam } from 'openai/resources';
-import { b as AIActionType } from './llm-planning-e16d1814.js';
-export { a as AiAssert, e as AiExtractElementInfo, A as AiLocateElement, f as AiLocateSection, g as adaptBboxToRect, c as callAiFn, d as describeUserPage, p as plan } from './llm-planning-e16d1814.js';
+import { b as AIActionType } from './llm-planning-0867ed8d.js';
+export { a as AiAssert, f as AiExtractElementInfo, A as AiLocateElement, g as AiLocateSection, h as adaptBboxToRect, c as callAiFn, d as describeUserPage, e as elementByPositionWithElementInfo, p as plan } from './llm-planning-0867ed8d.js';
 import { vlLocateMode } from '@midscene/shared/env';
 import { actionParser } from '@ui-tars/action-parser';
 import '@midscene/shared/constants';

package/dist/types/index.d.ts

@@ -1,7 +1,7 @@
-import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, B as BaseElement, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightAssertionResponse, A as AISingleElementResponse } from './types-8c83481c.js';
-export { t as AIAssertionResponse, r as AIDataExtractionResponse, p as AIElementCoordinatesResponse, o as AIElementLocatorResponse, q as AIElementResponse, l as AIResponseFormat, s as AISectionLocatorResponse, m as AISingleElementResponseById, n as AISingleElementResponseByPosition, k as AIUsageInfo, K as AgentAssertOpt, J as AgentWaitForOpt, a2 as BaseAgentParserOpt, C as CallAIFn, a1 as Color, x as DumpMeta, H as ElementById, j as ElementTreeNode, u as EnsureObject, a5 as ExecutionRecorderItem, an as ExecutionTaskAction, am as ExecutionTaskActionApply, al as ExecutionTaskInsightAssertion, ak as ExecutionTaskInsightAssertionApply, aj as ExecutionTaskInsightAssertionParam, ac as ExecutionTaskInsightDumpLog, ae as ExecutionTaskInsightLocate, ad as ExecutionTaskInsightLocateApply, ab as ExecutionTaskInsightLocateOutput, aa as ExecutionTaskInsightLocateParam, ai as ExecutionTaskInsightQuery, ah as ExecutionTaskInsightQueryApply, ag as ExecutionTaskInsightQueryOutput, af as ExecutionTaskInsightQueryParam, ap as ExecutionTaskPlanning, ao as ExecutionTaskPlanningApply, a9 as ExecutionTaskReturn, a6 as ExecutionTaskType, a7 as ExecutorContext, aN as FreeFn, aq as GroupedActionDump, z as InsightDump, v as InsightExtractParam, G as LiteUISection, as as LocateOption, w as LocateResultElement, i as MidsceneYamlFlowItem, ay as MidsceneYamlFlowItemAIAction, az as MidsceneYamlFlowItemAIAssert, aD as MidsceneYamlFlowItemAIBoolean, aH as MidsceneYamlFlowItemAIHover, aI as MidsceneYamlFlowItemAIInput, aJ as MidsceneYamlFlowItemAIKeyboardPress, aE as MidsceneYamlFlowItemAILocate, aC as MidsceneYamlFlowItemAINString, aB as MidsceneYamlFlowItemAINumber, aA as MidsceneYamlFlowItemAIQuery, aK as MidsceneYamlFlowItemAIScroll, aG as MidsceneYamlFlowItemAITap, aF as MidsceneYamlFlowItemAIWaitFor, aL as MidsceneYamlFlowItemEvaluateJavaScript, aM as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aw as MidsceneYamlScriptAndroidEnv, ax as MidsceneYamlScriptEnv, au as MidsceneYamlScriptEnvBase, av as MidsceneYamlScriptWebEnv, h as MidsceneYamlTask, O as OnTaskStartTip, ar as PageType, F as PartialInsightDumpFromSDK, T as PlanningAIResponse, Q as PlanningAction, Z as PlanningActionParamAssert, $ as PlanningActionParamError, W as PlanningActionParamHover, X as PlanningActionParamInputOrKeyPress, Y as PlanningActionParamScroll, _ as PlanningActionParamSleep, V as PlanningActionParamTap, a0 as PlanningActionParamWaitFor, N as PlanningLocateParam, a4 as PlaywrightParserOpt, P as Point, a3 as PuppeteerParserOpt, R as Rect, y as ReportDumpWithAttributes, aP as ScriptPlayerStatusValue, aO as ScriptPlayerTaskStatus, S as Size, a8 as TaskCacheInfo, at as scrollParam } from './types-8c83481c.js';
-import { c as callAiFn } from './llm-planning-e16d1814.js';
-export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-e16d1814.js';
+import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, B as BaseElement, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightAssertionResponse, A as AISingleElementResponse } from './types-81664b3b.js';
+export { t as AIAssertionResponse, r as AIDataExtractionResponse, p as AIElementCoordinatesResponse, o as AIElementLocatorResponse, q as AIElementResponse, l as AIResponseFormat, s as AISectionLocatorResponse, m as AISingleElementResponseById, n as AISingleElementResponseByPosition, k as AIUsageInfo, K as AgentAssertOpt, J as AgentWaitForOpt, a2 as BaseAgentParserOpt, C as CallAIFn, a1 as Color, x as DumpMeta, H as ElementById, j as ElementTreeNode, u as EnsureObject, a5 as ExecutionRecorderItem, an as ExecutionTaskAction, am as ExecutionTaskActionApply, al as ExecutionTaskInsightAssertion, ak as ExecutionTaskInsightAssertionApply, aj as ExecutionTaskInsightAssertionParam, ac as ExecutionTaskInsightDumpLog, ae as ExecutionTaskInsightLocate, ad as ExecutionTaskInsightLocateApply, ab as ExecutionTaskInsightLocateOutput, aa as ExecutionTaskInsightLocateParam, ai as ExecutionTaskInsightQuery, ah as ExecutionTaskInsightQueryApply, ag as ExecutionTaskInsightQueryOutput, af as ExecutionTaskInsightQueryParam, ap as ExecutionTaskPlanning, ao as ExecutionTaskPlanningApply, a9 as ExecutionTaskReturn, a6 as ExecutionTaskType, a7 as ExecutorContext, aN as FreeFn, aq as GroupedActionDump, z as InsightDump, v as InsightExtractParam, G as LiteUISection, as as LocateOption, w as LocateResultElement, i as MidsceneYamlFlowItem, ay as MidsceneYamlFlowItemAIAction, az as MidsceneYamlFlowItemAIAssert, aD as MidsceneYamlFlowItemAIBoolean, aH as MidsceneYamlFlowItemAIHover, aI as MidsceneYamlFlowItemAIInput, aJ as MidsceneYamlFlowItemAIKeyboardPress, aE as MidsceneYamlFlowItemAILocate, aC as MidsceneYamlFlowItemAINString, aB as MidsceneYamlFlowItemAINumber, aA as MidsceneYamlFlowItemAIQuery, aK as MidsceneYamlFlowItemAIScroll, aG as MidsceneYamlFlowItemAITap, aF as MidsceneYamlFlowItemAIWaitFor, aL as MidsceneYamlFlowItemEvaluateJavaScript, aM as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aw as MidsceneYamlScriptAndroidEnv, ax as MidsceneYamlScriptEnv, au as MidsceneYamlScriptEnvBase, av as MidsceneYamlScriptWebEnv, h as MidsceneYamlTask, O as OnTaskStartTip, ar as PageType, F as PartialInsightDumpFromSDK, T as PlanningAIResponse, Q as PlanningAction, Z as PlanningActionParamAssert, $ as PlanningActionParamError, W as PlanningActionParamHover, X as PlanningActionParamInputOrKeyPress, Y as PlanningActionParamScroll, _ as PlanningActionParamSleep, V as PlanningActionParamTap, a0 as PlanningActionParamWaitFor, N as PlanningLocateParam, a4 as PlaywrightParserOpt, P as Point, a3 as PuppeteerParserOpt, R as Rect, y as ReportDumpWithAttributes, aP as ScriptPlayerStatusValue, aO as ScriptPlayerTaskStatus, S as Size, a8 as TaskCacheInfo, at as scrollParam } from './types-81664b3b.js';
+import { c as callAiFn } from './llm-planning-0867ed8d.js';
+export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-0867ed8d.js';
 export { getVersion } from './utils.js';
 export { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';
 import '@midscene/shared/constants';

package/dist/types/{llm-planning-e16d1814.d.ts → llm-planning-0867ed8d.d.ts}

@@ -1,4 +1,4 @@
-import { k as AIUsageInfo, R as Rect, B as BaseElement, U as UIContext, A as AISingleElementResponse, n as AISingleElementResponseByPosition, o as AIElementLocatorResponse, H as ElementById, r as AIDataExtractionResponse, t as AIAssertionResponse, ar as PageType, T as PlanningAIResponse } from './types-8c83481c.js';
+import { k as AIUsageInfo, R as Rect, j as ElementTreeNode, B as BaseElement, U as UIContext, A as AISingleElementResponse, n as AISingleElementResponseByPosition, o as AIElementLocatorResponse, H as ElementById, r as AIDataExtractionResponse, t as AIAssertionResponse, ar as PageType, T as PlanningAIResponse } from './types-81664b3b.js';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 
 type AIArgs = [
@@ -17,6 +17,10 @@ declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Pro
 }>;
 declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number): Rect;
 
+declare function elementByPositionWithElementInfo(treeRoot: ElementTreeNode<BaseElement>, position: {
+    x: number;
+    y: number;
+}, strict?: boolean): BaseElement | undefined;
 declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, opt?: {
     truncateTextLength?: number;
     filterNonTextContent?: boolean;
@@ -88,4 +92,4 @@ declare function plan(userInstruction: string, opts: {
     actionContext?: string;
 }): Promise<PlanningAIResponse>;
 
-export { AiLocateElement as A, AiAssert as a, AIActionType as b, callAiFn as c, describeUserPage as d, AiExtractElementInfo as e, AiLocateSection as f, adaptBboxToRect as g, plan as p };
+export { AiLocateElement as A, AiAssert as a, AIActionType as b, callAiFn as c, describeUserPage as d, elementByPositionWithElementInfo as e, AiExtractElementInfo as f, AiLocateSection as g, adaptBboxToRect as h, plan as p };

package/dist/types/tree.d.ts

@@ -1,5 +1,5 @@
 import * as _midscene_shared_constants from '@midscene/shared/constants';
-import { B as BaseElement, j as ElementTreeNode } from './types-8c83481c.js';
+import { B as BaseElement, j as ElementTreeNode } from './types-81664b3b.js';
 import 'openai/resources';
 
 declare function truncateText(text: string | number | object | undefined, maxLength?: number): string;

package/dist/{lib/types-8c83481c.d.ts → types/types-81664b3b.d.ts}

@@ -143,6 +143,7 @@ declare abstract class BaseElement {
     abstract rect: Rect;
     abstract center: [number, number];
     abstract locator?: string;
+    abstract xpaths?: string[];
 }
 interface ElementTreeNode<ElementType extends BaseElement = BaseElement> {
     node: ElementType | null;
@@ -165,6 +166,7 @@ type AISingleElementResponseById = {
     id: string;
     reason?: string;
     text?: string;
+    xpaths?: string[];
 };
 type AISingleElementResponseByPosition = {
     position?: {
@@ -181,6 +183,7 @@ interface AIElementLocatorResponse {
         id: string;
         reason?: string;
         text?: string;
+        xpaths?: string[];
     }[];
     bbox?: [number, number, number, number];
     errors?: string[];
@@ -230,6 +233,11 @@ type LocateResultElement = {
     indexId?: number;
     center: [number, number];
     rect: Rect;
+    xpaths: string[];
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
 };
 interface LocateResult {
     element: LocateResultElement | null;

package/dist/types/utils.d.ts

@@ -1,4 +1,4 @@
-import { y as ReportDumpWithAttributes, R as Rect } from './types-8c83481c.js';
+import { y as ReportDumpWithAttributes, R as Rect } from './types-81664b3b.js';
 import '@midscene/shared/constants';
 import 'openai/resources';
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.16.10",
+  "version": "0.16.11-beta-20250512134434.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
@@ -44,7 +44,7 @@
     "langsmith": "0.3.7",
     "openai": "4.81.0",
     "socks-proxy-agent": "8.0.4",
-    "@midscene/shared": "0.16.10"
+    "@midscene/shared": "0.16.11-beta-20250512134434.0"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.60.6",