npm - @midscene/core - Versions diffs - 1.9.8 → 1.10.0 - Mend

@midscene/core 1.9.8 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (256) hide show

package/dist/lib/agent/run-gherkin-scenario.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"agent/run-gherkin-scenario.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/run-gherkin-scenario.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { AiActOptions } from './agent';\n\nexport type GherkinStepKeyword = 'Given' | 'When' | 'Then' | 'And' | 'But';\n\ntype RunGherkinScenarioStepAction = 'aiAct' | 'aiAssert';\n\nexport type RunGherkinScenarioOptions = AiActOptions & {\n context?: string;\n};\n\ntype GherkinPrimaryKeyword = 'Given' | 'When' | 'Then';\n\ntype ParsedGherkinStep = {\n keyword: GherkinStepKeyword;\n effectiveKeyword: GherkinPrimaryKeyword;\n text: string;\n lineNumber: number;\n};\n\ntype RawGherkinStep = {\n keyword: GherkinStepKeyword;\n text: string;\n lineNumber: number;\n};\n\ntype ParsedGherkinScenario = {\n scenario?: string;\n steps: ParsedGherkinStep[];\n};\n\ntype GherkinScenarioAgent = {\n aiAct: (taskPrompt: string, opt?: AiActOptions) => Promise<unknown>;\n aiAssert: (\n assertion: string,\n msg?: string,\n opt?: { context?: string; abortSignal?: AbortSignal },\n ) => Promise<unknown>;\n};\n\nconst stepKeywordPattern = /^(Given|When|Then|And|But)\\s+(.+)$/i;\nconst headerPattern = /^(Scenario):\\s*(.*)$/i;\nconst unsupportedHeaderPattern =\n /^(Feature|Background|Scenario Outline|Scenario Template|Examples|Rule):/i;\n\nconst normalizeStepKeyword = (keyword: string): GherkinStepKeyword => {\n const lowerKeyword = keyword.toLowerCase();\n if (lowerKeyword === 'given') return 'Given';\n if (lowerKeyword === 'when') return 'When';\n if (lowerKeyword === 'then') return 'Then';\n if (lowerKeyword === 'and') return 'And';\n return 'But';\n};\n\nconst isPrimaryKeyword = (\n keyword: GherkinStepKeyword,\n): keyword is GherkinPrimaryKeyword => {\n return keyword === 'Given' || keyword === 'When' || keyword === 'Then';\n};\n\nconst resolveSteps = (steps: RawGherkinStep[]): ParsedGherkinStep[] => {\n let previousPrimaryKeyword: GherkinPrimaryKeyword | undefined;\n\n return steps.map((step) => {\n const effectiveKeyword = isPrimaryKeyword(step.keyword)\n ? step.keyword\n : previousPrimaryKeyword;\n\n if (!effectiveKeyword) {\n throw new Error(\n `runGherkinScenario cannot resolve \"${step.keyword}\" at line ${step.lineNumber}; use Given, When, or Then before ${step.keyword}.`,\n );\n }\n\n previousPrimaryKeyword = effectiveKeyword;\n\n return {\n ...step,\n effectiveKeyword,\n };\n });\n};\n\nconst throwIfAborted = (abortSignal: AbortSignal | undefined) => {\n if (!abortSignal?.aborted) {\n return;\n }\n\n if (typeof abortSignal.throwIfAborted === 'function') {\n abortSignal.throwIfAborted();\n }\n\n throw new Error(\n `runGherkinScenario aborted: ${abortSignal.reason || 'signal already aborted'}`,\n );\n};\n\nexport const parseGherkinScenario = (\n scenarioText: string,\n): ParsedGherkinScenario => {\n const lines = scenarioText.split(/\\r?\\n/);\n const scenarioSteps: RawGherkinStep[] = [];\n const anonymousSteps: RawGherkinStep[] = [];\n let scenario: string | undefined;\n let scenarioCount = 0;\n let section: 'prelude' | 'scenario' = 'prelude';\n\n for (const [lineIndex, rawLine] of lines.entries()) {\n const lineNumber = lineIndex + 1;\n const line = rawLine.trim();\n\n if (!line || line.startsWith('#') || line.startsWith('@')) {\n continue;\n }\n\n if (line === '\"\"\"' || line === \"'''\") {\n throw new Error(\n `runGherkinScenario does not support doc strings; found one at line ${lineNumber}.`,\n );\n }\n\n if (line.startsWith('|')) {\n throw new Error(\n `runGherkinScenario does not support data tables; found one at line ${lineNumber}.`,\n );\n }\n\n if (unsupportedHeaderPattern.test(line)) {\n throw new Error(\n `runGherkinScenario does not support \"${line}\" at line ${lineNumber}.`,\n );\n }\n\n const headerMatch = line.match(headerPattern);\n if (headerMatch) {\n scenarioCount += 1;\n scenario = headerMatch[2].trim() || undefined;\n section = 'scenario';\n continue;\n }\n\n const stepMatch = line.match(stepKeywordPattern);\n if (stepMatch) {\n const step = {\n keyword: normalizeStepKeyword(stepMatch[1]),\n text: stepMatch[2].trim(),\n lineNumber,\n };\n\n if (section === 'scenario') {\n scenarioSteps.push(step);\n } else {\n anonymousSteps.push(step);\n }\n\n continue;\n }\n\n throw new Error(\n `runGherkinScenario does not support content at line ${lineNumber}: ${line}`,\n );\n }\n\n if (scenarioCount > 1) {\n throw new Error(\n `runGherkinScenario expects exactly one Scenario, but found ${scenarioCount}.`,\n );\n }\n\n if (scenarioCount === 1 && anonymousSteps.length > 0) {\n throw new Error(\n 'runGherkinScenario cannot mix anonymous steps with a Scenario block.',\n );\n }\n\n const rawSteps = scenarioCount === 1 ? scenarioSteps : anonymousSteps;\n\n if (rawSteps.length === 0) {\n throw new Error('runGherkinScenario requires at least one Gherkin step.');\n }\n\n return {\n scenario,\n steps: resolveSteps(rawSteps),\n };\n};\n\nconst buildStepPrompt = (step: ParsedGherkinStep) => {\n if (step.effectiveKeyword === 'Given') {\n return `Set up this precondition: ${step.text}`;\n }\n\n if (step.effectiveKeyword === 'When') {\n return `Perform this user action: ${step.text}`;\n }\n\n return `Verify that ${step.text}`;\n};\n\nconst describeStepExecution = (\n step: ParsedGherkinStep,\n action: RunGherkinScenarioStepAction,\n) => {\n const keywordMapping =\n step.keyword === step.effectiveKeyword\n ? step.keyword\n : `${step.keyword} as ${step.effectiveKeyword}`;\n\n if (step.effectiveKeyword === 'Given') {\n return `setting up the precondition (${keywordMapping} -> ${action})`;\n }\n\n if (step.effectiveKeyword === 'When') {\n return `performing the user action (${keywordMapping} -> ${action})`;\n }\n\n return `verifying the expected result (${keywordMapping} -> ${action})`;\n};\n\nconst describeErrorCause = (error: unknown) => {\n if (error instanceof Error && error.message) {\n return ` Original error: ${error.message}`;\n }\n\n return '';\n};\n\nexport const runGherkinScenario = async (\n agent: GherkinScenarioAgent,\n scenarioText: string,\n opt?: RunGherkinScenarioOptions,\n): Promise<void> => {\n const parsedScenario = parseGherkinScenario(scenarioText);\n const aiActOptions: AiActOptions = {\n ...opt,\n cacheable: false,\n };\n\n for (const step of parsedScenario.steps) {\n throwIfAborted(opt?.abortSignal);\n\n const action: RunGherkinScenarioStepAction =\n step.effectiveKeyword === 'Then' ? 'aiAssert' : 'aiAct';\n const prompt = buildStepPrompt(step);\n\n try {\n if (action === 'aiAct') {\n await agent.aiAct(prompt, aiActOptions);\n } else if (opt?.context || opt?.abortSignal) {\n await agent.aiAssert(prompt, undefined, {\n context: opt.context,\n abortSignal: opt.abortSignal,\n });\n } else {\n await agent.aiAssert(prompt);\n }\n } catch (error) {\n throw new Error(\n `runGherkinScenario failed while ${describeStepExecution(\n step,\n action,\n )} at line ${step.lineNumber}: ${step.keyword} ${step.text}.${describeErrorCause(\n error,\n )}`,\n {\n cause: error,\n },\n );\n }\n }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","stepKeywordPattern","headerPattern","unsupportedHeaderPattern","normalizeStepKeyword","keyword","lowerKeyword","isPrimaryKeyword","resolveSteps","steps","previousPrimaryKeyword","step","effectiveKeyword","Error","throwIfAborted","abortSignal","parseGherkinScenario","scenarioText","lines","scenarioSteps","anonymousSteps","scenario","scenarioCount","section","lineIndex","rawLine","lineNumber","line","headerMatch","undefined","stepMatch","rawSteps","buildStepPrompt","describeStepExecution","action","keywordMapping","describeErrorCause","error","runGherkinScenario","agent","opt","parsedScenario","aiActOptions","prompt"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACiCA,MAAMI,qBAAqB;AAC3B,MAAMC,gBAAgB;AACtB,MAAMC,2BACJ;AAEF,MAAMC,uBAAuB,CAACC;IAC5B,MAAMC,eAAeD,QAAQ,WAAW;IACxC,IAAIC,AAAiB,YAAjBA,cAA0B,OAAO;IACrC,IAAIA,AAAiB,WAAjBA,cAAyB,OAAO;IACpC,IAAIA,AAAiB,WAAjBA,cAAyB,OAAO;IACpC,IAAIA,AAAiB,UAAjBA,cAAwB,OAAO;IACnC,OAAO;AACT;AAEA,MAAMC,mBAAmB,CACvBF,UAEOA,AAAY,YAAZA,WAAuBA,AAAY,WAAZA,WAAsBA,AAAY,WAAZA;AAGtD,MAAMG,eAAe,CAACC;IACpB,IAAIC;IAEJ,OAAOD,MAAM,GAAG,CAAC,CAACE;QAChB,MAAMC,mBAAmBL,iBAAiBI,KAAK,OAAO,IAClDA,KAAK,OAAO,GACZD;QAEJ,IAAI,CAACE,kBACH,MAAM,IAAIC,MACR,CAAC,mCAAmC,EAAEF,KAAK,OAAO,CAAC,UAAU,EAAEA,KAAK,UAAU,CAAC,kCAAkC,EAAEA,KAAK,OAAO,CAAC,CAAC,CAAC;QAItID,yBAAyBE;QAEzB,OAAO;YACL,GAAGD,IAAI;YACPC;QACF;IACF;AACF;AAEA,MAAME,iBAAiB,CAACC;IACtB,IAAI,CAACA,aAAa,SAChB;IAGF,IAAI,AAAsC,cAAtC,OAAOA,YAAY,cAAc,EACnCA,YAAY,cAAc;IAG5B,MAAM,IAAIF,MACR,CAAC,4BAA4B,EAAEE,YAAY,MAAM,IAAI,0BAA0B;AAEnF;AAEO,MAAMC,uBAAuB,CAClCC;IAEA,MAAMC,QAAQD,aAAa,KAAK,CAAC;IACjC,MAAME,gBAAkC,EAAE;IAC1C,MAAMC,iBAAmC,EAAE;IAC3C,IAAIC;IACJ,IAAIC,gBAAgB;IACpB,IAAIC,UAAkC;IAEtC,KAAK,MAAM,CAACC,WAAWC,QAAQ,IAAIP,MAAM,OAAO,GAAI;QAClD,MAAMQ,aAAaF,YAAY;QAC/B,MAAMG,OAAOF,QAAQ,IAAI;QAEzB,IAAI,CAACE,QAAQA,KAAK,UAAU,CAAC,QAAQA,KAAK,UAAU,CAAC,MACnD;QAGF,IAAIA,AAAS,UAATA,QAAkBA,AAAS,UAATA,MACpB,MAAM,IAAId,MACR,CAAC,mEAAmE,EAAEa,WAAW,CAAC,CAAC;QAIvF,IAAIC,KAAK,UAAU,CAAC,MAClB,MAAM,IAAId,MACR,CAAC,mEAAmE,EAAEa,WAAW,CAAC,CAAC;QAIvF,IAAIvB,yBAAyB,IAAI,CAACwB,OAChC,MAAM,IAAId,MACR,CAAC,qCAAqC,EAAEc,KAAK,UAAU,EAAED,WAAW,CAAC,CAAC;QAI1E,MAAME,cAAcD,KAAK,KAAK,CAACzB;QAC/B,IAAI0B,aAAa;YACfN,iBAAiB;YACjBD,WAAWO,WAAW,CAAC,EAAE,CAAC,IAAI,MAAMC;YACpCN,UAAU;YACV;QACF;QAEA,MAAMO,YAAYH,KAAK,KAAK,CAAC1B;QAC7B,IAAI6B,WAAW;YACb,MAAMnB,OAAO;gBACX,SAASP,qBAAqB0B,SAAS,CAAC,EAAE;gBAC1C,MAAMA,SAAS,CAAC,EAAE,CAAC,IAAI;gBACvBJ;YACF;YAEA,IAAIH,AAAY,eAAZA,SACFJ,cAAc,IAAI,CAACR;iBAEnBS,eAAe,IAAI,CAACT;YAGtB;QACF;QAEA,MAAM,IAAIE,MACR,CAAC,oDAAoD,EAAEa,WAAW,EAAE,EAAEC,MAAM;IAEhF;IAEA,IAAIL,gBAAgB,GAClB,MAAM,IAAIT,MACR,CAAC,2DAA2D,EAAES,cAAc,CAAC,CAAC;IAIlF,IAAIA,AAAkB,MAAlBA,iBAAuBF,eAAe,MAAM,GAAG,GACjD,MAAM,IAAIP,MACR;IAIJ,MAAMkB,WAAWT,AAAkB,MAAlBA,gBAAsBH,gBAAgBC;IAEvD,IAAIW,AAAoB,MAApBA,SAAS,MAAM,EACjB,MAAM,IAAIlB,MAAM;IAGlB,OAAO;QACLQ;QACA,OAAOb,aAAauB;IACtB;AACF;AAEA,MAAMC,kBAAkB,CAACrB;IACvB,IAAIA,AAA0B,YAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,0BAA0B,EAAEA,KAAK,IAAI,EAAE;IAGjD,IAAIA,AAA0B,WAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,0BAA0B,EAAEA,KAAK,IAAI,EAAE;IAGjD,OAAO,CAAC,YAAY,EAAEA,KAAK,IAAI,EAAE;AACnC;AAEA,MAAMsB,wBAAwB,CAC5BtB,MACAuB;IAEA,MAAMC,iBACJxB,KAAK,OAAO,KAAKA,KAAK,gBAAgB,GAClCA,KAAK,OAAO,GACZ,GAAGA,KAAK,OAAO,CAAC,IAAI,EAAEA,KAAK,gBAAgB,EAAE;IAEnD,IAAIA,AAA0B,YAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,6BAA6B,EAAEwB,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;IAGvE,IAAIvB,AAA0B,WAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,4BAA4B,EAAEwB,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;IAGtE,OAAO,CAAC,+BAA+B,EAAEC,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;AACzE;AAEA,MAAME,qBAAqB,CAACC;IAC1B,IAAIA,iBAAiBxB,SAASwB,MAAM,OAAO,EACzC,OAAO,CAAC,iBAAiB,EAAEA,MAAM,OAAO,EAAE;IAG5C,OAAO;AACT;AAEO,MAAMC,qBAAqB,OAChCC,OACAtB,cACAuB;IAEA,MAAMC,iBAAiBzB,qBAAqBC;IAC5C,MAAMyB,eAA6B;QACjC,GAAGF,GAAG;QACN,WAAW;IACb;IAEA,KAAK,MAAM7B,QAAQ8B,eAAe,KAAK,CAAE;QACvC3B,eAAe0B,KAAK;QAEpB,MAAMN,SACJvB,AAA0B,WAA1BA,KAAK,gBAAgB,GAAc,aAAa;QAClD,MAAMgC,SAASX,gBAAgBrB;QAE/B,IAAI;YACF,IAAIuB,AAAW,YAAXA,QACF,MAAMK,MAAM,KAAK,CAACI,QAAQD;iBACrB,IAAIF,KAAK,WAAWA,KAAK,aAC9B,MAAMD,MAAM,QAAQ,CAACI,QAAQd,QAAW;gBACtC,SAASW,IAAI,OAAO;gBACpB,aAAaA,IAAI,WAAW;YAC9B;iBAEA,MAAMD,MAAM,QAAQ,CAACI;QAEzB,EAAE,OAAON,OAAO;YACd,MAAM,IAAIxB,MACR,CAAC,gCAAgC,EAAEoB,sBACjCtB,MACAuB,QACA,SAAS,EAAEvB,KAAK,UAAU,CAAC,EAAE,EAAEA,KAAK,OAAO,CAAC,CAAC,EAAEA,KAAK,IAAI,CAAC,CAAC,EAAEyB,mBAC5DC,QACC,EACH;gBACE,OAAOA;YACT;QAEJ;IACF;AACF"}

package/dist/lib/agent/task-builder.js CHANGED Viewed

@@ -28,6 +28,7 @@ __webpack_require__.d(__webpack_exports__, {
     TaskBuilder: ()=>TaskBuilder
 });
 const index_js_namespaceObject = require("../ai-model/index.js");
+const external_common_js_namespaceObject = require("../common.js");
 const external_task_timing_js_namespaceObject = require("../task-timing.js");
 const external_types_js_namespaceObject = require("../types.js");
 const external_utils_js_namespaceObject = require("../utils.js");
@@ -130,11 +131,10 @@ class TaskBuilder {
     async handleActionPlan(plan, context) {
         const planType = plan.type;
         const actionSpace = this.actionSpace;
-        const action = actionSpace.find((item)=>item.name === planType);
+        const action = (0, external_common_js_namespaceObject.findActionInActionSpaceOrThrow)(planType, actionSpace);
         const param = plan.param;
-        if (!action) throw new Error(`Action type '${planType}' not found`);
-        const locateFields = action ? (0, index_js_namespaceObject.findAllMidsceneLocatorField)(action.paramSchema) : [];
-        const requiredLocateFields = action ? (0, index_js_namespaceObject.findAllMidsceneLocatorField)(action.paramSchema, true) : [];
+        const locateFields = (0, index_js_namespaceObject.findAllMidsceneLocatorField)(action.paramSchema);
+        const requiredLocateFields = (0, index_js_namespaceObject.findAllMidsceneLocatorField)(action.paramSchema, true);
         locateFields.forEach((field)=>{
             if (param[field]) {
                 const locatePlan = locatePlanForLocate(param[field]);

package/dist/lib/agent/task-builder.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent/task-builder.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/task-builder.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n AIUsageInfo,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n PlanningLocateParam,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n ifPlanLocateParamHasLocatedPixelBbox,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nfunction invalidLocateElementReason(\n element: LocateResultElement,\n): string \| undefined {\n const values = [\n element.center?.[0],\n element.center?.[1],\n element.rect?.left,\n element.rect?.top,\n element.rect?.width,\n element.rect?.height,\n ];\n if (\n values.some((value) => typeof value !== 'number' \|\| !Number.isFinite(value))\n ) {\n return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n }\n if (element.rect.width <= 0 \|\| element.rect.height <= 0) {\n return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n }\n return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n param: string \| DetailedLocateParam,\n): DetailedLocateParam {\n if (typeof param === 'string') {\n return { prompt: param };\n }\n\n const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n const deepLocate = rest.deepLocate ?? deepThink;\n\n return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = normalizeLocateParam(param);\n const locatePlan: PlanningAction<PlanningLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n planningModel: ModelRuntime;\n defaultModel: ModelRuntime;\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n\n constructor({\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n }\n\n public async build(\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n planningModel,\n defaultModel,\n cacheable,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<PlanningLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<PlanningLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = actionSpace.find((item) => item.name === planType);\n const param = plan.param;\n\n if (!action) {\n throw new Error(`Action type '${planType}' not found`);\n }\n\n const locateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema)\n : [];\n\n const requiredLocateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema, true)\n : [];\n\n locateFields.forEach((field) => {\n if (param[field]) {\n // Always use createLocateTask for all locate params.\n // This ensures cache writing happens even when locatedPixelBbox is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n executor: async (param, taskContext) => {\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug(\n `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.beforeInvokeAction(action.name, param);\n debug(\n `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n })(),\n delayBeforeRunner > 0\n ? sleep(delayBeforeRunner)\n : Promise.resolve(),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n\n if (action.paramSchema) {\n try {\n param = parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n }\n\n setTimingFieldOnce(timing, 'callActionStart');\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(param, taskContext);\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug(\n `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.afterInvokeAction(action.name, param);\n debug(\n `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<PlanningLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n let locateParam = normalizeLocateParam(detailedLocateParam);\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(\n param,\n )\n ? param\n : undefined;\n\n assert(\n param?.prompt \|\| paramWithLocatedPixelBbox,\n `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n param,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n searchAreaRawChoiceMessage:\n dump.taskInfo?.searchAreaRawChoiceMessage,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n task.searchArea = dump.taskInfo?.searchArea;\n if (dump.taskInfo?.searchAreaUsage) {\n task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n const planLocatedElement = paramWithLocatedPixelBbox\n ? matchElementFromPlan(paramWithLocatedPixelBbox)\n : undefined;\n\n // from locatedPixelBbox (direct plan hit)\n // when deepLocate is enabled, locatedPixelBbox should be used as search\n // area hint, not as a final direct hit\n const elementFromPlan = param.deepLocate\n ? undefined\n : planLocatedElement;\n const isPlanDirectHit = !!elementFromPlan;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanDirectHit &&\n param.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [param.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? generateElementByRect(\n // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n transformLogicalRectToScreenshotRect(\n rectFromXpath,\n shrunkShotToLogicalRatio,\n ),\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt \|\| '',\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = param.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanDirectHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n param.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n const elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isCacheHit = !!elementFromCache;\n\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n const timing = taskContext.task.timing;\n if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n param,\n {\n context: uiContext,\n planLocatedElement,\n },\n defaultModel,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n\n const element =\n elementFromPlan \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n if (element) {\n const invalidElementReason = invalidLocateElementReason(element);\n if (invalidElementReason) {\n if (locateDump) {\n throw new ServiceError(invalidElementReason, locateDump);\n }\n throw new Error(invalidElementReason);\n }\n }\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read)\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n !isCacheHit &&\n (!isPlanDirectHit \|\| !locateCacheAlreadyExists) &&\n param?.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n let pointForCache: [number, number] = element.center;\n if (shrunkShotToLogicalRatio !== 1) {\n pointForCache = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n debug(\n 'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n element.center,\n pointForCache,\n );\n }\n\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt,\n modelRuntime: defaultModel,\n },\n );\n if (hasNonEmptyCache(feature)) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${param.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${param.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n hitBy = {\n from: 'Plan',\n context: {\n locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: param.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n onResult?.(element);\n\n return {\n output: {\n element: {\n ...element,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","hasNonEmptyCache","cache","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;AC8BA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPP,OAAO,IAAI,CAACO,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASC,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMvB,QAAQkB,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI/B,KAAK,CAAC+B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaF,oBAAoBL,KAAK,CAAC+B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEkC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC+B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEjC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,oBAAoB,EAAEyB,AAAAA,IAAAA,oCAAAA,oCAAAA,AAAAA,EAAqChC,KAAK,CAAC+B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtC1B,YACAP,KAAK,CAAC+B,MAAM,EACZhB,SACA,CAACmB;oBACClC,KAAK,CAAC+B,MAAM,GAAGG;gBACjB;gBAEFnB,QAAQ,KAAK,CAAC,IAAI,CAACkB;YACrB,OAAO;gBACLE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ElC,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMa,OAKF;YACF,MAAM;YACN,SAASb;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOlB,OAAOqC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtChD,MACE,oBACAkC,UACAvB,OACA,CAAC,4BAA4B,EAAEqC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,IAAAA,sBAAAA,MAAAA,AAAAA,EACEnC,KAAK,CAAC+B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAiB,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBhB,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMiB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCrD,MACE,CAAC,8DAA8D,EAAEoC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDX,MACE,CAAC,2DAA2D,EAAEoC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAgB,oBAAoB,IAChBE,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,MAAM,EAAES,wBAAwB,EAAE,GAAGR;gBACrC,IAAIQ,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQgD,AAAAA,IAAAA,yBAAAA,gBAAAA,AAAAA,EAAiBhD,OAAOyB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOE,OAAY;oBACnB,MAAM,IAAItB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEwB,MAAM,OAAO,CAAC,cAAc,EAAEnD,KAAK,SAAS,CAACE,QAAQ,EACtG;wBAAE,OAAOiD;oBAAM;gBAEnB;gBAGFT,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3BjD,MAAM,kBAAkBoC,OAAO,IAAI;gBACnC,MAAMyB,WAAWzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM0B,eAAe,MAAMD,SAASlD,OAAOqC;gBAC3CG,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC3BjD,MAAM,iBAAiBoC,OAAO,IAAI,EAAE,WAAW0B;gBAE/CX,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI2B,mBAAmB,GACrB,MAAMT,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMS;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpC/D,MACE,CAAC,6DAA6D,EAAEoC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEzB;wBACpDX,MACE,CAAC,0DAA0D,EAAEoC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACqB;IACrB;IAEQ,iBACNlB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOlB,OAAOqC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMqB,4BAA4B1B,AAAAA,IAAAA,oCAAAA,oCAAAA,AAAAA,EAChChC,SAEEA,QACAI;gBAEJ+B,IAAAA,sBAAAA,MAAAA,AAAAA,EACEnC,OAAO,UAAU0D,2BACjB,CAAC,iEAAiE,EAAE5D,KAAK,SAAS,CAChFE,QACC;gBAGL,IAAI,CAACuC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOI,WAAW;gBAElB,MAAM,EAAEQ,wBAAwB,EAAE,GAAGR;gBAErC,IAAIQ,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIgC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA1B,KAAK,KAAK,GAAG2B,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD1B,KAAK,UAAU,GAAG0B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,AAAAA,IAAAA,oCAAAA,oBAAAA,AAAAA,EAAqBP,6BACrBtD;gBAKJ,MAAM8D,kBAAkBlE,MAAM,UAAU,GACpCI,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDnE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFoE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACpE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMqE,mBAAmBD,gBACrBE,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EAEEC,AAAAA,IAAAA,oCAAAA,oCAAAA,AAAAA,EACEH,eACArB,2BAEF,AAAwB,YAAxB,OAAO/C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BI;gBAEJ,MAAMoE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAczE,MAAM,MAAM;gBAChC,MAAM0E,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,AAAAA,IAAAA,oCAAAA,qBAAAA,AAAAA,EACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAzE,MAAM,SAAS;gBAIvB,MAAM8E,mBAAmBF,yBACrBG,AAAAA,IAAAA,oCAAAA,mCAAAA,AAAAA,EACEH,wBACA7B,4BAEF3C;gBAEJ,MAAM4E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM3C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACmC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF3B,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;oBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC5D,OACA;wBACE,SAASuC;wBACTyB;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,kCAAAA,YAAYA,EAC/BrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRT,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC7B;gBAGF,MAAM5C,UACJwE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIvF,SAAS;oBACX,MAAMyF,uBAAuB1F,2BAA2BC;oBACxD,IAAIyF,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,kCAAAA,YAAYA,CAACC,sBAAsBxB;wBAE/C,MAAM,IAAIhC,MAAMwD;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B7F,iBAC/BmF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACE3F,WACA,IAAI,CAAC,SAAS,IACd,CAACsF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7CpF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIsF,gBAAkC5F,QAAQ,MAAM;oBACpD,IAAIqD,AAA6B,MAA7BA,0BAAgC;wBAClCuC,gBAAgB;4BACdC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;4BAC/BwC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;yBAChC;wBACD1D,MACE,8DACAK,QAAQ,MAAM,EACd4F;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOtF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,cAAcW;oBAChB;oBAEF,IAAIpB,iBAAiBiG,UAAU;wBAC7BnG,MACE,uCACAoF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACErF,MACE,yDACAoF;gBAGN,EAAE,OAAOxB,OAAO;oBACd5D,MAAM,mCAAmC4D;gBAC3C;qBAEA5D,MAAM;gBAIV,IAAI,CAACK,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIuB,kCAAAA,YAAYA,CACpB,CAAC,oBAAoB,EAAElF,MAAM,MAAM,EAAE,EACrC2D;oBAGJ,MAAM,IAAIhC,MAAM,CAAC,mBAAmB,EAAE3B,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIyF;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOzF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIgF,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF/B,WAAW5D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK6C,UAAU,aAAa;wBAC9B;oBACF;oBACAkD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IApkBA,YAAY,EACViC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTpE,WAAW,EACXqE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGpE;QACnB,IAAI,CAAC,eAAe,GAAGqE;IACzB;AAyjBF"}
1	+ {"version":3,"file":"agent/task-builder.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/task-builder.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { findActionInActionSpaceOrThrow } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n AIUsageInfo,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n PlanningLocateParam,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n ifPlanLocateParamHasLocatedPixelBbox,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nfunction invalidLocateElementReason(\n element: LocateResultElement,\n): string \| undefined {\n const values = [\n element.center?.[0],\n element.center?.[1],\n element.rect?.left,\n element.rect?.top,\n element.rect?.width,\n element.rect?.height,\n ];\n if (\n values.some((value) => typeof value !== 'number' \|\| !Number.isFinite(value))\n ) {\n return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n }\n if (element.rect.width <= 0 \|\| element.rect.height <= 0) {\n return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n }\n return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n param: string \| DetailedLocateParam,\n): DetailedLocateParam {\n if (typeof param === 'string') {\n return { prompt: param };\n }\n\n const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n const deepLocate = rest.deepLocate ?? deepThink;\n\n return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = normalizeLocateParam(param);\n const locatePlan: PlanningAction<PlanningLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n planningModel: ModelRuntime;\n defaultModel: ModelRuntime;\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n\n constructor({\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n }\n\n public async build(\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n planningModel,\n defaultModel,\n cacheable,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<PlanningLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<PlanningLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n const param = plan.param;\n\n const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n const requiredLocateFields = findAllMidsceneLocatorField(\n action.paramSchema,\n true,\n );\n\n locateFields.forEach((field) => {\n if (param[field]) {\n // Always use createLocateTask for all locate params.\n // This ensures cache writing happens even when locatedPixelBbox is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n executor: async (param, taskContext) => {\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug(\n `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.beforeInvokeAction(action.name, param);\n debug(\n `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n })(),\n delayBeforeRunner > 0\n ? sleep(delayBeforeRunner)\n : Promise.resolve(),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n\n if (action.paramSchema) {\n try {\n param = parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n }\n\n setTimingFieldOnce(timing, 'callActionStart');\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(param, taskContext);\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug(\n `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.afterInvokeAction(action.name, param);\n debug(\n `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<PlanningLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n let locateParam = normalizeLocateParam(detailedLocateParam);\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(\n param,\n )\n ? param\n : undefined;\n\n assert(\n param?.prompt \|\| paramWithLocatedPixelBbox,\n `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n param,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n searchAreaRawChoiceMessage:\n dump.taskInfo?.searchAreaRawChoiceMessage,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n task.searchArea = dump.taskInfo?.searchArea;\n if (dump.taskInfo?.searchAreaUsage) {\n task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n const planLocatedElement = paramWithLocatedPixelBbox\n ? matchElementFromPlan(paramWithLocatedPixelBbox)\n : undefined;\n\n // from locatedPixelBbox (direct plan hit)\n // when deepLocate is enabled, locatedPixelBbox should be used as search\n // area hint, not as a final direct hit\n const elementFromPlan = param.deepLocate\n ? undefined\n : planLocatedElement;\n const isPlanDirectHit = !!elementFromPlan;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanDirectHit &&\n param.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [param.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? generateElementByRect(\n // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n transformLogicalRectToScreenshotRect(\n rectFromXpath,\n shrunkShotToLogicalRatio,\n ),\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt \|\| '',\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = param.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanDirectHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n param.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n const elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isCacheHit = !!elementFromCache;\n\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n const timing = taskContext.task.timing;\n if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n param,\n {\n context: uiContext,\n planLocatedElement,\n },\n defaultModel,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n\n const element =\n elementFromPlan \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n if (element) {\n const invalidElementReason = invalidLocateElementReason(element);\n if (invalidElementReason) {\n if (locateDump) {\n throw new ServiceError(invalidElementReason, locateDump);\n }\n throw new Error(invalidElementReason);\n }\n }\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read)\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n !isCacheHit &&\n (!isPlanDirectHit \|\| !locateCacheAlreadyExists) &&\n param?.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n let pointForCache: [number, number] = element.center;\n if (shrunkShotToLogicalRatio !== 1) {\n pointForCache = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n debug(\n 'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n element.center,\n pointForCache,\n );\n }\n\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt,\n modelRuntime: defaultModel,\n },\n );\n if (hasNonEmptyCache(feature)) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${param.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${param.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n hitBy = {\n from: 'Plan',\n context: {\n locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: param.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n onResult?.(element);\n\n return {\n output: {\n element: {\n ...element,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","hasNonEmptyCache","cache","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","Error","shrunkShotToLogicalRatio","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC+BA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPP,OAAO,IAAI,CAACO,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASC,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BH,OAAO,WAAW;QAEnE,MAAMI,uBAAuBD,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAC3BH,OAAO,WAAW,EAClB;QAGFE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EAAE;gBAGhB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnDzC,MACE,uCACA,CAAC,YAAY,EAAEkC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,oBAAoB,EAAEwB,AAAAA,IAAAA,oCAAAA,oCAAAA,AAAAA,EAAqC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OAAO;gBACLE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEP,UAAU;gBAE3ElC,MAAM,CAAC,OAAO,EAAEyC,MAAM,6BAA6B,EAAEP,UAAU;YACjE;QACF;QAEA,MAAMY,OAKF;YACF,MAAM;YACN,SAASZ;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtC/C,MACE,oBACAkC,UACAvB,OACA,CAAC,4BAA4B,EAAEoC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,IAAAA,sBAAAA,MAAAA,AAAAA,EACElC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEP,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAgB,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBf,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMgB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCpD,MACE,CAAC,8DAA8D,EAAEoC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDX,MACE,CAAC,2DAA2D,EAAEoC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAe,oBAAoB,IAChBE,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,wCAAwC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,MAAM,EAAEU,wBAAwB,EAAE,GAAGT;gBACrC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIrB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQgD,AAAAA,IAAAA,yBAAAA,gBAAAA,AAAAA,EAAiBhD,OAAOyB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOE,OAAY;oBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEwB,MAAM,OAAO,CAAC,cAAc,EAAEnD,KAAK,SAAS,CAACE,QAAQ,EACtG;wBAAE,OAAOiD;oBAAM;gBAEnB;gBAGFV,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3BhD,MAAM,kBAAkBoC,OAAO,IAAI;gBACnC,MAAMyB,WAAWzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM0B,eAAe,MAAMD,SAASlD,OAAOoC;gBAC3CG,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC3BhD,MAAM,iBAAiBoC,OAAO,IAAI,EAAE,WAAW0B;gBAE/CZ,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,MAAMe,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI2B,mBAAmB,GACrB,MAAMV,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpC/D,MACE,CAAC,6DAA6D,EAAEoC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEzB;wBACpDX,MACE,CAAC,0DAA0D,EAAEoC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOkB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,uCAAuC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQc;gBACV;YACF;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACoB;IACrB;IAEQ,iBACNjB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMsB,4BAA4B3B,AAAAA,IAAAA,oCAAAA,oCAAAA,AAAAA,EAChC/B,SAEEA,QACAI;gBAEJ8B,IAAAA,sBAAAA,MAAAA,AAAAA,EACElC,OAAO,UAAU0D,2BACjB,CAAC,iEAAiE,EAAE5D,KAAK,SAAS,CAChFE,QACC;gBAGL,IAAI,CAACsC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOI,WAAW;gBAElB,MAAM,EAAES,wBAAwB,EAAE,GAAGT;gBAErC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIa;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb3B,KAAK,GAAG,GAAG;wBACT2B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA3B,KAAK,KAAK,GAAG4B,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD3B,KAAK,UAAU,GAAG2B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBACjB3B,KAAK,eAAe,GAAG2B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB3B,KAAK,iBAAiB,GAAG2B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,AAAAA,IAAAA,oCAAAA,oBAAAA,AAAAA,EAAqBP,6BACrBtD;gBAKJ,MAAM8D,kBAAkBlE,MAAM,UAAU,GACpCI,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDnE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFoE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACpE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMqE,mBAAmBD,gBACrBE,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EAEEC,AAAAA,IAAAA,oCAAAA,oCAAAA,AAAAA,EACEH,eACArB,2BAEF,AAAwB,YAAxB,OAAO/C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BI;gBAEJ,MAAMoE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAczE,MAAM,MAAM;gBAChC,MAAM0E,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,AAAAA,IAAAA,oCAAAA,qBAAAA,AAAAA,EACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAzE,MAAM,SAAS;gBAIvB,MAAM8E,mBAAmBF,yBACrBG,AAAAA,IAAAA,oCAAAA,mCAAAA,AAAAA,EACEH,wBACA7B,4BAEF3C;gBAEJ,MAAM4E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM5C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF5B,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;oBAC3BuB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC5D,OACA;wBACE,SAASsC;wBACT0B;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,kCAAAA,YAAYA,EAC/BrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRV,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJwE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIvF,SAAS;oBACX,MAAMyF,uBAAuB1F,2BAA2BC;oBACxD,IAAIyF,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,kCAAAA,YAAYA,CAACC,sBAAsBxB;wBAE/C,MAAM,IAAIb,MAAMqC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B7F,iBAC/BmF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACE3F,WACA,IAAI,CAAC,SAAS,IACd,CAACsF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7CpF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIsF,gBAAkC5F,QAAQ,MAAM;oBACpD,IAAIqD,AAA6B,MAA7BA,0BAAgC;wBAClCuC,gBAAgB;4BACdC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;4BAC/BwC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;yBAChC;wBACD1D,MACE,8DACAK,QAAQ,MAAM,EACd4F;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOtF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,cAAcW;oBAChB;oBAEF,IAAIpB,iBAAiBiG,UAAU;wBAC7BnG,MACE,uCACAoF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACErF,MACE,yDACAoF;gBAGN,EAAE,OAAOxB,OAAO;oBACd5D,MAAM,mCAAmC4D;gBAC3C;qBAEA5D,MAAM;gBAIV,IAAI,CAACK,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIuB,kCAAAA,YAAYA,CACpB,CAAC,oBAAoB,EAAElF,MAAM,MAAM,EAAE,EACrC2D;oBAGJ,MAAM,IAAIb,MAAM,CAAC,mBAAmB,EAAE9C,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIyF;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOzF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIgF,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF/B,WAAW5D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK4C,UAAU,aAAa;wBAC9B;oBACF;oBACAmD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IA/jBA,YAAY,EACViC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTpE,WAAW,EACXqE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGpE;QACnB,IAAI,CAAC,eAAe,GAAGqE;IACzB;AAojBF"}

package/dist/lib/agent/tasks.js CHANGED Viewed

@@ -164,10 +164,6 @@ class TaskExecutor {
         }
     }
     async runAction(userPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal, reportOptions) {
-        if (deepLocate && !planningModel.adapter.planning.supportsActionDeepLocate) {
-            warnLog(`The "deepLocate" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`);
-            deepLocate = false;
-        }
         const conversationHistory = new index_js_namespaceObject.ConversationHistory();
         const session = this.createExecutionSession((0, external_ui_utils_js_namespaceObject.taskTitleStr)(reportOptions?.type || 'Act', reportOptions?.prompt || (0, external_common_js_namespaceObject.userPromptToString)(userPrompt)));
         const runner = session.getRunner();
@@ -318,7 +314,7 @@ class TaskExecutor {
             runner
         };
     }
-    createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt) {
+    createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt, executionOptions) {
         const queryTask = {
             type: 'Insight',
             subType: type,
@@ -367,7 +363,7 @@ class TaskExecutor {
                     extraPageDescription = await (0, extractor_namespaceObject.descriptionOfTree)(tree, 200, false, opt?.domIncluded === 'visible-only');
                 }
                 try {
-                    extractResult = await this.service.extract(demandInput, modelRuntime, opt, extraPageDescription, multimodalPrompt, uiContext);
+                    extractResult = await this.service.extract(demandInput, modelRuntime, opt, extraPageDescription, multimodalPrompt, uiContext, executionOptions);
                 } catch (error) {
                     if (error instanceof external_types_js_namespaceObject.ServiceError) applyDump(error.dump);
                     throw error;
@@ -394,9 +390,9 @@ class TaskExecutor {
         };
         return queryTask;
     }
-    async createTypeQueryExecution(type, demand, modelRuntime, opt, multimodalPrompt) {
+    async createTypeQueryExecution(type, demand, modelRuntime, opt, multimodalPrompt, executionOptions) {
         const session = this.createExecutionSession((0, external_ui_utils_js_namespaceObject.taskTitleStr)(type, 'string' == typeof demand ? demand : JSON.stringify(demand)));
-        const queryTask = await this.createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt);
+        const queryTask = await this.createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt, executionOptions);
         const runner = session.getRunner();
         const result = await session.appendAndRun(queryTask);
         if (!result) throw new Error('result of taskExecutor.flush() is undefined in function createTypeQueryTask');

package/dist/lib/agent/tasks.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent/tasks.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/tasks.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n type TMultimodalPrompt,\n type TUserPrompt,\n getReadableTimeString,\n multimodalPromptToChatMessages,\n userPromptToMultimodalPrompt,\n userPromptToString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n DeviceAction,\n ExecutionTask,\n ExecutionTaskApply,\n ExecutionTaskInsightQueryApply,\n ExecutionTaskPlanningApply,\n ExecutionTaskProgressOptions,\n MidsceneYamlFlowItem,\n PlanningAIResponse,\n PlanningAction,\n PlanningActionParamWaitFor,\n PlanningLocateParam,\n ServiceDump,\n ServiceExtractOption,\n ServiceExtractParam,\n UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n output: OutputType;\n thought?: string;\n runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n onTaskUpdate?: (\n runner: TaskRunner,\n error?: TaskExecutionError,\n ) => Promise<void> \| void;\n}\n\nexport type ActionReportOptions = {\n type?: TaskTitleType;\n prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap each task's planning feedback so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. This is the\n// single place that truncates feedback before it is sent to the model; action\n// implementations should hand over the untruncated value.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(feedback: string): string {\n if (feedback.length <= maxPlanningFeedbackLength) {\n return feedback;\n }\n\n return `${feedback.slice(0, maxPlanningFeedbackLength)}\n...[truncated, ${feedback.length - maxPlanningFeedbackLength} more characters]`;\n}\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n interface: AbstractInterface;\n\n service: Service;\n\n taskCache?: TaskCache;\n\n private readonly providedActionSpace: DeviceAction[];\n\n private readonly taskBuilder: TaskBuilder;\n\n onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n private readonly hooks?: TaskExecutorHooks;\n\n replanningCycleLimit?: number;\n\n waitAfterAction?: number;\n\n useDeviceTime?: boolean;\n\n // @deprecated use .interface instead\n get page() {\n return this.interface;\n }\n\n constructor(\n interfaceInstance: AbstractInterface,\n service: Service,\n opts: {\n taskCache?: TaskCache;\n onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n replanningCycleLimit?: number;\n waitAfterAction?: number;\n useDeviceTime?: boolean;\n hooks?: TaskExecutorHooks;\n actionSpace: DeviceAction[];\n },\n ) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = opts.taskCache;\n this.onTaskStartCallback = opts?.onTaskStart;\n this.replanningCycleLimit = opts.replanningCycleLimit;\n this.waitAfterAction = opts.waitAfterAction;\n this.useDeviceTime = opts.useDeviceTime;\n this.hooks = opts.hooks;\n this.providedActionSpace = opts.actionSpace;\n this.taskBuilder = new TaskBuilder({\n interfaceInstance,\n service,\n taskCache: opts.taskCache,\n actionSpace: this.getActionSpace(),\n waitAfterAction: opts.waitAfterAction,\n });\n }\n\n private createExecutionSession(\n title: string,\n options?: { tasks?: ExecutionTaskApply[]; uiContext?: UIContext },\n ) {\n return new ExecutionSession(\n title,\n () =>\n options?.uiContext\n ? Promise.resolve(options.uiContext)\n : Promise.resolve(this.service.contextRetrieverFn()),\n {\n onTaskStart: this.onTaskStartCallback,\n tasks: options?.tasks,\n onTaskUpdate: this.hooks?.onTaskUpdate,\n },\n );\n }\n\n private getActionSpace(): DeviceAction[] {\n return this.providedActionSpace;\n }\n\n /*\n Set the pending feedback message consumed by the next planning round.\n * The message is always prefixed with the current time. When a body is\n * provided it is appended after the timestamp; otherwise only the time\n * context is recorded. This is the single entry point for writing\n * `pendingFeedbackMessage` so the time prefix stays consistent.\n /\n private setPendingFeedbackMessage(\n conversationHistory: ConversationHistory,\n timeString: string,\n body?: string,\n ) {\n conversationHistory.pendingFeedbackMessage = body\n ? `Time: ${timeString}, ${body}`\n : `Current time: ${timeString}`;\n }\n\n /\n Collect feedback produced by executed tasks for the next planning round.\n * Returns undefined when no task reported feedback.\n /\n private collectPlanningFeedback(tasks: ExecutionTask[]): string \| undefined {\n const feedbackMessages = tasks.flatMap(({ planningFeedback }) =>\n planningFeedback ? [truncatePlanningFeedback(planningFeedback)] : [],\n );\n return feedbackMessages.length > 0\n ? feedbackMessages.join('\\n\\n')\n : undefined;\n }\n\n /\n Get a readable time string. When device time is enabled, use the\n * device-formatted wall-clock time directly so host timezone formatting does\n * not reinterpret a device timestamp.\n * @param format - Optional format string\n * @returns A formatted time string\n /\n private async getTimeString(format?: string): Promise<string> {\n if (this.useDeviceTime) {\n if (this.interface.getDeviceLocalTimeString) {\n try {\n return await this.interface.getDeviceLocalTimeString(format);\n } catch (error) {\n warnLog(\n `Failed to get device time string, falling back to runtime time: ${error}`,\n );\n }\n } else {\n warnLog(\n 'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n );\n }\n }\n\n return getReadableTimeString(format);\n }\n\n public async convertPlanToExecutable(\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n options?: {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n },\n ) {\n return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n }\n\n async loadYamlFlowAsPlanning(\n userInstruction: TUserPrompt,\n yamlString: string,\n reportOptions?: ActionReportOptions,\n ) {\n const session = this.createExecutionSession(\n taskTitleStr(\n reportOptions?.type \|\| 'Act',\n reportOptions?.prompt \|\| userPromptToString(userInstruction),\n ),\n );\n\n const task: ExecutionTaskPlanningApply = {\n type: 'Planning',\n subType: 'LoadYaml',\n param: {\n userInstruction,\n ...(reportOptions?.prompt\n ? { userInstructionDisplay: reportOptions.prompt }\n : {}),\n },\n executor: async (param, executorContext) => {\n const { uiContext } = executorContext;\n assert(uiContext, 'uiContext is required for Planning task');\n return {\n output: {\n actions: [],\n shouldContinuePlanning: false,\n log: '',\n yamlString,\n },\n cache: {\n hit: true,\n },\n hitBy: {\n from: 'Cache',\n context: {\n yamlString,\n },\n },\n };\n },\n };\n const runner = session.getRunner();\n await session.appendAndRun(task);\n\n return {\n runner,\n };\n }\n\n async runPlans(\n title: string,\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n options?: { uiContext?: UIContext },\n ): Promise<ExecutionResult> {\n const session = this.createExecutionSession(title, options);\n const { tasks } = await this.convertPlanToExecutable(\n plans,\n planningModel,\n defaultModel,\n );\n const runner = session.getRunner();\n const result = await session.appendAndRun(tasks);\n const { output } = result ?? {};\n return {\n output,\n runner,\n };\n }\n\n async action(\n userPrompt: TUserPrompt,\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n includeLocateInPlanning: boolean,\n aiActContext?: string,\n cacheable?: boolean,\n replanningCycleLimitOverride?: number,\n imagesIncludeCount?: number,\n deepThink?: boolean,\n fileChooserAccept?: string[],\n deepLocate?: boolean,\n abortSignal?: AbortSignal,\n reportOptions?: ActionReportOptions,\n ): Promise<\n ExecutionResult<\n \| {\n yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n output?: string;\n }\n \| undefined\n >\n > {\n return withFileChooser(this.interface, fileChooserAccept, async () => {\n return this.runAction(\n userPrompt,\n planningModel,\n defaultModel,\n includeLocateInPlanning,\n aiActContext,\n cacheable,\n replanningCycleLimitOverride,\n imagesIncludeCount,\n deepThink,\n deepLocate,\n abortSignal,\n reportOptions,\n );\n });\n }\n\n /\n Called when the task is about to replan. Marks every cache-hit locate task\n * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n * did not finish the task, so the element each cache hit produced is suspect.\n * The upcoming re-locate of the same prompt then replaces the bad entry in\n * place instead of appending a duplicate that would re-poison the cache on the\n * next run (#2529).\n \n Marking a locate that was actually fine is harmless: the step is only ever\n * replaced if the same prompt is located again (i.e. the step is redone),\n * which does not happen for a locate that already succeeded.\n */\n private invalidateFailedCacheHitLocates(\n runner: TaskRunner,\n fromIndex: number,\n ) {\n if (!this.taskCache) {\n return;\n }\n for (let i = fromIndex; i < runner.tasks.length; i++) {\n const task = runner.tasks[i];\n if (\n task.type === 'Planning' &&\n task.subType === 'Locate' &&\n task.hitBy?.from === 'Cache'\n ) {\n const prompt = (task.param as PlanningLocateParam \| undefined)?.prompt;\n if (prompt) {\n this.taskCache.markLocateCacheStale(prompt);\n }\n }\n }\n }\n\n private async runAction(\n userPrompt: TUserPrompt,\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n includeLocateInPlanning: boolean,\n aiActContext?: string,\n cacheable?: boolean,\n replanningCycleLimitOverride?: number,\n imagesIncludeCount?: number,\n deepThink?: boolean,\n deepLocate?: boolean,\n abortSignal?: AbortSignal,\n reportOptions?: ActionReportOptions,\n ): Promise<\n ExecutionResult<\n \| {\n yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n output?: string;\n }\n \| undefined\n >\n > {\n if (\n deepLocate &&\n !planningModel.adapter.planning.supportsActionDeepLocate\n ) {\n warnLog(\n `The \"deepLocate\" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`,\n );\n deepLocate = false;\n }\n\n const conversationHistory = new ConversationHistory();\n\n const session = this.createExecutionSession(\n taskTitleStr(\n reportOptions?.type \|\| 'Act',\n reportOptions?.prompt \|\| userPromptToString(userPrompt),\n ),\n );\n const runner = session.getRunner();\n\n let replanCount = 0;\n const yamlFlow: MidsceneYamlFlowItem[] = [];\n const replanningCycleLimit =\n replanningCycleLimitOverride ?? this.replanningCycleLimit;\n assert(\n replanningCycleLimit !== undefined,\n 'replanningCycleLimit is required for TaskExecutor.action',\n );\n\n let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n let outputString: string \| undefined;\n\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n const referenceImageMessages = await multimodalPromptToChatMessages(\n userPromptToMultimodalPrompt(userPrompt),\n );\n\n // Main planning loop - unified plan/replan logic\n while (true) {\n // Check abort signal before each planning cycle\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n\n // Get sub-goal status text if available\n const subGoalStatus = conversationHistory.subGoalsToText() \|\| undefined;\n\n // Get memories text if available\n const memoriesStatus = conversationHistory.memoriesToText() \|\| undefined;\n\n const result = await session.appendAndRun(\n {\n type: 'Planning',\n subType: 'Plan',\n param: {\n userInstruction: userPrompt,\n ...(reportOptions?.prompt\n ? { userInstructionDisplay: reportOptions.prompt }\n : {}),\n aiActContext,\n imagesIncludeCount,\n deepThink,\n ...(subGoalStatus ? { subGoalStatus } : {}),\n ...(memoriesStatus ? { memoriesStatus } : {}),\n },\n executor: async (param, executorContext) => {\n const { uiContext } = executorContext;\n assert(uiContext, 'uiContext is required for Planning task');\n const timing = executorContext.task.timing;\n\n const actionSpace = this.getActionSpace();\n debug(\n 'actionSpace for this interface is:',\n actionSpace.map((action) => action.name).join(', '),\n );\n assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n if (actionSpace.length === 0) {\n console.warn(\n `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n );\n }\n\n const planImpl =\n planningModel.adapter.planning.kind === 'custom'\n ? planningModel.adapter.planning.planFn\n : genericXmlPlan;\n\n let planResult: Awaited<ReturnType<typeof planImpl>>;\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n planResult = await planImpl(param.userInstruction, {\n context: uiContext,\n actionContext: param.aiActContext,\n actionSpace,\n modelRuntime: planningModel,\n conversationHistory,\n includeLocateInPlanning,\n imagesIncludeCount,\n deepThink,\n referenceImageMessages,\n abortSignal,\n });\n } catch (planError) {\n if (planError instanceof AIResponseParseError) {\n // Record usage and rawResponse even when parsing fails\n executorContext.task.usage = withUsageIntent(\n planError.usage,\n 'planning',\n );\n executorContext.task.log = {\n ...(executorContext.task.log \|\| {}),\n rawResponse: planError.rawResponse,\n rawChoiceMessage: planError.rawChoiceMessage,\n };\n }\n throw planError;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n debug('planResult', JSON.stringify(planResult, null, 2));\n\n const {\n actions,\n thought,\n log,\n memory,\n error,\n usage,\n rawResponse,\n rawChoiceMessage,\n reasoning_content,\n finalizeSuccess,\n finalizeMessage,\n updateSubGoals,\n markFinishedIndexes,\n } = planResult;\n outputString = finalizeMessage;\n\n executorContext.task.log = {\n ...(executorContext.task.log \|\| {}),\n rawResponse,\n rawChoiceMessage,\n };\n executorContext.task.usage = withUsageIntent(usage, 'planning');\n executorContext.task.reasoning_content = reasoning_content;\n executorContext.task.output = {\n actions: actions \|\| [],\n log,\n thought,\n memory,\n yamlFlow: planResult.yamlFlow,\n output: finalizeMessage,\n shouldContinuePlanning: planResult.shouldContinuePlanning,\n updateSubGoals,\n markFinishedIndexes,\n };\n executorContext.uiContext = uiContext;\n\n assert(!error, `Failed to continue: ${error}\\n${log \|\| ''}`);\n\n // Check if task was finalized with failure\n if (finalizeSuccess === false) {\n assert(\n false,\n `Task failed: ${finalizeMessage \|\| 'No error message provided'}\\n${log \|\| ''}`,\n );\n }\n\n return {\n cache: {\n hit: false,\n },\n } as any;\n },\n },\n {\n allowWhenError: true,\n },\n );\n\n const planResult = result?.output as PlanningAIResponse \| undefined;\n\n // Execute planned actions\n const plans = planResult?.actions \|\| [];\n yamlFlow.push(...(planResult?.yamlFlow \|\| []));\n\n let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n try {\n executables = await this.convertPlanToExecutable(\n plans,\n planningModel,\n defaultModel,\n {\n cacheable,\n deepLocate,\n abortSignal,\n },\n );\n } catch (error) {\n return session.appendErrorPlan(\n `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n plans,\n )}`,\n );\n }\n if (conversationHistory.pendingFeedbackMessage) {\n console.warn(\n 'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n conversationHistory.pendingFeedbackMessage,\n );\n }\n\n // Capture the time context for the next planning call before running.\n const initialTimeString = await this.getTimeString();\n\n const taskCountBeforeRun = runner.tasks.length;\n try {\n await session.appendAndRun(executables.tasks);\n this.setPendingFeedbackMessage(\n conversationHistory,\n initialTimeString,\n this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n );\n } catch (error: any) {\n // errorFlag = true;\n errorCountInOnePlanningLoop++;\n const timeString = await this.getTimeString();\n this.setPendingFeedbackMessage(\n conversationHistory,\n timeString,\n `Error executing running tasks: ${error?.message \|\| String(error)}`,\n );\n debug(\n 'error when executing running tasks, but continue to run if it is not too many errors:',\n error instanceof Error ? error.message : String(error),\n 'current error count in one planning loop:',\n errorCountInOnePlanningLoop,\n );\n }\n\n if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n return session.appendErrorPlan('Too many errors in one planning loop');\n }\n\n // Check abort signal after executing actions\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n\n // // Check if task is complete\n if (!planResult?.shouldContinuePlanning) {\n break;\n }\n\n // We are about to replan, which means the batch we just ran did not finish\n // the task. Any locate task in that batch that was served from cache\n // produced an element that failed to complete the step (the action threw,\n // or it clicked the wrong element and the goal was not reached). Mark those\n // cache entries stale so the re-locate of the same prompt replaces them in\n // place instead of appending a poisoning duplicate that would be matched\n // first on the next run (#2529).\n this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n // Increment replan count for next iteration\n ++replanCount;\n\n if (replanCount > replanningCycleLimit) {\n const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n return session.appendErrorPlan(errorMsg);\n }\n\n if (!conversationHistory.pendingFeedbackMessage) {\n const timeString = await this.getTimeString();\n conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n }\n }\n\n return {\n output: {\n yamlFlow,\n output: outputString,\n },\n runner,\n };\n }\n\n private createTypeQueryTask(\n type: 'Query' \| 'Boolean' \| 'Number' \| 'String' \| 'Assert' \| 'WaitFor',\n demand: ServiceExtractParam,\n modelRuntime: ModelRuntime,\n opt?: ServiceExtractOption,\n multimodalPrompt?: TMultimodalPrompt,\n ) {\n const queryTask: ExecutionTaskInsightQueryApply = {\n type: 'Insight',\n subType: type,\n param: {\n domIncluded: opt?.domIncluded,\n dataDemand: multimodalPrompt\n ? ({\n demand,\n multimodalPrompt,\n } as never)\n : demand, // for user param presentation in report right sidebar\n },\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let queryDump: ServiceDump \| undefined;\n const applyDump = (dump: ServiceDump) => {\n queryDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n searchAreaRawChoiceMessage:\n dump.taskInfo?.searchAreaRawChoiceMessage,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n // Get context for query operations\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Query task');\n\n const ifTypeRestricted = type !== 'Query';\n let demandInput = demand;\n let keyOfResult = 'result';\n if (ifTypeRestricted && (type === 'Assert' \|\| type === 'WaitFor')) {\n keyOfResult = 'StatementIsTruthy';\n demandInput = {\n [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n };\n } else if (ifTypeRestricted) {\n keyOfResult = type;\n demandInput = {\n [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n };\n }\n\n let extractResult;\n\n let extraPageDescription = '';\n if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n debug('appending tree info for page');\n const tree = await this.interface.getElementsNodeTree();\n extraPageDescription = await descriptionOfTree(\n tree,\n 200,\n false,\n opt?.domIncluded === 'visible-only',\n );\n }\n\n try {\n extractResult = await this.service.extract<any>(\n demandInput,\n modelRuntime,\n opt,\n extraPageDescription,\n multimodalPrompt,\n uiContext,\n );\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n }\n\n const { data, thought, dump } = extractResult;\n applyDump(dump);\n\n let outputResult = data;\n if (ifTypeRestricted) {\n // If AI returned a plain string instead of structured format, use it directly\n if (typeof data === 'string') {\n outputResult = data;\n } else if (type === 'WaitFor') {\n if (data === null \|\| data === undefined) {\n outputResult = false;\n } else {\n outputResult = (data as any)[keyOfResult];\n }\n } else if (data === null \|\| data === undefined) {\n outputResult = null;\n } else {\n // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n if (data?.[keyOfResult] !== undefined) {\n outputResult = (data as any)[keyOfResult];\n } else if (data?.result !== undefined) {\n outputResult = (data as any).result;\n } else {\n assert(false, 'No result in query data');\n }\n }\n }\n\n if (type === 'Assert' && !outputResult) {\n task.thought = thought;\n throw new Error(`Assertion failed: ${thought}`);\n }\n\n return {\n output: outputResult,\n log: queryDump,\n thought,\n };\n },\n };\n\n return queryTask;\n }\n async createTypeQueryExecution<T>(\n type: 'Query' \| 'Boolean' \| 'Number' \| 'String' \| 'Assert',\n demand: ServiceExtractParam,\n modelRuntime: ModelRuntime,\n opt?: ServiceExtractOption,\n multimodalPrompt?: TMultimodalPrompt,\n ): Promise<ExecutionResult<T>> {\n const session = this.createExecutionSession(\n taskTitleStr(\n type,\n typeof demand === 'string' ? demand : JSON.stringify(demand),\n ),\n );\n\n const queryTask = await this.createTypeQueryTask(\n type,\n demand,\n modelRuntime,\n opt,\n multimodalPrompt,\n );\n\n const runner = session.getRunner();\n const result = await session.appendAndRun(queryTask);\n\n if (!result) {\n throw new Error(\n 'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n );\n }\n\n const { output, thought } = result;\n\n return {\n output,\n thought,\n runner,\n };\n }\n\n async waitFor(\n assertion: TUserPrompt,\n opt: PlanningActionParamWaitFor,\n modelRuntime: ModelRuntime,\n ): Promise<ExecutionResult<void>> {\n const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n const description = `waitFor: ${textPrompt}`;\n const session = this.createExecutionSession(\n taskTitleStr('WaitFor', description),\n );\n const runner = session.getRunner();\n const {\n timeoutMs,\n checkIntervalMs,\n domIncluded,\n screenshotIncluded,\n ...restOpt\n } = opt;\n const serviceExtractOpt: ServiceExtractOption = {\n domIncluded,\n screenshotIncluded,\n ...restOpt,\n };\n\n assert(assertion, 'No assertion for waitFor');\n assert(timeoutMs, 'No timeoutMs for waitFor');\n assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n assert(\n checkIntervalMs <= timeoutMs,\n `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n );\n\n const overallStartTime = Date.now();\n let lastCheckStart = overallStartTime;\n let errorThought = '';\n // Continue checking as long as the previous iteration began within the timeout window.\n while (lastCheckStart - overallStartTime <= timeoutMs) {\n const currentCheckStart = Date.now();\n lastCheckStart = currentCheckStart;\n const queryTask = await this.createTypeQueryTask(\n 'WaitFor',\n textPrompt,\n modelRuntime,\n serviceExtractOpt,\n multimodalPrompt,\n );\n\n const result = (await session.appendAndRun(queryTask)) as\n \| {\n output: boolean;\n thought?: string;\n }\n \| undefined;\n\n if (result?.output) {\n return {\n output: undefined,\n runner,\n };\n }\n\n errorThought =\n result?.thought \|\|\n (!result && `No result from assertion: ${textPrompt}`) \|\|\n `unknown error when waiting for assertion: ${textPrompt}`;\n const now = Date.now();\n if (now - currentCheckStart < checkIntervalMs) {\n const elapsed = now - currentCheckStart;\n const timeRemaining = checkIntervalMs - elapsed;\n const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n modelRuntime,\n modelRuntime,\n );\n if (sleepTasks[0]) {\n await session.appendAndRun(sleepTasks[0]);\n }\n }\n }\n\n return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n }\n}\n\nexport async function withFileChooser<T>(\n interfaceInstance: AbstractInterface,\n fileChooserAccept: string[] \| undefined,\n action: () => Promise<T>,\n): Promise<T> {\n if (!fileChooserAccept?.length) {\n return action();\n }\n\n if (!interfaceInstance.registerFileChooserListener) {\n throw new Error(\n `File upload is not supported on ${interfaceInstance.interfaceType}`,\n );\n }\n\n const handler = async (chooser: FileChooserHandler) => {\n await chooser.accept(fileChooserAccept);\n };\n\n const { dispose, getError } =\n await interfaceInstance.registerFileChooserListener(handler);\n try {\n const result = await action();\n // Check for errors that occurred during file chooser handling\n const error = await getError();\n if (error) {\n throw error;\n }\n return result;\n } finally {\n dispose();\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","TaskExecutor","title","options","ExecutionSession","Promise","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","undefined","format","error","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","task","param","executorContext","uiContext","assert","runner","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","fromIndex","i","prompt","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","errorCountInOnePlanningLoop","outputString","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","subGoalStatus","memoriesStatus","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","rawChoiceMessage","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","taskCountBeforeRun","String","Error","errorMsg","type","demand","modelRuntime","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC0DA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAM9C,MAAMC,4BAA4B;AAElC,SAASC,yBAAyBC,QAAgB;IAChD,IAAIA,SAAS,MAAM,IAAIF,2BACrB,OAAOE;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGF,2BAA2B;eAC1C,EAAEE,SAAS,MAAM,GAAGF,0BAA0B,iBAAiB,CAAC;AAC/E;AAIO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAAiE,EACjE;QACA,OAAO,IAAIC,8CAAAA,gBAAgBA,CACzBF,OACA,IACEC,SAAS,YACLE,QAAQ,OAAO,CAACF,QAAQ,SAAS,IACjCE,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASQ,0BACNG,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CAAC,CAAC,EAAEE,gBAAgB,EAAE,GAC1DA,mBAAmB;gBAACZ,yBAAyBY;aAAkB,GAAG,EAAE;QAEtE,OAAOD,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBE;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdlB,QACE,CAAC,gEAAgE,EAAEkB,OAAO;QAE9E;aAEAlB,QACE;QAKN,OAAOmB,AAAAA,IAAAA,mCAAAA,qBAAAA,AAAAA,EAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACa,OAAOC,eAAeC,cAAcf;IACpE;IAEA,MAAM,uBACJgB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBL;QAIhD,MAAMM,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLN;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMU,SAASR,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACG;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJ5B,KAAa,EACbc,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAAmC,EACT;QAC1B,MAAMmB,UAAU,IAAI,CAAC,sBAAsB,CAACpB,OAAOC;QACnD,MAAM,EAAEM,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDO,OACAC,eACAC;QAEF,MAAMY,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACb;QAC1C,MAAM,EAAEuB,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAF;QACF;IACF;IAEA,MAAM,OACJG,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,OAAOsB,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAhB,eACAC,cACAgB,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC,aACArB;IAGN;IAcQ,gCACNS,MAAkB,EAClBc,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAIf,OAAO,KAAK,CAAC,MAAM,EAAEe,IAAK;YACpD,MAAMpB,OAAOK,OAAO,KAAK,CAACe,EAAE;YAC5B,IACEpB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMqB,SAAUrB,KAAK,KAAK,EAAsC;gBAChE,IAAIqB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZb,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,IACEoB,cACA,CAACxB,cAAc,OAAO,CAAC,QAAQ,CAAC,wBAAwB,EACxD;YACArB,QACE,CAAC,mGAAmG,EAAEqB,cAAc,MAAM,CAAC,WAAW,IAAI,UAAU,sBAAsB,CAAC;YAE7KwB,aAAa;QACf;QAEA,MAAMnC,sBAAsB,IAAIyC,yBAAAA,mBAAmBA;QAEnD,MAAMzB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBS;QAGhD,MAAMH,SAASR,QAAQ,SAAS;QAEhC,IAAI0B,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJb,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DR,IAAAA,sBAAAA,MAAAA,AAAAA,EACEqB,AAAyBtC,WAAzBsC,sBACA;QAGF,IAAIC,8BAA8B;QAClC,IAAIC;QAEJ,IAAIV,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMW,yBAAyB,MAAMC,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EACnCC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6BtB;QAI/B,MAAO,KAAM;YAEX,IAAIS,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMc,gBAAgBlD,oBAAoB,cAAc,MAAMM;YAG9D,MAAM6C,iBAAiBnD,oBAAoB,cAAc,MAAMM;YAE/D,MAAMmB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjB,GAAIZ,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACNc;oBACAG;oBACAC;oBACA,GAAIiB,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO/B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;oBAClB,MAAM8B,SAAS/B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAMgC,cAAc,IAAI,CAAC,cAAc;oBACvCjE,MACE,sCACAiE,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD/B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOgC,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJ9C,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrC+C,kCAAAA,cAAcA;oBAEpB,IAAIC;oBACJ,IAAI;wBACFC,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAASrC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjCiC;4BACA,cAAc1C;4BACdX;4BACA4B;4BACAI;4BACAC;4BACAc;4BACAX;wBACF;oBACF,EAAE,OAAOyB,WAAW;wBAClB,IAAIA,qBAAqBC,yBAAAA,oBAAoBA,EAAE;4BAE7CzC,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAC3BF,UAAU,KAAK,EACf;4BAEFxC,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAawC,UAAU,WAAW;gCAClC,kBAAkBA,UAAU,gBAAgB;4BAC9C;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBR,QAAQ;oBAC7B;oBACAhE,MAAM,cAAc4E,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN5D,KAAK,EACL6D,KAAK,EACLC,WAAW,EACXC,gBAAgB,EAChBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGjB;oBACJb,eAAe4B;oBAEfrD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCiD;wBACAC;oBACF;oBACAlD,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgBM,OAAO;oBACpDhD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGmD;oBACzCnD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS4C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQe;wBACR,wBAAwBf,WAAW,sBAAsB;wBACzDgB;wBACAC;oBACF;oBACAvD,gBAAgB,SAAS,GAAGC;oBAE5BC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACf,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAE2D,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBACFlD,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,OACA,CAAC,aAAa,EAAEmD,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAalC,QAAQ;YAG3B,MAAMf,QAAQiD,YAAY,WAAW,EAAE;YACvChB,SAAS,IAAI,IAAKgB,YAAY,YAAY,EAAE;YAE5C,IAAIkB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CnE,OACAC,eACAC,cACA;oBACEkB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO5B,OAAO;gBACd,OAAOQ,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAER,MAAM,SAAS,EAAEwD,KAAK,SAAS,CAC5EtD,QACC;YAEP;YACA,IAAIV,oBAAoB,sBAAsB,EAC5CwD,QAAQ,IAAI,CACV,8FACAxD,oBAAoB,sBAAsB;YAK9C,MAAM8E,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqBvD,OAAO,KAAK,CAAC,MAAM;YAC9C,IAAI;gBACF,MAAMR,QAAQ,YAAY,CAAC6D,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5B7E,qBACA8E,mBACA,IAAI,CAAC,uBAAuB,CAACtD,OAAO,KAAK,CAAC,KAAK,CAACuD;YAEpD,EAAE,OAAOvE,OAAY;gBAEnBqC;gBACA,MAAM5C,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEO,OAAO,WAAWwE,OAAOxE,QAAQ;gBAErEpB,MACE,yFACAoB,iBAAiByE,QAAQzE,MAAM,OAAO,GAAGwE,OAAOxE,QAChD,6CACAqC;YAEJ;YAEA,IAAIA,8BAA8BtD,uCAChC,OAAOyB,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACuB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAACnC,QAAQuD;YAG7C,EAAErC;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMsC,WAAW,CAAC,UAAU,EAAEtC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO5B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAClF,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACN0C;gBACA,QAAQG;YACV;YACAtB;QACF;IACF;IAEQ,oBACN2D,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOhE,OAAOqE;gBACtB,MAAM,EAAEtE,IAAI,EAAE,GAAGsE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZzE,KAAK,GAAG,GAAG;wBACTyE;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACAzE,KAAK,KAAK,GAAG4C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgB6B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjBzE,KAAK,iBAAiB,GAAGyE,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMtE,YAAYmE,YAAY,SAAS;gBACvClE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElB,MAAMuE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,AAAAA,IAAAA,8BAAAA,yBAAAA,AAAAA,EAA0Bb,MAAMC;oBACjD;gBACF,OAAO,IAAIS,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAEC,AAAAA,IAAAA,8BAAAA,yBAAAA,AAAAA,EAA0Bb,MAAMC;oBACjD;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DlG,MAAM;oBACN,MAAM+G,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,AAAAA,IAAAA,0BAAAA,iBAAAA,AAAAA,EAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,cACAC,KACAY,sBACAX,kBACAjE;gBAEJ,EAAE,OAAOd,OAAO;oBACd,IAAIA,iBAAiB6F,kCAAAA,YAAYA,EAC/BV,UAAUnF,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE8F,IAAI,EAAEpC,OAAO,EAAE0B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAKzF,QAC1BiG,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAWhG,QAC1BiG,eAAgBD,KAAa,MAAM;qBAEnC/E,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,OAAO;gBAKpB,IAAI4D,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCpF,KAAK,OAAO,GAAG+C;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQqC;oBACR,KAAKb;oBACLxB;gBACF;YACF;QACF;QAEA,OAAOsB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,cACAC,KACAC;QAGF,MAAM/D,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACwE;QAE1C,IAAI,CAAC/D,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEwC,OAAO,EAAE,GAAGzC;QAE5B,OAAO;YACLC;YACAwC;YACA1C;QACF;IACF;IAEA,MAAM,QACJgF,SAAsB,EACtBlB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,AAAAA,IAAAA,kCAAAA,WAAAA,AAAAA,EAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMzF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,WAAW0F;QAE1B,MAAMnF,SAASR,QAAQ,SAAS;QAChC,MAAM,EACJ4F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEAzF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOiF,WAAW;QAClBjF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqF,WAAW;QAClBrF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsF,iBAAiB;QAExBtF,IAAAA,sBAAAA,MAAAA,AAAAA,EACEsF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,cACA4B,mBACA1B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACwE;YAO3C,IAAI/D,QAAQ,QACV,OAAO;gBACL,QAAQnB;gBACRkB;YACF;YAGF6F,eACE5F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEgF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMtD,UAAU,CAAC,kBAAkB,EAAE2C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGvD;oBAAQ;iBAAE,EAC9DmB,cACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAM1G,QAAQ,YAAY,CAAC0G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO1G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEqG,cAAc;IACnE;IAx0BA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,yCAAAA,WAAWA,CAAC;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AA4yBF;AAEO,eAAexF,gBACpBsF,iBAAoC,EACpCzF,iBAAuC,EACvCoB,MAAwB;IAExB,IAAI,CAACpB,mBAAmB,QACtB,OAAOoB;IAGT,IAAI,CAACqE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC9F;IACvB;IAEA,MAAM,EAAE+F,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMtG,SAAS,MAAM6B;QAErB,MAAM9C,QAAQ,MAAM0H;QACpB,IAAI1H,OACF,MAAMA;QAER,OAAOiB;IACT,SAAU;QACRwG;IACF;AACF"}
1	+ {"version":3,"file":"agent/tasks.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/tasks.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n type TMultimodalPrompt,\n type TUserPrompt,\n getReadableTimeString,\n multimodalPromptToChatMessages,\n userPromptToMultimodalPrompt,\n userPromptToString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n DeviceAction,\n ExecutionTask,\n ExecutionTaskApply,\n ExecutionTaskInsightQueryApply,\n ExecutionTaskPlanningApply,\n ExecutionTaskProgressOptions,\n MidsceneYamlFlowItem,\n PlanningAIResponse,\n PlanningAction,\n PlanningActionParamWaitFor,\n PlanningLocateParam,\n ServiceDump,\n ServiceExtractOption,\n ServiceExtractParam,\n UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n output: OutputType;\n thought?: string;\n runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n onTaskUpdate?: (\n runner: TaskRunner,\n error?: TaskExecutionError,\n ) => Promise<void> \| void;\n}\n\nexport type ActionReportOptions = {\n type?: TaskTitleType;\n prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap each task's planning feedback so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. This is the\n// single place that truncates feedback before it is sent to the model; action\n// implementations should hand over the untruncated value.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(feedback: string): string {\n if (feedback.length <= maxPlanningFeedbackLength) {\n return feedback;\n }\n\n return `${feedback.slice(0, maxPlanningFeedbackLength)}\n...[truncated, ${feedback.length - maxPlanningFeedbackLength} more characters]`;\n}\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n interface: AbstractInterface;\n\n service: Service;\n\n taskCache?: TaskCache;\n\n private readonly providedActionSpace: DeviceAction[];\n\n private readonly taskBuilder: TaskBuilder;\n\n onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n private readonly hooks?: TaskExecutorHooks;\n\n replanningCycleLimit?: number;\n\n waitAfterAction?: number;\n\n useDeviceTime?: boolean;\n\n // @deprecated use .interface instead\n get page() {\n return this.interface;\n }\n\n constructor(\n interfaceInstance: AbstractInterface,\n service: Service,\n opts: {\n taskCache?: TaskCache;\n onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n replanningCycleLimit?: number;\n waitAfterAction?: number;\n useDeviceTime?: boolean;\n hooks?: TaskExecutorHooks;\n actionSpace: DeviceAction[];\n },\n ) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = opts.taskCache;\n this.onTaskStartCallback = opts?.onTaskStart;\n this.replanningCycleLimit = opts.replanningCycleLimit;\n this.waitAfterAction = opts.waitAfterAction;\n this.useDeviceTime = opts.useDeviceTime;\n this.hooks = opts.hooks;\n this.providedActionSpace = opts.actionSpace;\n this.taskBuilder = new TaskBuilder({\n interfaceInstance,\n service,\n taskCache: opts.taskCache,\n actionSpace: this.getActionSpace(),\n waitAfterAction: opts.waitAfterAction,\n });\n }\n\n private createExecutionSession(\n title: string,\n options?: { tasks?: ExecutionTaskApply[]; uiContext?: UIContext },\n ) {\n return new ExecutionSession(\n title,\n () =>\n options?.uiContext\n ? Promise.resolve(options.uiContext)\n : Promise.resolve(this.service.contextRetrieverFn()),\n {\n onTaskStart: this.onTaskStartCallback,\n tasks: options?.tasks,\n onTaskUpdate: this.hooks?.onTaskUpdate,\n },\n );\n }\n\n private getActionSpace(): DeviceAction[] {\n return this.providedActionSpace;\n }\n\n /*\n Set the pending feedback message consumed by the next planning round.\n * The message is always prefixed with the current time. When a body is\n * provided it is appended after the timestamp; otherwise only the time\n * context is recorded. This is the single entry point for writing\n * `pendingFeedbackMessage` so the time prefix stays consistent.\n /\n private setPendingFeedbackMessage(\n conversationHistory: ConversationHistory,\n timeString: string,\n body?: string,\n ) {\n conversationHistory.pendingFeedbackMessage = body\n ? `Time: ${timeString}, ${body}`\n : `Current time: ${timeString}`;\n }\n\n /\n Collect feedback produced by executed tasks for the next planning round.\n * Returns undefined when no task reported feedback.\n /\n private collectPlanningFeedback(tasks: ExecutionTask[]): string \| undefined {\n const feedbackMessages = tasks.flatMap(({ planningFeedback }) =>\n planningFeedback ? [truncatePlanningFeedback(planningFeedback)] : [],\n );\n return feedbackMessages.length > 0\n ? feedbackMessages.join('\\n\\n')\n : undefined;\n }\n\n /\n Get a readable time string. When device time is enabled, use the\n * device-formatted wall-clock time directly so host timezone formatting does\n * not reinterpret a device timestamp.\n * @param format - Optional format string\n * @returns A formatted time string\n /\n private async getTimeString(format?: string): Promise<string> {\n if (this.useDeviceTime) {\n if (this.interface.getDeviceLocalTimeString) {\n try {\n return await this.interface.getDeviceLocalTimeString(format);\n } catch (error) {\n warnLog(\n `Failed to get device time string, falling back to runtime time: ${error}`,\n );\n }\n } else {\n warnLog(\n 'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n );\n }\n }\n\n return getReadableTimeString(format);\n }\n\n public async convertPlanToExecutable(\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n options?: {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n },\n ) {\n return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n }\n\n async loadYamlFlowAsPlanning(\n userInstruction: TUserPrompt,\n yamlString: string,\n reportOptions?: ActionReportOptions,\n ) {\n const session = this.createExecutionSession(\n taskTitleStr(\n reportOptions?.type \|\| 'Act',\n reportOptions?.prompt \|\| userPromptToString(userInstruction),\n ),\n );\n\n const task: ExecutionTaskPlanningApply = {\n type: 'Planning',\n subType: 'LoadYaml',\n param: {\n userInstruction,\n ...(reportOptions?.prompt\n ? { userInstructionDisplay: reportOptions.prompt }\n : {}),\n },\n executor: async (param, executorContext) => {\n const { uiContext } = executorContext;\n assert(uiContext, 'uiContext is required for Planning task');\n return {\n output: {\n actions: [],\n shouldContinuePlanning: false,\n log: '',\n yamlString,\n },\n cache: {\n hit: true,\n },\n hitBy: {\n from: 'Cache',\n context: {\n yamlString,\n },\n },\n };\n },\n };\n const runner = session.getRunner();\n await session.appendAndRun(task);\n\n return {\n runner,\n };\n }\n\n async runPlans(\n title: string,\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n options?: { uiContext?: UIContext },\n ): Promise<ExecutionResult> {\n const session = this.createExecutionSession(title, options);\n const { tasks } = await this.convertPlanToExecutable(\n plans,\n planningModel,\n defaultModel,\n );\n const runner = session.getRunner();\n const result = await session.appendAndRun(tasks);\n const { output } = result ?? {};\n return {\n output,\n runner,\n };\n }\n\n async action(\n userPrompt: TUserPrompt,\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n includeLocateInPlanning: boolean,\n aiActContext?: string,\n cacheable?: boolean,\n replanningCycleLimitOverride?: number,\n imagesIncludeCount?: number,\n deepThink?: boolean,\n fileChooserAccept?: string[],\n deepLocate?: boolean,\n abortSignal?: AbortSignal,\n reportOptions?: ActionReportOptions,\n ): Promise<\n ExecutionResult<\n \| {\n yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n output?: string;\n }\n \| undefined\n >\n > {\n return withFileChooser(this.interface, fileChooserAccept, async () => {\n return this.runAction(\n userPrompt,\n planningModel,\n defaultModel,\n includeLocateInPlanning,\n aiActContext,\n cacheable,\n replanningCycleLimitOverride,\n imagesIncludeCount,\n deepThink,\n deepLocate,\n abortSignal,\n reportOptions,\n );\n });\n }\n\n /\n Called when the task is about to replan. Marks every cache-hit locate task\n * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n * did not finish the task, so the element each cache hit produced is suspect.\n * The upcoming re-locate of the same prompt then replaces the bad entry in\n * place instead of appending a duplicate that would re-poison the cache on the\n * next run (#2529).\n \n Marking a locate that was actually fine is harmless: the step is only ever\n * replaced if the same prompt is located again (i.e. the step is redone),\n * which does not happen for a locate that already succeeded.\n */\n private invalidateFailedCacheHitLocates(\n runner: TaskRunner,\n fromIndex: number,\n ) {\n if (!this.taskCache) {\n return;\n }\n for (let i = fromIndex; i < runner.tasks.length; i++) {\n const task = runner.tasks[i];\n if (\n task.type === 'Planning' &&\n task.subType === 'Locate' &&\n task.hitBy?.from === 'Cache'\n ) {\n const prompt = (task.param as PlanningLocateParam \| undefined)?.prompt;\n if (prompt) {\n this.taskCache.markLocateCacheStale(prompt);\n }\n }\n }\n }\n\n private async runAction(\n userPrompt: TUserPrompt,\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n includeLocateInPlanning: boolean,\n aiActContext?: string,\n cacheable?: boolean,\n replanningCycleLimitOverride?: number,\n imagesIncludeCount?: number,\n deepThink?: boolean,\n deepLocate?: boolean,\n abortSignal?: AbortSignal,\n reportOptions?: ActionReportOptions,\n ): Promise<\n ExecutionResult<\n \| {\n yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n output?: string;\n }\n \| undefined\n >\n > {\n const conversationHistory = new ConversationHistory();\n\n const session = this.createExecutionSession(\n taskTitleStr(\n reportOptions?.type \|\| 'Act',\n reportOptions?.prompt \|\| userPromptToString(userPrompt),\n ),\n );\n const runner = session.getRunner();\n\n let replanCount = 0;\n const yamlFlow: MidsceneYamlFlowItem[] = [];\n const replanningCycleLimit =\n replanningCycleLimitOverride ?? this.replanningCycleLimit;\n assert(\n replanningCycleLimit !== undefined,\n 'replanningCycleLimit is required for TaskExecutor.action',\n );\n\n let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n let outputString: string \| undefined;\n\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n const referenceImageMessages = await multimodalPromptToChatMessages(\n userPromptToMultimodalPrompt(userPrompt),\n );\n\n // Main planning loop - unified plan/replan logic\n while (true) {\n // Check abort signal before each planning cycle\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n\n // Get sub-goal status text if available\n const subGoalStatus = conversationHistory.subGoalsToText() \|\| undefined;\n\n // Get memories text if available\n const memoriesStatus = conversationHistory.memoriesToText() \|\| undefined;\n\n const result = await session.appendAndRun(\n {\n type: 'Planning',\n subType: 'Plan',\n param: {\n userInstruction: userPrompt,\n ...(reportOptions?.prompt\n ? { userInstructionDisplay: reportOptions.prompt }\n : {}),\n aiActContext,\n imagesIncludeCount,\n deepThink,\n ...(subGoalStatus ? { subGoalStatus } : {}),\n ...(memoriesStatus ? { memoriesStatus } : {}),\n },\n executor: async (param, executorContext) => {\n const { uiContext } = executorContext;\n assert(uiContext, 'uiContext is required for Planning task');\n const timing = executorContext.task.timing;\n\n const actionSpace = this.getActionSpace();\n debug(\n 'actionSpace for this interface is:',\n actionSpace.map((action) => action.name).join(', '),\n );\n assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n if (actionSpace.length === 0) {\n console.warn(\n `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n );\n }\n\n const planImpl =\n planningModel.adapter.planning.kind === 'custom'\n ? planningModel.adapter.planning.planFn\n : genericXmlPlan;\n\n let planResult: Awaited<ReturnType<typeof planImpl>>;\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n planResult = await planImpl(param.userInstruction, {\n context: uiContext,\n actionContext: param.aiActContext,\n actionSpace,\n modelRuntime: planningModel,\n conversationHistory,\n includeLocateInPlanning,\n imagesIncludeCount,\n deepThink,\n referenceImageMessages,\n abortSignal,\n });\n } catch (planError) {\n if (planError instanceof AIResponseParseError) {\n // Record usage and rawResponse even when parsing fails\n executorContext.task.usage = withUsageIntent(\n planError.usage,\n 'planning',\n );\n executorContext.task.log = {\n ...(executorContext.task.log \|\| {}),\n rawResponse: planError.rawResponse,\n rawChoiceMessage: planError.rawChoiceMessage,\n };\n }\n throw planError;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n debug('planResult', JSON.stringify(planResult, null, 2));\n\n const {\n actions,\n thought,\n log,\n memory,\n error,\n usage,\n rawResponse,\n rawChoiceMessage,\n reasoning_content,\n finalizeSuccess,\n finalizeMessage,\n updateSubGoals,\n markFinishedIndexes,\n } = planResult;\n outputString = finalizeMessage;\n\n executorContext.task.log = {\n ...(executorContext.task.log \|\| {}),\n rawResponse,\n rawChoiceMessage,\n };\n executorContext.task.usage = withUsageIntent(usage, 'planning');\n executorContext.task.reasoning_content = reasoning_content;\n executorContext.task.output = {\n actions: actions \|\| [],\n log,\n thought,\n memory,\n yamlFlow: planResult.yamlFlow,\n output: finalizeMessage,\n shouldContinuePlanning: planResult.shouldContinuePlanning,\n updateSubGoals,\n markFinishedIndexes,\n };\n executorContext.uiContext = uiContext;\n\n assert(!error, `Failed to continue: ${error}\\n${log \|\| ''}`);\n\n // Check if task was finalized with failure\n if (finalizeSuccess === false) {\n assert(\n false,\n `Task failed: ${finalizeMessage \|\| 'No error message provided'}\\n${log \|\| ''}`,\n );\n }\n\n return {\n cache: {\n hit: false,\n },\n } as any;\n },\n },\n {\n allowWhenError: true,\n },\n );\n\n const planResult = result?.output as PlanningAIResponse \| undefined;\n\n // Execute planned actions\n const plans = planResult?.actions \|\| [];\n yamlFlow.push(...(planResult?.yamlFlow \|\| []));\n\n let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n try {\n executables = await this.convertPlanToExecutable(\n plans,\n planningModel,\n defaultModel,\n {\n cacheable,\n deepLocate,\n abortSignal,\n },\n );\n } catch (error) {\n return session.appendErrorPlan(\n `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n plans,\n )}`,\n );\n }\n if (conversationHistory.pendingFeedbackMessage) {\n console.warn(\n 'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n conversationHistory.pendingFeedbackMessage,\n );\n }\n\n // Capture the time context for the next planning call before running.\n const initialTimeString = await this.getTimeString();\n\n const taskCountBeforeRun = runner.tasks.length;\n try {\n await session.appendAndRun(executables.tasks);\n this.setPendingFeedbackMessage(\n conversationHistory,\n initialTimeString,\n this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n );\n } catch (error: any) {\n // errorFlag = true;\n errorCountInOnePlanningLoop++;\n const timeString = await this.getTimeString();\n this.setPendingFeedbackMessage(\n conversationHistory,\n timeString,\n `Error executing running tasks: ${error?.message \|\| String(error)}`,\n );\n debug(\n 'error when executing running tasks, but continue to run if it is not too many errors:',\n error instanceof Error ? error.message : String(error),\n 'current error count in one planning loop:',\n errorCountInOnePlanningLoop,\n );\n }\n\n if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n return session.appendErrorPlan('Too many errors in one planning loop');\n }\n\n // Check abort signal after executing actions\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n\n // // Check if task is complete\n if (!planResult?.shouldContinuePlanning) {\n break;\n }\n\n // We are about to replan, which means the batch we just ran did not finish\n // the task. Any locate task in that batch that was served from cache\n // produced an element that failed to complete the step (the action threw,\n // or it clicked the wrong element and the goal was not reached). Mark those\n // cache entries stale so the re-locate of the same prompt replaces them in\n // place instead of appending a poisoning duplicate that would be matched\n // first on the next run (#2529).\n this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n // Increment replan count for next iteration\n ++replanCount;\n\n if (replanCount > replanningCycleLimit) {\n const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n return session.appendErrorPlan(errorMsg);\n }\n\n if (!conversationHistory.pendingFeedbackMessage) {\n const timeString = await this.getTimeString();\n conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n }\n }\n\n return {\n output: {\n yamlFlow,\n output: outputString,\n },\n runner,\n };\n }\n\n private createTypeQueryTask(\n type: 'Query' \| 'Boolean' \| 'Number' \| 'String' \| 'Assert' \| 'WaitFor',\n demand: ServiceExtractParam,\n modelRuntime: ModelRuntime,\n opt?: ServiceExtractOption,\n multimodalPrompt?: TMultimodalPrompt,\n executionOptions?: {\n abortSignal?: AbortSignal;\n },\n ) {\n const queryTask: ExecutionTaskInsightQueryApply = {\n type: 'Insight',\n subType: type,\n param: {\n domIncluded: opt?.domIncluded,\n dataDemand: multimodalPrompt\n ? ({\n demand,\n multimodalPrompt,\n } as never)\n : demand, // for user param presentation in report right sidebar\n },\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let queryDump: ServiceDump \| undefined;\n const applyDump = (dump: ServiceDump) => {\n queryDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n searchAreaRawChoiceMessage:\n dump.taskInfo?.searchAreaRawChoiceMessage,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n // Get context for query operations\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Query task');\n\n const ifTypeRestricted = type !== 'Query';\n let demandInput = demand;\n let keyOfResult = 'result';\n if (ifTypeRestricted && (type === 'Assert' \|\| type === 'WaitFor')) {\n keyOfResult = 'StatementIsTruthy';\n demandInput = {\n [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n };\n } else if (ifTypeRestricted) {\n keyOfResult = type;\n demandInput = {\n [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n };\n }\n\n let extractResult;\n\n let extraPageDescription = '';\n if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n debug('appending tree info for page');\n const tree = await this.interface.getElementsNodeTree();\n extraPageDescription = await descriptionOfTree(\n tree,\n 200,\n false,\n opt?.domIncluded === 'visible-only',\n );\n }\n\n try {\n extractResult = await this.service.extract<any>(\n demandInput,\n modelRuntime,\n opt,\n extraPageDescription,\n multimodalPrompt,\n uiContext,\n executionOptions,\n );\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n }\n\n const { data, thought, dump } = extractResult;\n applyDump(dump);\n\n let outputResult = data;\n if (ifTypeRestricted) {\n // If AI returned a plain string instead of structured format, use it directly\n if (typeof data === 'string') {\n outputResult = data;\n } else if (type === 'WaitFor') {\n if (data === null \|\| data === undefined) {\n outputResult = false;\n } else {\n outputResult = (data as any)[keyOfResult];\n }\n } else if (data === null \|\| data === undefined) {\n outputResult = null;\n } else {\n // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n if (data?.[keyOfResult] !== undefined) {\n outputResult = (data as any)[keyOfResult];\n } else if (data?.result !== undefined) {\n outputResult = (data as any).result;\n } else {\n assert(false, 'No result in query data');\n }\n }\n }\n\n if (type === 'Assert' && !outputResult) {\n task.thought = thought;\n throw new Error(`Assertion failed: ${thought}`);\n }\n\n return {\n output: outputResult,\n log: queryDump,\n thought,\n };\n },\n };\n\n return queryTask;\n }\n async createTypeQueryExecution<T>(\n type: 'Query' \| 'Boolean' \| 'Number' \| 'String' \| 'Assert',\n demand: ServiceExtractParam,\n modelRuntime: ModelRuntime,\n opt?: ServiceExtractOption,\n multimodalPrompt?: TMultimodalPrompt,\n executionOptions?: {\n abortSignal?: AbortSignal;\n },\n ): Promise<ExecutionResult<T>> {\n const session = this.createExecutionSession(\n taskTitleStr(\n type,\n typeof demand === 'string' ? demand : JSON.stringify(demand),\n ),\n );\n\n const queryTask = await this.createTypeQueryTask(\n type,\n demand,\n modelRuntime,\n opt,\n multimodalPrompt,\n executionOptions,\n );\n\n const runner = session.getRunner();\n const result = await session.appendAndRun(queryTask);\n\n if (!result) {\n throw new Error(\n 'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n );\n }\n\n const { output, thought } = result;\n\n return {\n output,\n thought,\n runner,\n };\n }\n\n async waitFor(\n assertion: TUserPrompt,\n opt: PlanningActionParamWaitFor,\n modelRuntime: ModelRuntime,\n ): Promise<ExecutionResult<void>> {\n const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n const description = `waitFor: ${textPrompt}`;\n const session = this.createExecutionSession(\n taskTitleStr('WaitFor', description),\n );\n const runner = session.getRunner();\n const {\n timeoutMs,\n checkIntervalMs,\n domIncluded,\n screenshotIncluded,\n ...restOpt\n } = opt;\n const serviceExtractOpt: ServiceExtractOption = {\n domIncluded,\n screenshotIncluded,\n ...restOpt,\n };\n\n assert(assertion, 'No assertion for waitFor');\n assert(timeoutMs, 'No timeoutMs for waitFor');\n assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n assert(\n checkIntervalMs <= timeoutMs,\n `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n );\n\n const overallStartTime = Date.now();\n let lastCheckStart = overallStartTime;\n let errorThought = '';\n // Continue checking as long as the previous iteration began within the timeout window.\n while (lastCheckStart - overallStartTime <= timeoutMs) {\n const currentCheckStart = Date.now();\n lastCheckStart = currentCheckStart;\n const queryTask = await this.createTypeQueryTask(\n 'WaitFor',\n textPrompt,\n modelRuntime,\n serviceExtractOpt,\n multimodalPrompt,\n );\n\n const result = (await session.appendAndRun(queryTask)) as\n \| {\n output: boolean;\n thought?: string;\n }\n \| undefined;\n\n if (result?.output) {\n return {\n output: undefined,\n runner,\n };\n }\n\n errorThought =\n result?.thought \|\|\n (!result && `No result from assertion: ${textPrompt}`) \|\|\n `unknown error when waiting for assertion: ${textPrompt}`;\n const now = Date.now();\n if (now - currentCheckStart < checkIntervalMs) {\n const elapsed = now - currentCheckStart;\n const timeRemaining = checkIntervalMs - elapsed;\n const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n modelRuntime,\n modelRuntime,\n );\n if (sleepTasks[0]) {\n await session.appendAndRun(sleepTasks[0]);\n }\n }\n }\n\n return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n }\n}\n\nexport async function withFileChooser<T>(\n interfaceInstance: AbstractInterface,\n fileChooserAccept: string[] \| undefined,\n action: () => Promise<T>,\n): Promise<T> {\n if (!fileChooserAccept?.length) {\n return action();\n }\n\n if (!interfaceInstance.registerFileChooserListener) {\n throw new Error(\n `File upload is not supported on ${interfaceInstance.interfaceType}`,\n );\n }\n\n const handler = async (chooser: FileChooserHandler) => {\n await chooser.accept(fileChooserAccept);\n };\n\n const { dispose, getError } =\n await interfaceInstance.registerFileChooserListener(handler);\n try {\n const result = await action();\n // Check for errors that occurred during file chooser handling\n const error = await getError();\n if (error) {\n throw error;\n }\n return result;\n } finally {\n dispose();\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","TaskExecutor","title","options","ExecutionSession","Promise","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","undefined","format","error","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","task","param","executorContext","uiContext","assert","runner","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","fromIndex","i","prompt","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","errorCountInOnePlanningLoop","outputString","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","subGoalStatus","memoriesStatus","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","rawChoiceMessage","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","taskCountBeforeRun","String","Error","errorMsg","type","demand","modelRuntime","opt","multimodalPrompt","executionOptions","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC0DA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAM9C,MAAMC,4BAA4B;AAElC,SAASC,yBAAyBC,QAAgB;IAChD,IAAIA,SAAS,MAAM,IAAIF,2BACrB,OAAOE;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGF,2BAA2B;eAC1C,EAAEE,SAAS,MAAM,GAAGF,0BAA0B,iBAAiB,CAAC;AAC/E;AAIO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAAiE,EACjE;QACA,OAAO,IAAIC,8CAAAA,gBAAgBA,CACzBF,OACA,IACEC,SAAS,YACLE,QAAQ,OAAO,CAACF,QAAQ,SAAS,IACjCE,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASQ,0BACNG,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CAAC,CAAC,EAAEE,gBAAgB,EAAE,GAC1DA,mBAAmB;gBAACZ,yBAAyBY;aAAkB,GAAG,EAAE;QAEtE,OAAOD,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBE;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdlB,QACE,CAAC,gEAAgE,EAAEkB,OAAO;QAE9E;aAEAlB,QACE;QAKN,OAAOmB,AAAAA,IAAAA,mCAAAA,qBAAAA,AAAAA,EAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACa,OAAOC,eAAeC,cAAcf;IACpE;IAEA,MAAM,uBACJgB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBL;QAIhD,MAAMM,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLN;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMU,SAASR,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACG;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJ5B,KAAa,EACbc,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAAmC,EACT;QAC1B,MAAMmB,UAAU,IAAI,CAAC,sBAAsB,CAACpB,OAAOC;QACnD,MAAM,EAAEM,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDO,OACAC,eACAC;QAEF,MAAMY,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACb;QAC1C,MAAM,EAAEuB,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAF;QACF;IACF;IAEA,MAAM,OACJG,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,OAAOsB,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAhB,eACAC,cACAgB,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC,aACArB;IAGN;IAcQ,gCACNS,MAAkB,EAClBc,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAIf,OAAO,KAAK,CAAC,MAAM,EAAEe,IAAK;YACpD,MAAMpB,OAAOK,OAAO,KAAK,CAACe,EAAE;YAC5B,IACEpB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMqB,SAAUrB,KAAK,KAAK,EAAsC;gBAChE,IAAIqB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZb,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,MAAMf,sBAAsB,IAAIyC,yBAAAA,mBAAmBA;QAEnD,MAAMzB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBS;QAGhD,MAAMH,SAASR,QAAQ,SAAS;QAEhC,IAAI0B,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJb,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DR,IAAAA,sBAAAA,MAAAA,AAAAA,EACEqB,AAAyBtC,WAAzBsC,sBACA;QAGF,IAAIC,8BAA8B;QAClC,IAAIC;QAEJ,IAAIV,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMW,yBAAyB,MAAMC,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EACnCC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6BtB;QAI/B,MAAO,KAAM;YAEX,IAAIS,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMc,gBAAgBlD,oBAAoB,cAAc,MAAMM;YAG9D,MAAM6C,iBAAiBnD,oBAAoB,cAAc,MAAMM;YAE/D,MAAMmB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjB,GAAIZ,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACNc;oBACAG;oBACAC;oBACA,GAAIiB,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO/B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;oBAClB,MAAM8B,SAAS/B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAMgC,cAAc,IAAI,CAAC,cAAc;oBACvCjE,MACE,sCACAiE,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD/B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOgC,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJ9C,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrC+C,kCAAAA,cAAcA;oBAEpB,IAAIC;oBACJ,IAAI;wBACFC,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAASrC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjCiC;4BACA,cAAc1C;4BACdX;4BACA4B;4BACAI;4BACAC;4BACAc;4BACAX;wBACF;oBACF,EAAE,OAAOyB,WAAW;wBAClB,IAAIA,qBAAqBC,yBAAAA,oBAAoBA,EAAE;4BAE7CzC,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAC3BF,UAAU,KAAK,EACf;4BAEFxC,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAawC,UAAU,WAAW;gCAClC,kBAAkBA,UAAU,gBAAgB;4BAC9C;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBR,QAAQ;oBAC7B;oBACAhE,MAAM,cAAc4E,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN5D,KAAK,EACL6D,KAAK,EACLC,WAAW,EACXC,gBAAgB,EAChBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGjB;oBACJb,eAAe4B;oBAEfrD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCiD;wBACAC;oBACF;oBACAlD,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgBM,OAAO;oBACpDhD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGmD;oBACzCnD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS4C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQe;wBACR,wBAAwBf,WAAW,sBAAsB;wBACzDgB;wBACAC;oBACF;oBACAvD,gBAAgB,SAAS,GAAGC;oBAE5BC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACf,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAE2D,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBACFlD,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,OACA,CAAC,aAAa,EAAEmD,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAalC,QAAQ;YAG3B,MAAMf,QAAQiD,YAAY,WAAW,EAAE;YACvChB,SAAS,IAAI,IAAKgB,YAAY,YAAY,EAAE;YAE5C,IAAIkB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CnE,OACAC,eACAC,cACA;oBACEkB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO5B,OAAO;gBACd,OAAOQ,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAER,MAAM,SAAS,EAAEwD,KAAK,SAAS,CAC5EtD,QACC;YAEP;YACA,IAAIV,oBAAoB,sBAAsB,EAC5CwD,QAAQ,IAAI,CACV,8FACAxD,oBAAoB,sBAAsB;YAK9C,MAAM8E,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqBvD,OAAO,KAAK,CAAC,MAAM;YAC9C,IAAI;gBACF,MAAMR,QAAQ,YAAY,CAAC6D,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5B7E,qBACA8E,mBACA,IAAI,CAAC,uBAAuB,CAACtD,OAAO,KAAK,CAAC,KAAK,CAACuD;YAEpD,EAAE,OAAOvE,OAAY;gBAEnBqC;gBACA,MAAM5C,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEO,OAAO,WAAWwE,OAAOxE,QAAQ;gBAErEpB,MACE,yFACAoB,iBAAiByE,QAAQzE,MAAM,OAAO,GAAGwE,OAAOxE,QAChD,6CACAqC;YAEJ;YAEA,IAAIA,8BAA8BtD,uCAChC,OAAOyB,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACuB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAACnC,QAAQuD;YAG7C,EAAErC;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMsC,WAAW,CAAC,UAAU,EAAEtC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO5B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAClF,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACN0C;gBACA,QAAQG;YACV;YACAtB;QACF;IACF;IAEQ,oBACN2D,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAEC,EACD;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASN;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOhE,OAAOsE;gBACtB,MAAM,EAAEvE,IAAI,EAAE,GAAGuE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ1E,KAAK,GAAG,GAAG;wBACT0E;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA1E,KAAK,KAAK,GAAG4C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgB8B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1E,KAAK,iBAAiB,GAAG0E,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMvE,YAAYoE,YAAY,SAAS;gBACvCnE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElB,MAAMwE,mBAAmBX,AAAS,YAATA;gBACzB,IAAIY,cAAcX;gBAClB,IAAIY,cAAc;gBAClB,IAAIF,oBAAqBX,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEa,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,AAAAA,IAAAA,8BAAAA,yBAAAA,AAAAA,EAA0Bd,MAAMC;oBACjD;gBACF,OAAO,IAAIU,kBAAkB;oBAC3BE,cAAcb;oBACdY,cAAc;wBACZ,CAACC,YAAY,EAAEC,AAAAA,IAAAA,8BAAAA,yBAAAA,AAAAA,EAA0Bd,MAAMC;oBACjD;gBACF;gBAEA,IAAIc;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIb,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DlG,MAAM;oBACN,MAAMgH,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,AAAAA,IAAAA,0BAAAA,iBAAAA,AAAAA,EAC3BD,MACA,KACA,OACAd,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAV,cACAC,KACAa,sBACAZ,kBACAjE,WACAkE;gBAEJ,EAAE,OAAOhF,OAAO;oBACd,IAAIA,iBAAiB8F,kCAAAA,YAAYA,EAC/BV,UAAUpF,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE+F,IAAI,EAAErC,OAAO,EAAE2B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAIpB,AAAS,cAATA,MAEPqB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAK1F,QAC1BkG,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAWjG,QAC1BkG,eAAgBD,KAAa,MAAM;qBAEnChF,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,OAAO;gBAKpB,IAAI4D,AAAS,aAATA,QAAqB,CAACqB,cAAc;oBACtCrF,KAAK,OAAO,GAAG+C;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQsC;oBACR,KAAKb;oBACLzB;gBACF;YACF;QACF;QAEA,OAAOuB;IACT;IACA,MAAM,yBACJN,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAEC,EAC4B;QAC7B,MAAMxE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMK,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CN,MACAC,QACAC,cACAC,KACAC,kBACAC;QAGF,MAAMhE,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACyE;QAE1C,IAAI,CAAChE,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEwC,OAAO,EAAE,GAAGzC;QAE5B,OAAO;YACLC;YACAwC;YACA1C;QACF;IACF;IAEA,MAAM,QACJiF,SAAsB,EACtBnB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEqB,UAAU,EAAEnB,gBAAgB,EAAE,GAAGoB,AAAAA,IAAAA,kCAAAA,WAAAA,AAAAA,EAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM1F,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,WAAW2F;QAE1B,MAAMpF,SAASR,QAAQ,SAAS;QAChC,MAAM,EACJ6F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG3B;QACJ,MAAM4B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA1F,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOkF,WAAW;QAClBlF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsF,WAAW;QAClBtF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuF,iBAAiB;QAExBvF,IAAAA,sBAAAA,MAAAA,AAAAA,EACEuF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACArB,cACA6B,mBACA3B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACyE;YAO3C,IAAIhE,QAAQ,QACV,OAAO;gBACL,QAAQnB;gBACRkB;YACF;YAGF8F,eACE7F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEiF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMvD,UAAU,CAAC,kBAAkB,EAAE4C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGxD;oBAAQ;iBAAE,EAC9DmB,cACAA;gBAEF,IAAIsC,UAAU,CAAC,EAAE,EACf,MAAM3G,QAAQ,YAAY,CAAC2G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO3G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEsG,cAAc;IACnE;IAt0BA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,yCAAAA,WAAWA,CAAC;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AA0yBF;AAEO,eAAezF,gBACpBuF,iBAAoC,EACpC1F,iBAAuC,EACvCoB,MAAwB;IAExB,IAAI,CAACpB,mBAAmB,QACtB,OAAOoB;IAGT,IAAI,CAACsE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI3C,MACR,CAAC,gCAAgC,EAAE2C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC/F;IACvB;IAEA,MAAM,EAAEgG,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMvG,SAAS,MAAM6B;QAErB,MAAM9C,QAAQ,MAAM2H;QACpB,IAAI3H,OACF,MAAMA;QAER,OAAOiB;IACT,SAAU;QACRyG;IACF;AACF"}

package/dist/lib/agent/utils.js CHANGED Viewed

@@ -62,6 +62,18 @@ const external_dayjs_namespaceObject = require("dayjs");
 var external_dayjs_default = /*#__PURE__*/ __webpack_require__.n(external_dayjs_namespaceObject);
 const external_task_cache_js_namespaceObject = require("./task-cache.js");
 const agentDebug = (0, logger_namespaceObject.getDebug)('agent');
+const screenshotDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i;
+const inferBase64ImageFormat = (base64Body)=>{
+    if (base64Body.startsWith('iVBORw0KGgo')) return 'png';
+    return 'jpeg';
+};
+const normalizeScreenshotBase64 = (screenshotBase64)=>{
+    const trimmedBase64 = screenshotBase64.trim();
+    if (screenshotDataUrlPattern.test(trimmedBase64)) return trimmedBase64;
+    const base64Body = trimmedBase64.replace(/\s/g, '');
+    (0, utils_namespaceObject.assert)(base64Body, 'screenshotBase64 must include image data');
+    return (0, img_namespaceObject.createImgBase64ByFormat)(inferBase64ImageFormat(base64Body), base64Body);
+};
 const legacyScrollTypeMap = {
     once: 'singleAction',
     untilBottom: 'scrollToBottom',
@@ -143,7 +155,7 @@ async function commonContextParser(interfaceInstance, _opt) {
     };
 }
 async function createScreenshotBoundUIContext(screenshotBase64, opt) {
-    const normalizedScreenshotBase64 = (0, img_namespaceObject.normalizeBase64Image)(screenshotBase64);
+    const normalizedScreenshotBase64 = normalizeScreenshotBase64(screenshotBase64);
     const actualScreenshotSize = await (0, img_namespaceObject.imageInfoOfBase64)(normalizedScreenshotBase64);
     if (opt.screenshotSize && (opt.screenshotSize.width !== actualScreenshotSize.width || opt.screenshotSize.height !== actualScreenshotSize.height)) agentDebug('describeElementAtPoint screenshotSize mismatch, use actual size', {
         provided: opt.screenshotSize,
@@ -212,7 +224,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.9.8";
+const getMidsceneVersion = ()=>"1.10.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,