@midscene/core 1.9.8 → 1.10.0

package/dist/es/agent/run-gherkin-scenario.mjs

@@ -0,0 +1,114 @@
+const stepKeywordPattern = /^(Given|When|Then|And|But)\s+(.+)$/i;
+const headerPattern = /^(Scenario):\s*(.*)$/i;
+const unsupportedHeaderPattern = /^(Feature|Background|Scenario Outline|Scenario Template|Examples|Rule):/i;
+const normalizeStepKeyword = (keyword)=>{
+    const lowerKeyword = keyword.toLowerCase();
+    if ('given' === lowerKeyword) return 'Given';
+    if ('when' === lowerKeyword) return 'When';
+    if ('then' === lowerKeyword) return 'Then';
+    if ('and' === lowerKeyword) return 'And';
+    return 'But';
+};
+const isPrimaryKeyword = (keyword)=>'Given' === keyword || 'When' === keyword || 'Then' === keyword;
+const resolveSteps = (steps)=>{
+    let previousPrimaryKeyword;
+    return steps.map((step)=>{
+        const effectiveKeyword = isPrimaryKeyword(step.keyword) ? step.keyword : previousPrimaryKeyword;
+        if (!effectiveKeyword) throw new Error(`runGherkinScenario cannot resolve "${step.keyword}" at line ${step.lineNumber}; use Given, When, or Then before ${step.keyword}.`);
+        previousPrimaryKeyword = effectiveKeyword;
+        return {
+            ...step,
+            effectiveKeyword
+        };
+    });
+};
+const throwIfAborted = (abortSignal)=>{
+    if (!abortSignal?.aborted) return;
+    if ('function' == typeof abortSignal.throwIfAborted) abortSignal.throwIfAborted();
+    throw new Error(`runGherkinScenario aborted: ${abortSignal.reason || 'signal already aborted'}`);
+};
+const parseGherkinScenario = (scenarioText)=>{
+    const lines = scenarioText.split(/\r?\n/);
+    const scenarioSteps = [];
+    const anonymousSteps = [];
+    let scenario;
+    let scenarioCount = 0;
+    let section = 'prelude';
+    for (const [lineIndex, rawLine] of lines.entries()){
+        const lineNumber = lineIndex + 1;
+        const line = rawLine.trim();
+        if (!line || line.startsWith('#') || line.startsWith('@')) continue;
+        if ('"""' === line || "'''" === line) throw new Error(`runGherkinScenario does not support doc strings; found one at line ${lineNumber}.`);
+        if (line.startsWith('|')) throw new Error(`runGherkinScenario does not support data tables; found one at line ${lineNumber}.`);
+        if (unsupportedHeaderPattern.test(line)) throw new Error(`runGherkinScenario does not support "${line}" at line ${lineNumber}.`);
+        const headerMatch = line.match(headerPattern);
+        if (headerMatch) {
+            scenarioCount += 1;
+            scenario = headerMatch[2].trim() || void 0;
+            section = 'scenario';
+            continue;
+        }
+        const stepMatch = line.match(stepKeywordPattern);
+        if (stepMatch) {
+            const step = {
+                keyword: normalizeStepKeyword(stepMatch[1]),
+                text: stepMatch[2].trim(),
+                lineNumber
+            };
+            if ('scenario' === section) scenarioSteps.push(step);
+            else anonymousSteps.push(step);
+            continue;
+        }
+        throw new Error(`runGherkinScenario does not support content at line ${lineNumber}: ${line}`);
+    }
+    if (scenarioCount > 1) throw new Error(`runGherkinScenario expects exactly one Scenario, but found ${scenarioCount}.`);
+    if (1 === scenarioCount && anonymousSteps.length > 0) throw new Error('runGherkinScenario cannot mix anonymous steps with a Scenario block.');
+    const rawSteps = 1 === scenarioCount ? scenarioSteps : anonymousSteps;
+    if (0 === rawSteps.length) throw new Error('runGherkinScenario requires at least one Gherkin step.');
+    return {
+        scenario,
+        steps: resolveSteps(rawSteps)
+    };
+};
+const buildStepPrompt = (step)=>{
+    if ('Given' === step.effectiveKeyword) return `Set up this precondition: ${step.text}`;
+    if ('When' === step.effectiveKeyword) return `Perform this user action: ${step.text}`;
+    return `Verify that ${step.text}`;
+};
+const describeStepExecution = (step, action)=>{
+    const keywordMapping = step.keyword === step.effectiveKeyword ? step.keyword : `${step.keyword} as ${step.effectiveKeyword}`;
+    if ('Given' === step.effectiveKeyword) return `setting up the precondition (${keywordMapping} -> ${action})`;
+    if ('When' === step.effectiveKeyword) return `performing the user action (${keywordMapping} -> ${action})`;
+    return `verifying the expected result (${keywordMapping} -> ${action})`;
+};
+const describeErrorCause = (error)=>{
+    if (error instanceof Error && error.message) return ` Original error: ${error.message}`;
+    return '';
+};
+const runGherkinScenario = async (agent, scenarioText, opt)=>{
+    const parsedScenario = parseGherkinScenario(scenarioText);
+    const aiActOptions = {
+        ...opt,
+        cacheable: false
+    };
+    for (const step of parsedScenario.steps){
+        throwIfAborted(opt?.abortSignal);
+        const action = 'Then' === step.effectiveKeyword ? 'aiAssert' : 'aiAct';
+        const prompt = buildStepPrompt(step);
+        try {
+            if ('aiAct' === action) await agent.aiAct(prompt, aiActOptions);
+            else if (opt?.context || opt?.abortSignal) await agent.aiAssert(prompt, void 0, {
+                context: opt.context,
+                abortSignal: opt.abortSignal
+            });
+            else await agent.aiAssert(prompt);
+        } catch (error) {
+            throw new Error(`runGherkinScenario failed while ${describeStepExecution(step, action)} at line ${step.lineNumber}: ${step.keyword} ${step.text}.${describeErrorCause(error)}`, {
+                cause: error
+            });
+        }
+    }
+};
+export { parseGherkinScenario, runGherkinScenario };
+
+//# sourceMappingURL=run-gherkin-scenario.mjs.map

package/dist/es/agent/run-gherkin-scenario.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/run-gherkin-scenario.mjs","sources":["../../../src/agent/run-gherkin-scenario.ts"],"sourcesContent":["import type { AiActOptions } from './agent';\n\nexport type GherkinStepKeyword = 'Given' | 'When' | 'Then' | 'And' | 'But';\n\ntype RunGherkinScenarioStepAction = 'aiAct' | 'aiAssert';\n\nexport type RunGherkinScenarioOptions = AiActOptions & {\n  context?: string;\n};\n\ntype GherkinPrimaryKeyword = 'Given' | 'When' | 'Then';\n\ntype ParsedGherkinStep = {\n  keyword: GherkinStepKeyword;\n  effectiveKeyword: GherkinPrimaryKeyword;\n  text: string;\n  lineNumber: number;\n};\n\ntype RawGherkinStep = {\n  keyword: GherkinStepKeyword;\n  text: string;\n  lineNumber: number;\n};\n\ntype ParsedGherkinScenario = {\n  scenario?: string;\n  steps: ParsedGherkinStep[];\n};\n\ntype GherkinScenarioAgent = {\n  aiAct: (taskPrompt: string, opt?: AiActOptions) => Promise<unknown>;\n  aiAssert: (\n    assertion: string,\n    msg?: string,\n    opt?: { context?: string; abortSignal?: AbortSignal },\n  ) => Promise<unknown>;\n};\n\nconst stepKeywordPattern = /^(Given|When|Then|And|But)\\s+(.+)$/i;\nconst headerPattern = /^(Scenario):\\s*(.*)$/i;\nconst unsupportedHeaderPattern =\n  /^(Feature|Background|Scenario Outline|Scenario Template|Examples|Rule):/i;\n\nconst normalizeStepKeyword = (keyword: string): GherkinStepKeyword => {\n  const lowerKeyword = keyword.toLowerCase();\n  if (lowerKeyword === 'given') return 'Given';\n  if (lowerKeyword === 'when') return 'When';\n  if (lowerKeyword === 'then') return 'Then';\n  if (lowerKeyword === 'and') return 'And';\n  return 'But';\n};\n\nconst isPrimaryKeyword = (\n  keyword: GherkinStepKeyword,\n): keyword is GherkinPrimaryKeyword => {\n  return keyword === 'Given' || keyword === 'When' || keyword === 'Then';\n};\n\nconst resolveSteps = (steps: RawGherkinStep[]): ParsedGherkinStep[] => {\n  let previousPrimaryKeyword: GherkinPrimaryKeyword | undefined;\n\n  return steps.map((step) => {\n    const effectiveKeyword = isPrimaryKeyword(step.keyword)\n      ? step.keyword\n      : previousPrimaryKeyword;\n\n    if (!effectiveKeyword) {\n      throw new Error(\n        `runGherkinScenario cannot resolve \"${step.keyword}\" at line ${step.lineNumber}; use Given, When, or Then before ${step.keyword}.`,\n      );\n    }\n\n    previousPrimaryKeyword = effectiveKeyword;\n\n    return {\n      ...step,\n      effectiveKeyword,\n    };\n  });\n};\n\nconst throwIfAborted = (abortSignal: AbortSignal | undefined) => {\n  if (!abortSignal?.aborted) {\n    return;\n  }\n\n  if (typeof abortSignal.throwIfAborted === 'function') {\n    abortSignal.throwIfAborted();\n  }\n\n  throw new Error(\n    `runGherkinScenario aborted: ${abortSignal.reason || 'signal already aborted'}`,\n  );\n};\n\nexport const parseGherkinScenario = (\n  scenarioText: string,\n): ParsedGherkinScenario => {\n  const lines = scenarioText.split(/\\r?\\n/);\n  const scenarioSteps: RawGherkinStep[] = [];\n  const anonymousSteps: RawGherkinStep[] = [];\n  let scenario: string | undefined;\n  let scenarioCount = 0;\n  let section: 'prelude' | 'scenario' = 'prelude';\n\n  for (const [lineIndex, rawLine] of lines.entries()) {\n    const lineNumber = lineIndex + 1;\n    const line = rawLine.trim();\n\n    if (!line || line.startsWith('#') || line.startsWith('@')) {\n      continue;\n    }\n\n    if (line === '\"\"\"' || line === \"'''\") {\n      throw new Error(\n        `runGherkinScenario does not support doc strings; found one at line ${lineNumber}.`,\n      );\n    }\n\n    if (line.startsWith('|')) {\n      throw new Error(\n        `runGherkinScenario does not support data tables; found one at line ${lineNumber}.`,\n      );\n    }\n\n    if (unsupportedHeaderPattern.test(line)) {\n      throw new Error(\n        `runGherkinScenario does not support \"${line}\" at line ${lineNumber}.`,\n      );\n    }\n\n    const headerMatch = line.match(headerPattern);\n    if (headerMatch) {\n      scenarioCount += 1;\n      scenario = headerMatch[2].trim() || undefined;\n      section = 'scenario';\n      continue;\n    }\n\n    const stepMatch = line.match(stepKeywordPattern);\n    if (stepMatch) {\n      const step = {\n        keyword: normalizeStepKeyword(stepMatch[1]),\n        text: stepMatch[2].trim(),\n        lineNumber,\n      };\n\n      if (section === 'scenario') {\n        scenarioSteps.push(step);\n      } else {\n        anonymousSteps.push(step);\n      }\n\n      continue;\n    }\n\n    throw new Error(\n      `runGherkinScenario does not support content at line ${lineNumber}: ${line}`,\n    );\n  }\n\n  if (scenarioCount > 1) {\n    throw new Error(\n      `runGherkinScenario expects exactly one Scenario, but found ${scenarioCount}.`,\n    );\n  }\n\n  if (scenarioCount === 1 && anonymousSteps.length > 0) {\n    throw new Error(\n      'runGherkinScenario cannot mix anonymous steps with a Scenario block.',\n    );\n  }\n\n  const rawSteps = scenarioCount === 1 ? scenarioSteps : anonymousSteps;\n\n  if (rawSteps.length === 0) {\n    throw new Error('runGherkinScenario requires at least one Gherkin step.');\n  }\n\n  return {\n    scenario,\n    steps: resolveSteps(rawSteps),\n  };\n};\n\nconst buildStepPrompt = (step: ParsedGherkinStep) => {\n  if (step.effectiveKeyword === 'Given') {\n    return `Set up this precondition: ${step.text}`;\n  }\n\n  if (step.effectiveKeyword === 'When') {\n    return `Perform this user action: ${step.text}`;\n  }\n\n  return `Verify that ${step.text}`;\n};\n\nconst describeStepExecution = (\n  step: ParsedGherkinStep,\n  action: RunGherkinScenarioStepAction,\n) => {\n  const keywordMapping =\n    step.keyword === step.effectiveKeyword\n      ? step.keyword\n      : `${step.keyword} as ${step.effectiveKeyword}`;\n\n  if (step.effectiveKeyword === 'Given') {\n    return `setting up the precondition (${keywordMapping} -> ${action})`;\n  }\n\n  if (step.effectiveKeyword === 'When') {\n    return `performing the user action (${keywordMapping} -> ${action})`;\n  }\n\n  return `verifying the expected result (${keywordMapping} -> ${action})`;\n};\n\nconst describeErrorCause = (error: unknown) => {\n  if (error instanceof Error && error.message) {\n    return ` Original error: ${error.message}`;\n  }\n\n  return '';\n};\n\nexport const runGherkinScenario = async (\n  agent: GherkinScenarioAgent,\n  scenarioText: string,\n  opt?: RunGherkinScenarioOptions,\n): Promise<void> => {\n  const parsedScenario = parseGherkinScenario(scenarioText);\n  const aiActOptions: AiActOptions = {\n    ...opt,\n    cacheable: false,\n  };\n\n  for (const step of parsedScenario.steps) {\n    throwIfAborted(opt?.abortSignal);\n\n    const action: RunGherkinScenarioStepAction =\n      step.effectiveKeyword === 'Then' ? 'aiAssert' : 'aiAct';\n    const prompt = buildStepPrompt(step);\n\n    try {\n      if (action === 'aiAct') {\n        await agent.aiAct(prompt, aiActOptions);\n      } else if (opt?.context || opt?.abortSignal) {\n        await agent.aiAssert(prompt, undefined, {\n          context: opt.context,\n          abortSignal: opt.abortSignal,\n        });\n      } else {\n        await agent.aiAssert(prompt);\n      }\n    } catch (error) {\n      throw new Error(\n        `runGherkinScenario failed while ${describeStepExecution(\n          step,\n          action,\n        )} at line ${step.lineNumber}: ${step.keyword} ${step.text}.${describeErrorCause(\n          error,\n        )}`,\n        {\n          cause: error,\n        },\n      );\n    }\n  }\n};\n"],"names":["stepKeywordPattern","headerPattern","unsupportedHeaderPattern","normalizeStepKeyword","keyword","lowerKeyword","isPrimaryKeyword","resolveSteps","steps","previousPrimaryKeyword","step","effectiveKeyword","Error","throwIfAborted","abortSignal","parseGherkinScenario","scenarioText","lines","scenarioSteps","anonymousSteps","scenario","scenarioCount","section","lineIndex","rawLine","lineNumber","line","headerMatch","undefined","stepMatch","rawSteps","buildStepPrompt","describeStepExecution","action","keywordMapping","describeErrorCause","error","runGherkinScenario","agent","opt","parsedScenario","aiActOptions","prompt"],"mappings":"AAuCA,MAAMA,qBAAqB;AAC3B,MAAMC,gBAAgB;AACtB,MAAMC,2BACJ;AAEF,MAAMC,uBAAuB,CAACC;IAC5B,MAAMC,eAAeD,QAAQ,WAAW;IACxC,IAAIC,AAAiB,YAAjBA,cAA0B,OAAO;IACrC,IAAIA,AAAiB,WAAjBA,cAAyB,OAAO;IACpC,IAAIA,AAAiB,WAAjBA,cAAyB,OAAO;IACpC,IAAIA,AAAiB,UAAjBA,cAAwB,OAAO;IACnC,OAAO;AACT;AAEA,MAAMC,mBAAmB,CACvBF,UAEOA,AAAY,YAAZA,WAAuBA,AAAY,WAAZA,WAAsBA,AAAY,WAAZA;AAGtD,MAAMG,eAAe,CAACC;IACpB,IAAIC;IAEJ,OAAOD,MAAM,GAAG,CAAC,CAACE;QAChB,MAAMC,mBAAmBL,iBAAiBI,KAAK,OAAO,IAClDA,KAAK,OAAO,GACZD;QAEJ,IAAI,CAACE,kBACH,MAAM,IAAIC,MACR,CAAC,mCAAmC,EAAEF,KAAK,OAAO,CAAC,UAAU,EAAEA,KAAK,UAAU,CAAC,kCAAkC,EAAEA,KAAK,OAAO,CAAC,CAAC,CAAC;QAItID,yBAAyBE;QAEzB,OAAO;YACL,GAAGD,IAAI;YACPC;QACF;IACF;AACF;AAEA,MAAME,iBAAiB,CAACC;IACtB,IAAI,CAACA,aAAa,SAChB;IAGF,IAAI,AAAsC,cAAtC,OAAOA,YAAY,cAAc,EACnCA,YAAY,cAAc;IAG5B,MAAM,IAAIF,MACR,CAAC,4BAA4B,EAAEE,YAAY,MAAM,IAAI,0BAA0B;AAEnF;AAEO,MAAMC,uBAAuB,CAClCC;IAEA,MAAMC,QAAQD,aAAa,KAAK,CAAC;IACjC,MAAME,gBAAkC,EAAE;IAC1C,MAAMC,iBAAmC,EAAE;IAC3C,IAAIC;IACJ,IAAIC,gBAAgB;IACpB,IAAIC,UAAkC;IAEtC,KAAK,MAAM,CAACC,WAAWC,QAAQ,IAAIP,MAAM,OAAO,GAAI;QAClD,MAAMQ,aAAaF,YAAY;QAC/B,MAAMG,OAAOF,QAAQ,IAAI;QAEzB,IAAI,CAACE,QAAQA,KAAK,UAAU,CAAC,QAAQA,KAAK,UAAU,CAAC,MACnD;QAGF,IAAIA,AAAS,UAATA,QAAkBA,AAAS,UAATA,MACpB,MAAM,IAAId,MACR,CAAC,mEAAmE,EAAEa,WAAW,CAAC,CAAC;QAIvF,IAAIC,KAAK,UAAU,CAAC,MAClB,MAAM,IAAId,MACR,CAAC,mEAAmE,EAAEa,WAAW,CAAC,CAAC;QAIvF,IAAIvB,yBAAyB,IAAI,CAACwB,OAChC,MAAM,IAAId,MACR,CAAC,qCAAqC,EAAEc,KAAK,UAAU,EAAED,WAAW,CAAC,CAAC;QAI1E,MAAME,cAAcD,KAAK,KAAK,CAACzB;QAC/B,IAAI0B,aAAa;YACfN,iBAAiB;YACjBD,WAAWO,WAAW,CAAC,EAAE,CAAC,IAAI,MAAMC;YACpCN,UAAU;YACV;QACF;QAEA,MAAMO,YAAYH,KAAK,KAAK,CAAC1B;QAC7B,IAAI6B,WAAW;YACb,MAAMnB,OAAO;gBACX,SAASP,qBAAqB0B,SAAS,CAAC,EAAE;gBAC1C,MAAMA,SAAS,CAAC,EAAE,CAAC,IAAI;gBACvBJ;YACF;YAEA,IAAIH,AAAY,eAAZA,SACFJ,cAAc,IAAI,CAACR;iBAEnBS,eAAe,IAAI,CAACT;YAGtB;QACF;QAEA,MAAM,IAAIE,MACR,CAAC,oDAAoD,EAAEa,WAAW,EAAE,EAAEC,MAAM;IAEhF;IAEA,IAAIL,gBAAgB,GAClB,MAAM,IAAIT,MACR,CAAC,2DAA2D,EAAES,cAAc,CAAC,CAAC;IAIlF,IAAIA,AAAkB,MAAlBA,iBAAuBF,eAAe,MAAM,GAAG,GACjD,MAAM,IAAIP,MACR;IAIJ,MAAMkB,WAAWT,AAAkB,MAAlBA,gBAAsBH,gBAAgBC;IAEvD,IAAIW,AAAoB,MAApBA,SAAS,MAAM,EACjB,MAAM,IAAIlB,MAAM;IAGlB,OAAO;QACLQ;QACA,OAAOb,aAAauB;IACtB;AACF;AAEA,MAAMC,kBAAkB,CAACrB;IACvB,IAAIA,AAA0B,YAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,0BAA0B,EAAEA,KAAK,IAAI,EAAE;IAGjD,IAAIA,AAA0B,WAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,0BAA0B,EAAEA,KAAK,IAAI,EAAE;IAGjD,OAAO,CAAC,YAAY,EAAEA,KAAK,IAAI,EAAE;AACnC;AAEA,MAAMsB,wBAAwB,CAC5BtB,MACAuB;IAEA,MAAMC,iBACJxB,KAAK,OAAO,KAAKA,KAAK,gBAAgB,GAClCA,KAAK,OAAO,GACZ,GAAGA,KAAK,OAAO,CAAC,IAAI,EAAEA,KAAK,gBAAgB,EAAE;IAEnD,IAAIA,AAA0B,YAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,6BAA6B,EAAEwB,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;IAGvE,IAAIvB,AAA0B,WAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,4BAA4B,EAAEwB,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;IAGtE,OAAO,CAAC,+BAA+B,EAAEC,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;AACzE;AAEA,MAAME,qBAAqB,CAACC;IAC1B,IAAIA,iBAAiBxB,SAASwB,MAAM,OAAO,EACzC,OAAO,CAAC,iBAAiB,EAAEA,MAAM,OAAO,EAAE;IAG5C,OAAO;AACT;AAEO,MAAMC,qBAAqB,OAChCC,OACAtB,cACAuB;IAEA,MAAMC,iBAAiBzB,qBAAqBC;IAC5C,MAAMyB,eAA6B;QACjC,GAAGF,GAAG;QACN,WAAW;IACb;IAEA,KAAK,MAAM7B,QAAQ8B,eAAe,KAAK,CAAE;QACvC3B,eAAe0B,KAAK;QAEpB,MAAMN,SACJvB,AAA0B,WAA1BA,KAAK,gBAAgB,GAAc,aAAa;QAClD,MAAMgC,SAASX,gBAAgBrB;QAE/B,IAAI;YACF,IAAIuB,AAAW,YAAXA,QACF,MAAMK,MAAM,KAAK,CAACI,QAAQD;iBACrB,IAAIF,KAAK,WAAWA,KAAK,aAC9B,MAAMD,MAAM,QAAQ,CAACI,QAAQd,QAAW;gBACtC,SAASW,IAAI,OAAO;gBACpB,aAAaA,IAAI,WAAW;YAC9B;iBAEA,MAAMD,MAAM,QAAQ,CAACI;QAEzB,EAAE,OAAON,OAAO;YACd,MAAM,IAAIxB,MACR,CAAC,gCAAgC,EAAEoB,sBACjCtB,MACAuB,QACA,SAAS,EAAEvB,KAAK,UAAU,CAAC,EAAE,EAAEA,KAAK,OAAO,CAAC,CAAC,EAAEA,KAAK,IAAI,CAAC,CAAC,EAAEyB,mBAC5DC,QACC,EACH;gBACE,OAAOA;YACT;QAEJ;IACF;AACF"}

package/dist/es/agent/task-builder.mjs

@@ -1,4 +1,5 @@
 import { findAllMidsceneLocatorField, parseActionParam } from "../ai-model/index.mjs";
+import { findActionInActionSpaceOrThrow } from "../common.mjs";
 import { setTimingFieldOnce } from "../task-timing.mjs";
 import { ServiceError } from "../types.mjs";
 import { sleep } from "../utils.mjs";
@@ -101,11 +102,10 @@ class TaskBuilder {
     async handleActionPlan(plan, context) {
         const planType = plan.type;
         const actionSpace = this.actionSpace;
-        const action = actionSpace.find((item)=>item.name === planType);
+        const action = findActionInActionSpaceOrThrow(planType, actionSpace);
         const param = plan.param;
-        if (!action) throw new Error(`Action type '${planType}' not found`);
-        const locateFields = action ? findAllMidsceneLocatorField(action.paramSchema) : [];
-        const requiredLocateFields = action ? findAllMidsceneLocatorField(action.paramSchema, true) : [];
+        const locateFields = findAllMidsceneLocatorField(action.paramSchema);
+        const requiredLocateFields = findAllMidsceneLocatorField(action.paramSchema, true);
         locateFields.forEach((field)=>{
             if (param[field]) {
                 const locatePlan = locatePlanForLocate(param[field]);

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifPlanLocateParamHasLocatedPixelBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [\n    element.center?.[0],\n    element.center?.[1],\n    element.rect?.left,\n    element.rect?.top,\n    element.rect?.width,\n    element.rect?.height,\n  ];\n  if (\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  if (element.rect.width <= 0 || element.rect.height <= 0) {\n    return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelBbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(\n          param,\n        )\n          ? param\n          : undefined;\n\n        assert(\n          param?.prompt || paramWithLocatedPixelBbox,\n          `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelBbox\n          ? matchElementFromPlan(paramWithLocatedPixelBbox)\n          : undefined;\n\n        // from locatedPixelBbox (direct plan hit)\n        // when deepLocate is enabled, locatedPixelBbox should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = param.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;AAoCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMvB,QAAQkB,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI/B,KAAK,CAAC+B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaF,oBAAoBL,KAAK,CAAC+B,MAAM;gBACnD3C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC+B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEjC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,oBAAoB,EAAEyB,qCAAqChC,KAAK,CAAC+B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtC1B,YACAP,KAAK,CAAC+B,MAAM,EACZhB,SACA,CAACmB;oBACClC,KAAK,CAAC+B,MAAM,GAAGG;gBACjB;gBAEFnB,QAAQ,KAAK,CAAC,IAAI,CAACkB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3EnC,MAAM,CAAC,OAAO,EAAE2C,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMa,OAKF;YACF,MAAM;YACN,SAASb;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOlB,OAAOqC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCjD,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEqC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACEnC,KAAK,CAAC+B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAiB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBhB,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMiB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCtD,MACE,CAAC,8DAA8D,EAAEqC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDZ,MACE,CAAC,2DAA2D,EAAEqC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAgB,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAES,wBAAwB,EAAE,GAAGR;gBACrC,IAAIQ,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQgD,iBAAiBhD,OAAOyB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOE,OAAY;oBACnB,MAAM,IAAItB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEwB,MAAM,OAAO,CAAC,cAAc,EAAEnD,KAAK,SAAS,CAACE,QAAQ,EACtG;wBAAE,OAAOiD;oBAAM;gBAEnB;gBAGFT,mBAAmBF,QAAQ;gBAE3BlD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAMyB,WAAWzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM0B,eAAe,MAAMD,SAASlD,OAAOqC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BlD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW0B;gBAE/CX,mBAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI2B,mBAAmB,GACrB,MAAMT,MAAMS;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpChE,MACE,CAAC,6DAA6D,EAAEqC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEzB;wBACpDZ,MACE,CAAC,0DAA0D,EAAEqC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACqB;IACrB;IAEQ,iBACNlB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOlB,OAAOqC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMqB,4BAA4B1B,qCAChChC,SAEEA,QACAI;gBAEJ+B,OACEnC,OAAO,UAAU0D,2BACjB,CAAC,iEAAiE,EAAE5D,KAAK,SAAS,CAChFE,QACC;gBAGL,IAAI,CAACuC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEQ,wBAAwB,EAAE,GAAGR;gBAErC,IAAIQ,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIgC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA1B,KAAK,KAAK,GAAG2B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD1B,KAAK,UAAU,GAAG0B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,qBAAqBP,6BACrBtD;gBAKJ,MAAM8D,kBAAkBlE,MAAM,UAAU,GACpCI,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDnE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFoE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACpE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMqE,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACArB,2BAEF,AAAwB,YAAxB,OAAO/C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BI;gBAEJ,MAAMoE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAczE,MAAM,MAAM;gBAChC,MAAM0E,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAzE,MAAM,SAAS;gBAIvB,MAAM8E,mBAAmBF,yBACrBG,oCACEH,wBACA7B,4BAEF3C;gBAEJ,MAAM4E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM3C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACmC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF3B,mBAAmBF,QAAQ;oBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC5D,OACA;wBACE,SAASuC;wBACTyB;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,cACnBrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRT,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM5C,UACJwE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIvF,SAAS;oBACX,MAAMyF,uBAAuB1F,2BAA2BC;oBACxD,IAAIyF,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,aAAaC,sBAAsBxB;wBAE/C,MAAM,IAAIhC,MAAMwD;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B9F,iBAC/BoF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACE3F,WACA,IAAI,CAAC,SAAS,IACd,CAACsF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7CpF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIsF,gBAAkC5F,QAAQ,MAAM;oBACpD,IAAIqD,AAA6B,MAA7BA,0BAAgC;wBAClCuC,gBAAgB;4BACdC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;4BAC/BwC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;yBAChC;wBACD3D,MACE,8DACAM,QAAQ,MAAM,EACd4F;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOtF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,cAAcW;oBAChB;oBAEF,IAAIrB,iBAAiBkG,UAAU;wBAC7BpG,MACE,uCACAqF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACEtF,MACE,yDACAqF;gBAGN,EAAE,OAAOxB,OAAO;oBACd7D,MAAM,mCAAmC6D;gBAC3C;qBAEA7D,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIuB,aACR,CAAC,oBAAoB,EAAElF,MAAM,MAAM,EAAE,EACrC2D;oBAGJ,MAAM,IAAIhC,MAAM,CAAC,mBAAmB,EAAE3B,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIyF;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOzF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIgF,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF/B,WAAW5D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK6C,UAAU,aAAa;wBAC9B;oBACF;oBACAkD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IApkBA,YAAY,EACViC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTpE,WAAW,EACXqE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGpE;QACnB,IAAI,CAAC,eAAe,GAAGqE;IACzB;AAyjBF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { findActionInActionSpaceOrThrow } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifPlanLocateParamHasLocatedPixelBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [\n    element.center?.[0],\n    element.center?.[1],\n    element.rect?.left,\n    element.rect?.top,\n    element.rect?.width,\n    element.rect?.height,\n  ];\n  if (\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  if (element.rect.width <= 0 || element.rect.height <= 0) {\n    return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    const requiredLocateFields = findAllMidsceneLocatorField(\n      action.paramSchema,\n      true,\n    );\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelBbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(\n          param,\n        )\n          ? param\n          : undefined;\n\n        assert(\n          param?.prompt || paramWithLocatedPixelBbox,\n          `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelBbox\n          ? matchElementFromPlan(paramWithLocatedPixelBbox)\n          : undefined;\n\n        // from locatedPixelBbox (direct plan hit)\n        // when deepLocate is enabled, locatedPixelBbox should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = param.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","Error","shrunkShotToLogicalRatio","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnE,MAAMI,uBAAuBD,4BAC3BH,OAAO,WAAW,EAClB;QAGFE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EAAE;gBAGhB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,oBAAoB,EAAEwB,qCAAqC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEP,UAAU;gBAE3EnC,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;YACjE;QACF;QAEA,MAAMY,OAKF;YACF,MAAM;YACN,SAASZ;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtChD,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEoC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACElC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEP,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAgB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBf,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMgB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCrD,MACE,CAAC,8DAA8D,EAAEqC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDZ,MACE,CAAC,2DAA2D,EAAEqC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAe,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,wCAAwC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEU,wBAAwB,EAAE,GAAGT;gBACrC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIrB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQgD,iBAAiBhD,OAAOyB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOE,OAAY;oBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEwB,MAAM,OAAO,CAAC,cAAc,EAAEnD,KAAK,SAAS,CAACE,QAAQ,EACtG;wBAAE,OAAOiD;oBAAM;gBAEnB;gBAGFV,mBAAmBF,QAAQ;gBAE3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAMyB,WAAWzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM0B,eAAe,MAAMD,SAASlD,OAAOoC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW0B;gBAE/CZ,mBAAmBF,QAAQ;gBAE3B,MAAMe,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI2B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpChE,MACE,CAAC,6DAA6D,EAAEqC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEzB;wBACpDZ,MACE,CAAC,0DAA0D,EAAEqC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOkB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,uCAAuC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQc;gBACV;YACF;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACoB;IACrB;IAEQ,iBACNjB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMsB,4BAA4B3B,qCAChC/B,SAEEA,QACAI;gBAEJ8B,OACElC,OAAO,UAAU0D,2BACjB,CAAC,iEAAiE,EAAE5D,KAAK,SAAS,CAChFE,QACC;gBAGL,IAAI,CAACsC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAES,wBAAwB,EAAE,GAAGT;gBAErC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIa;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb3B,KAAK,GAAG,GAAG;wBACT2B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA3B,KAAK,KAAK,GAAG4B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD3B,KAAK,UAAU,GAAG2B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBACjB3B,KAAK,eAAe,GAAG2B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB3B,KAAK,iBAAiB,GAAG2B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,qBAAqBP,6BACrBtD;gBAKJ,MAAM8D,kBAAkBlE,MAAM,UAAU,GACpCI,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDnE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFoE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACpE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMqE,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACArB,2BAEF,AAAwB,YAAxB,OAAO/C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BI;gBAEJ,MAAMoE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAczE,MAAM,MAAM;gBAChC,MAAM0E,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAzE,MAAM,SAAS;gBAIvB,MAAM8E,mBAAmBF,yBACrBG,oCACEH,wBACA7B,4BAEF3C;gBAEJ,MAAM4E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM5C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF5B,mBAAmBF,QAAQ;oBAC3BuB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC5D,OACA;wBACE,SAASsC;wBACT0B;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,cACnBrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRV,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJwE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIvF,SAAS;oBACX,MAAMyF,uBAAuB1F,2BAA2BC;oBACxD,IAAIyF,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,aAAaC,sBAAsBxB;wBAE/C,MAAM,IAAIb,MAAMqC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B9F,iBAC/BoF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACE3F,WACA,IAAI,CAAC,SAAS,IACd,CAACsF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7CpF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIsF,gBAAkC5F,QAAQ,MAAM;oBACpD,IAAIqD,AAA6B,MAA7BA,0BAAgC;wBAClCuC,gBAAgB;4BACdC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;4BAC/BwC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;yBAChC;wBACD3D,MACE,8DACAM,QAAQ,MAAM,EACd4F;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOtF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,cAAcW;oBAChB;oBAEF,IAAIrB,iBAAiBkG,UAAU;wBAC7BpG,MACE,uCACAqF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACEtF,MACE,yDACAqF;gBAGN,EAAE,OAAOxB,OAAO;oBACd7D,MAAM,mCAAmC6D;gBAC3C;qBAEA7D,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIuB,aACR,CAAC,oBAAoB,EAAElF,MAAM,MAAM,EAAE,EACrC2D;oBAGJ,MAAM,IAAIb,MAAM,CAAC,mBAAmB,EAAE9C,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIyF;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOzF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIgF,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF/B,WAAW5D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK4C,UAAU,aAAa;wBAC9B;oBACF;oBACAmD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IA/jBA,YAAY,EACViC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTpE,WAAW,EACXqE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGpE;QACnB,IAAI,CAAC,eAAe,GAAGqE;IACzB;AAojBF"}

package/dist/es/agent/tasks.mjs

@@ -133,10 +133,6 @@ class TaskExecutor {
         }
     }
     async runAction(userPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal, reportOptions) {
-        if (deepLocate && !planningModel.adapter.planning.supportsActionDeepLocate) {
-            warnLog(`The "deepLocate" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`);
-            deepLocate = false;
-        }
         const conversationHistory = new ConversationHistory();
         const session = this.createExecutionSession(taskTitleStr(reportOptions?.type || 'Act', reportOptions?.prompt || userPromptToString(userPrompt)));
         const runner = session.getRunner();
@@ -287,7 +283,7 @@ class TaskExecutor {
             runner
         };
     }
-    createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt) {
+    createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt, executionOptions) {
         const queryTask = {
             type: 'Insight',
             subType: type,
@@ -336,7 +332,7 @@ class TaskExecutor {
                     extraPageDescription = await descriptionOfTree(tree, 200, false, opt?.domIncluded === 'visible-only');
                 }
                 try {
-                    extractResult = await this.service.extract(demandInput, modelRuntime, opt, extraPageDescription, multimodalPrompt, uiContext);
+                    extractResult = await this.service.extract(demandInput, modelRuntime, opt, extraPageDescription, multimodalPrompt, uiContext, executionOptions);
                 } catch (error) {
                     if (error instanceof ServiceError) applyDump(error.dump);
                     throw error;
@@ -363,9 +359,9 @@ class TaskExecutor {
         };
         return queryTask;
     }
-    async createTypeQueryExecution(type, demand, modelRuntime, opt, multimodalPrompt) {
+    async createTypeQueryExecution(type, demand, modelRuntime, opt, multimodalPrompt, executionOptions) {
         const session = this.createExecutionSession(taskTitleStr(type, 'string' == typeof demand ? demand : JSON.stringify(demand)));
-        const queryTask = await this.createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt);
+        const queryTask = await this.createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt, executionOptions);
         const runner = session.getRunner();
         const result = await session.appendAndRun(queryTask);
         if (!result) throw new Error('result of taskExecutor.flush() is undefined in function createTypeQueryTask');

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeviceAction,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  PlanningLocateParam,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nexport type ActionReportOptions = {\n  type?: TaskTitleType;\n  prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap each task's planning feedback so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. This is the\n// single place that truncates feedback before it is sent to the model; action\n// implementations should hand over the untruncated value.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(feedback: string): string {\n  if (feedback.length <= maxPlanningFeedbackLength) {\n    return feedback;\n  }\n\n  return `${feedback.slice(0, maxPlanningFeedbackLength)}\n...[truncated, ${feedback.length - maxPlanningFeedbackLength} more characters]`;\n}\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[]; uiContext?: UIContext },\n  ) {\n    return new ExecutionSession(\n      title,\n      () =>\n        options?.uiContext\n          ? Promise.resolve(options.uiContext)\n          : Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Set the pending feedback message consumed by the next planning round.\n   * The message is always prefixed with the current time. When a body is\n   * provided it is appended after the timestamp; otherwise only the time\n   * context is recorded. This is the single entry point for writing\n   * `pendingFeedbackMessage` so the time prefix stays consistent.\n   */\n  private setPendingFeedbackMessage(\n    conversationHistory: ConversationHistory,\n    timeString: string,\n    body?: string,\n  ) {\n    conversationHistory.pendingFeedbackMessage = body\n      ? `Time: ${timeString}, ${body}`\n      : `Current time: ${timeString}`;\n  }\n\n  /**\n   * Collect feedback produced by executed tasks for the next planning round.\n   * Returns undefined when no task reported feedback.\n   */\n  private collectPlanningFeedback(tasks: ExecutionTask[]): string | undefined {\n    const feedbackMessages = tasks.flatMap(({ planningFeedback }) =>\n      planningFeedback ? [truncatePlanningFeedback(planningFeedback)] : [],\n    );\n    return feedbackMessages.length > 0\n      ? feedbackMessages.join('\\n\\n')\n      : undefined;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(\n    userInstruction: TUserPrompt,\n    yamlString: string,\n    reportOptions?: ActionReportOptions,\n  ) {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userInstruction),\n      ),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n        ...(reportOptions?.prompt\n          ? { userInstructionDisplay: reportOptions.prompt }\n          : {}),\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: { uiContext?: UIContext },\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title, options);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      planningModel,\n      defaultModel,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        includeLocateInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n        reportOptions,\n      );\n    });\n  }\n\n  /**\n   * Called when the task is about to replan. Marks every cache-hit locate task\n   * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n   * did not finish the task, so the element each cache hit produced is suspect.\n   * The upcoming re-locate of the same prompt then replaces the bad entry in\n   * place instead of appending a duplicate that would re-poison the cache on the\n   * next run (#2529).\n   *\n   * Marking a locate that was actually fine is harmless: the step is only ever\n   * replaced if the same prompt is located again (i.e. the step is redone),\n   * which does not happen for a locate that already succeeded.\n   */\n  private invalidateFailedCacheHitLocates(\n    runner: TaskRunner,\n    fromIndex: number,\n  ) {\n    if (!this.taskCache) {\n      return;\n    }\n    for (let i = fromIndex; i < runner.tasks.length; i++) {\n      const task = runner.tasks[i];\n      if (\n        task.type === 'Planning' &&\n        task.subType === 'Locate' &&\n        task.hitBy?.from === 'Cache'\n      ) {\n        const prompt = (task.param as PlanningLocateParam | undefined)?.prompt;\n        if (prompt) {\n          this.taskCache.markLocateCacheStale(prompt);\n        }\n      }\n    }\n  }\n\n  private async runAction(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    if (\n      deepLocate &&\n      !planningModel.adapter.planning.supportsActionDeepLocate\n    ) {\n      warnLog(\n        `The \"deepLocate\" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`,\n      );\n      deepLocate = false;\n    }\n\n    const conversationHistory = new ConversationHistory();\n\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userPrompt),\n      ),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n\n    if (abortSignal?.aborted) {\n      return session.appendErrorPlan(\n        `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n      );\n    }\n    const referenceImageMessages = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(userPrompt),\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            ...(reportOptions?.prompt\n              ? { userInstructionDisplay: reportOptions.prompt }\n              : {}),\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const timing = executorContext.task.timing;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : genericXmlPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: uiContext,\n                actionContext: param.aiActContext,\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                referenceImageMessages,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                  rawChoiceMessage: planError.rawChoiceMessage,\n                };\n              }\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              rawChoiceMessage,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n              rawChoiceMessage,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = uiContext;\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              assert(\n                false,\n                `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`,\n              );\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Capture the time context for the next planning call before running.\n      const initialTimeString = await this.getTimeString();\n\n      const taskCountBeforeRun = runner.tasks.length;\n      try {\n        await session.appendAndRun(executables.tasks);\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          initialTimeString,\n          this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n        );\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          timeString,\n          `Error executing running tasks: ${error?.message || String(error)}`,\n        );\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // We are about to replan, which means the batch we just ran did not finish\n      // the task. Any locate task in that batch that was served from cache\n      // produced an element that failed to complete the step (the action threw,\n      // or it clicked the wrong element and the goal was not reached). Mark those\n      // cache entries stale so the re-locate of the same prompt replaces them in\n      // place instead of appending a poisoning duplicate that would be matched\n      // first on the next run (#2529).\n      this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelRuntime,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = await getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","TaskExecutor","title","options","ExecutionSession","Promise","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","undefined","format","error","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","task","param","executorContext","uiContext","assert","runner","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","fromIndex","i","prompt","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","errorCountInOnePlanningLoop","outputString","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","subGoalStatus","memoriesStatus","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","rawChoiceMessage","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","taskCountBeforeRun","String","Error","errorMsg","type","demand","modelRuntime","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAM9C,MAAMC,4BAA4B;AAElC,SAASC,yBAAyBC,QAAgB;IAChD,IAAIA,SAAS,MAAM,IAAIF,2BACrB,OAAOE;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGF,2BAA2B;eAC1C,EAAEE,SAAS,MAAM,GAAGF,0BAA0B,iBAAiB,CAAC;AAC/E;AAIO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAAiE,EACjE;QACA,OAAO,IAAIC,iBACTF,OACA,IACEC,SAAS,YACLE,QAAQ,OAAO,CAACF,QAAQ,SAAS,IACjCE,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASQ,0BACNG,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CAAC,CAAC,EAAEE,gBAAgB,EAAE,GAC1DA,mBAAmB;gBAACZ,yBAAyBY;aAAkB,GAAG,EAAE;QAEtE,OAAOD,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBE;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdlB,QACE,CAAC,gEAAgE,EAAEkB,OAAO;QAE9E;aAEAlB,QACE;QAKN,OAAOmB,sBAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACa,OAAOC,eAAeC,cAAcf;IACpE;IAEA,MAAM,uBACJgB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,mBAAmBL;QAIhD,MAAMM,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLN;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMU,SAASR,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACG;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJ5B,KAAa,EACbc,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAAmC,EACT;QAC1B,MAAMmB,UAAU,IAAI,CAAC,sBAAsB,CAACpB,OAAOC;QACnD,MAAM,EAAEM,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDO,OACAC,eACAC;QAEF,MAAMY,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACb;QAC1C,MAAM,EAAEuB,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAF;QACF;IACF;IAEA,MAAM,OACJG,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,OAAOsB,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAhB,eACAC,cACAgB,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC,aACArB;IAGN;IAcQ,gCACNS,MAAkB,EAClBc,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAIf,OAAO,KAAK,CAAC,MAAM,EAAEe,IAAK;YACpD,MAAMpB,OAAOK,OAAO,KAAK,CAACe,EAAE;YAC5B,IACEpB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMqB,SAAUrB,KAAK,KAAK,EAAsC;gBAChE,IAAIqB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZb,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,IACEoB,cACA,CAACxB,cAAc,OAAO,CAAC,QAAQ,CAAC,wBAAwB,EACxD;YACArB,QACE,CAAC,mGAAmG,EAAEqB,cAAc,MAAM,CAAC,WAAW,IAAI,UAAU,sBAAsB,CAAC;YAE7KwB,aAAa;QACf;QAEA,MAAMnC,sBAAsB,IAAIyC;QAEhC,MAAMzB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,mBAAmBS;QAGhD,MAAMH,SAASR,QAAQ,SAAS;QAEhC,IAAI0B,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJb,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DR,OACEqB,AAAyBtC,WAAzBsC,sBACA;QAGF,IAAIC,8BAA8B;QAClC,IAAIC;QAEJ,IAAIV,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMW,yBAAyB,MAAMC,+BACnCC,6BAA6BtB;QAI/B,MAAO,KAAM;YAEX,IAAIS,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMc,gBAAgBlD,oBAAoB,cAAc,MAAMM;YAG9D,MAAM6C,iBAAiBnD,oBAAoB,cAAc,MAAMM;YAE/D,MAAMmB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjB,GAAIZ,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACNc;oBACAG;oBACAC;oBACA,GAAIiB,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO/B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM8B,SAAS/B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAMgC,cAAc,IAAI,CAAC,cAAc;oBACvCjE,MACE,sCACAiE,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD/B,OAAOgC,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJ9C,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrC+C;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAASrC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjCiC;4BACA,cAAc1C;4BACdX;4BACA4B;4BACAI;4BACAC;4BACAc;4BACAX;wBACF;oBACF,EAAE,OAAOyB,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7CzC,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,gBAC3BF,UAAU,KAAK,EACf;4BAEFxC,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAawC,UAAU,WAAW;gCAClC,kBAAkBA,UAAU,gBAAgB;4BAC9C;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBR,QAAQ;oBAC7B;oBACAhE,MAAM,cAAc4E,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN5D,KAAK,EACL6D,KAAK,EACLC,WAAW,EACXC,gBAAgB,EAChBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGjB;oBACJb,eAAe4B;oBAEfrD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCiD;wBACAC;oBACF;oBACAlD,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,gBAAgBM,OAAO;oBACpDhD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGmD;oBACzCnD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS4C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQe;wBACR,wBAAwBf,WAAW,sBAAsB;wBACzDgB;wBACAC;oBACF;oBACAvD,gBAAgB,SAAS,GAAGC;oBAE5BC,OAAO,CAACf,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAE2D,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBACFlD,OACE,OACA,CAAC,aAAa,EAAEmD,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAalC,QAAQ;YAG3B,MAAMf,QAAQiD,YAAY,WAAW,EAAE;YACvChB,SAAS,IAAI,IAAKgB,YAAY,YAAY,EAAE;YAE5C,IAAIkB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CnE,OACAC,eACAC,cACA;oBACEkB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO5B,OAAO;gBACd,OAAOQ,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAER,MAAM,SAAS,EAAEwD,KAAK,SAAS,CAC5EtD,QACC;YAEP;YACA,IAAIV,oBAAoB,sBAAsB,EAC5CwD,QAAQ,IAAI,CACV,8FACAxD,oBAAoB,sBAAsB;YAK9C,MAAM8E,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqBvD,OAAO,KAAK,CAAC,MAAM;YAC9C,IAAI;gBACF,MAAMR,QAAQ,YAAY,CAAC6D,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5B7E,qBACA8E,mBACA,IAAI,CAAC,uBAAuB,CAACtD,OAAO,KAAK,CAAC,KAAK,CAACuD;YAEpD,EAAE,OAAOvE,OAAY;gBAEnBqC;gBACA,MAAM5C,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEO,OAAO,WAAWwE,OAAOxE,QAAQ;gBAErEpB,MACE,yFACAoB,iBAAiByE,QAAQzE,MAAM,OAAO,GAAGwE,OAAOxE,QAChD,6CACAqC;YAEJ;YAEA,IAAIA,8BAA8BtD,uCAChC,OAAOyB,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACuB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAACnC,QAAQuD;YAG7C,EAAErC;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMsC,WAAW,CAAC,UAAU,EAAEtC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO5B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAClF,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACN0C;gBACA,QAAQG;YACV;YACAtB;QACF;IACF;IAEQ,oBACN2D,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOhE,OAAOqE;gBACtB,MAAM,EAAEtE,IAAI,EAAE,GAAGsE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZzE,KAAK,GAAG,GAAG;wBACTyE;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACAzE,KAAK,KAAK,GAAG4C,gBAAgB6B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjBzE,KAAK,iBAAiB,GAAGyE,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMtE,YAAYmE,YAAY,SAAS;gBACvClE,OAAOD,WAAW;gBAElB,MAAMuE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF,OAAO,IAAIS,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DlG,MAAM;oBACN,MAAM+G,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,cACAC,KACAY,sBACAX,kBACAjE;gBAEJ,EAAE,OAAOd,OAAO;oBACd,IAAIA,iBAAiB6F,cACnBV,UAAUnF,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE8F,IAAI,EAAEpC,OAAO,EAAE0B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAKzF,QAC1BiG,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAWhG,QAC1BiG,eAAgBD,KAAa,MAAM;qBAEnC/E,OAAO,OAAO;gBAKpB,IAAI4D,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCpF,KAAK,OAAO,GAAG+C;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQqC;oBACR,KAAKb;oBACLxB;gBACF;YACF;QACF;QAEA,OAAOsB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,cACAC,KACAC;QAGF,MAAM/D,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACwE;QAE1C,IAAI,CAAC/D,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEwC,OAAO,EAAE,GAAGzC;QAE5B,OAAO;YACLC;YACAwC;YACA1C;QACF;IACF;IAEA,MAAM,QACJgF,SAAsB,EACtBlB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMzF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW0F;QAE1B,MAAMnF,SAASR,QAAQ,SAAS;QAChC,MAAM,EACJ4F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEAzF,OAAOiF,WAAW;QAClBjF,OAAOqF,WAAW;QAClBrF,OAAOsF,iBAAiB;QAExBtF,OACEsF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,cACA4B,mBACA1B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACwE;YAO3C,IAAI/D,QAAQ,QACV,OAAO;gBACL,QAAQnB;gBACRkB;YACF;YAGF6F,eACE5F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEgF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMtD,UAAU,CAAC,kBAAkB,EAAE2C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGvD;oBAAQ;iBAAE,EAC9DmB,cACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAM1G,QAAQ,YAAY,CAAC0G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO1G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEqG,cAAc;IACnE;IAx0BA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AA4yBF;AAEO,eAAexF,gBACpBsF,iBAAoC,EACpCzF,iBAAuC,EACvCoB,MAAwB;IAExB,IAAI,CAACpB,mBAAmB,QACtB,OAAOoB;IAGT,IAAI,CAACqE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC9F;IACvB;IAEA,MAAM,EAAE+F,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMtG,SAAS,MAAM6B;QAErB,MAAM9C,QAAQ,MAAM0H;QACpB,IAAI1H,OACF,MAAMA;QAER,OAAOiB;IACT,SAAU;QACRwG;IACF;AACF"}
+{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeviceAction,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  PlanningLocateParam,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nexport type ActionReportOptions = {\n  type?: TaskTitleType;\n  prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap each task's planning feedback so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. This is the\n// single place that truncates feedback before it is sent to the model; action\n// implementations should hand over the untruncated value.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(feedback: string): string {\n  if (feedback.length <= maxPlanningFeedbackLength) {\n    return feedback;\n  }\n\n  return `${feedback.slice(0, maxPlanningFeedbackLength)}\n...[truncated, ${feedback.length - maxPlanningFeedbackLength} more characters]`;\n}\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[]; uiContext?: UIContext },\n  ) {\n    return new ExecutionSession(\n      title,\n      () =>\n        options?.uiContext\n          ? Promise.resolve(options.uiContext)\n          : Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Set the pending feedback message consumed by the next planning round.\n   * The message is always prefixed with the current time. When a body is\n   * provided it is appended after the timestamp; otherwise only the time\n   * context is recorded. This is the single entry point for writing\n   * `pendingFeedbackMessage` so the time prefix stays consistent.\n   */\n  private setPendingFeedbackMessage(\n    conversationHistory: ConversationHistory,\n    timeString: string,\n    body?: string,\n  ) {\n    conversationHistory.pendingFeedbackMessage = body\n      ? `Time: ${timeString}, ${body}`\n      : `Current time: ${timeString}`;\n  }\n\n  /**\n   * Collect feedback produced by executed tasks for the next planning round.\n   * Returns undefined when no task reported feedback.\n   */\n  private collectPlanningFeedback(tasks: ExecutionTask[]): string | undefined {\n    const feedbackMessages = tasks.flatMap(({ planningFeedback }) =>\n      planningFeedback ? [truncatePlanningFeedback(planningFeedback)] : [],\n    );\n    return feedbackMessages.length > 0\n      ? feedbackMessages.join('\\n\\n')\n      : undefined;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(\n    userInstruction: TUserPrompt,\n    yamlString: string,\n    reportOptions?: ActionReportOptions,\n  ) {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userInstruction),\n      ),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n        ...(reportOptions?.prompt\n          ? { userInstructionDisplay: reportOptions.prompt }\n          : {}),\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: { uiContext?: UIContext },\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title, options);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      planningModel,\n      defaultModel,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        includeLocateInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n        reportOptions,\n      );\n    });\n  }\n\n  /**\n   * Called when the task is about to replan. Marks every cache-hit locate task\n   * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n   * did not finish the task, so the element each cache hit produced is suspect.\n   * The upcoming re-locate of the same prompt then replaces the bad entry in\n   * place instead of appending a duplicate that would re-poison the cache on the\n   * next run (#2529).\n   *\n   * Marking a locate that was actually fine is harmless: the step is only ever\n   * replaced if the same prompt is located again (i.e. the step is redone),\n   * which does not happen for a locate that already succeeded.\n   */\n  private invalidateFailedCacheHitLocates(\n    runner: TaskRunner,\n    fromIndex: number,\n  ) {\n    if (!this.taskCache) {\n      return;\n    }\n    for (let i = fromIndex; i < runner.tasks.length; i++) {\n      const task = runner.tasks[i];\n      if (\n        task.type === 'Planning' &&\n        task.subType === 'Locate' &&\n        task.hitBy?.from === 'Cache'\n      ) {\n        const prompt = (task.param as PlanningLocateParam | undefined)?.prompt;\n        if (prompt) {\n          this.taskCache.markLocateCacheStale(prompt);\n        }\n      }\n    }\n  }\n\n  private async runAction(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    const conversationHistory = new ConversationHistory();\n\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userPrompt),\n      ),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n\n    if (abortSignal?.aborted) {\n      return session.appendErrorPlan(\n        `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n      );\n    }\n    const referenceImageMessages = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(userPrompt),\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            ...(reportOptions?.prompt\n              ? { userInstructionDisplay: reportOptions.prompt }\n              : {}),\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const timing = executorContext.task.timing;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : genericXmlPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: uiContext,\n                actionContext: param.aiActContext,\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                referenceImageMessages,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                  rawChoiceMessage: planError.rawChoiceMessage,\n                };\n              }\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              rawChoiceMessage,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n              rawChoiceMessage,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = uiContext;\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              assert(\n                false,\n                `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`,\n              );\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Capture the time context for the next planning call before running.\n      const initialTimeString = await this.getTimeString();\n\n      const taskCountBeforeRun = runner.tasks.length;\n      try {\n        await session.appendAndRun(executables.tasks);\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          initialTimeString,\n          this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n        );\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          timeString,\n          `Error executing running tasks: ${error?.message || String(error)}`,\n        );\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // We are about to replan, which means the batch we just ran did not finish\n      // the task. Any locate task in that batch that was served from cache\n      // produced an element that failed to complete the step (the action threw,\n      // or it clicked the wrong element and the goal was not reached). Mark those\n      // cache entries stale so the re-locate of the same prompt replaces them in\n      // place instead of appending a poisoning duplicate that would be matched\n      // first on the next run (#2529).\n      this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n            executionOptions,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelRuntime,\n      opt,\n      multimodalPrompt,\n      executionOptions,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = await getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","TaskExecutor","title","options","ExecutionSession","Promise","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","undefined","format","error","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","task","param","executorContext","uiContext","assert","runner","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","fromIndex","i","prompt","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","errorCountInOnePlanningLoop","outputString","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","subGoalStatus","memoriesStatus","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","rawChoiceMessage","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","taskCountBeforeRun","String","Error","errorMsg","type","demand","modelRuntime","opt","multimodalPrompt","executionOptions","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAM9C,MAAMC,4BAA4B;AAElC,SAASC,yBAAyBC,QAAgB;IAChD,IAAIA,SAAS,MAAM,IAAIF,2BACrB,OAAOE;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGF,2BAA2B;eAC1C,EAAEE,SAAS,MAAM,GAAGF,0BAA0B,iBAAiB,CAAC;AAC/E;AAIO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAAiE,EACjE;QACA,OAAO,IAAIC,iBACTF,OACA,IACEC,SAAS,YACLE,QAAQ,OAAO,CAACF,QAAQ,SAAS,IACjCE,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASQ,0BACNG,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CAAC,CAAC,EAAEE,gBAAgB,EAAE,GAC1DA,mBAAmB;gBAACZ,yBAAyBY;aAAkB,GAAG,EAAE;QAEtE,OAAOD,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBE;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdlB,QACE,CAAC,gEAAgE,EAAEkB,OAAO;QAE9E;aAEAlB,QACE;QAKN,OAAOmB,sBAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACa,OAAOC,eAAeC,cAAcf;IACpE;IAEA,MAAM,uBACJgB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,mBAAmBL;QAIhD,MAAMM,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLN;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMU,SAASR,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACG;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJ5B,KAAa,EACbc,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAAmC,EACT;QAC1B,MAAMmB,UAAU,IAAI,CAAC,sBAAsB,CAACpB,OAAOC;QACnD,MAAM,EAAEM,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDO,OACAC,eACAC;QAEF,MAAMY,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACb;QAC1C,MAAM,EAAEuB,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAF;QACF;IACF;IAEA,MAAM,OACJG,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,OAAOsB,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAhB,eACAC,cACAgB,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC,aACArB;IAGN;IAcQ,gCACNS,MAAkB,EAClBc,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAIf,OAAO,KAAK,CAAC,MAAM,EAAEe,IAAK;YACpD,MAAMpB,OAAOK,OAAO,KAAK,CAACe,EAAE;YAC5B,IACEpB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMqB,SAAUrB,KAAK,KAAK,EAAsC;gBAChE,IAAIqB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZb,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,MAAMf,sBAAsB,IAAIyC;QAEhC,MAAMzB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,mBAAmBS;QAGhD,MAAMH,SAASR,QAAQ,SAAS;QAEhC,IAAI0B,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJb,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DR,OACEqB,AAAyBtC,WAAzBsC,sBACA;QAGF,IAAIC,8BAA8B;QAClC,IAAIC;QAEJ,IAAIV,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMW,yBAAyB,MAAMC,+BACnCC,6BAA6BtB;QAI/B,MAAO,KAAM;YAEX,IAAIS,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMc,gBAAgBlD,oBAAoB,cAAc,MAAMM;YAG9D,MAAM6C,iBAAiBnD,oBAAoB,cAAc,MAAMM;YAE/D,MAAMmB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjB,GAAIZ,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACNc;oBACAG;oBACAC;oBACA,GAAIiB,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO/B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM8B,SAAS/B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAMgC,cAAc,IAAI,CAAC,cAAc;oBACvCjE,MACE,sCACAiE,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD/B,OAAOgC,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJ9C,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrC+C;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAASrC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjCiC;4BACA,cAAc1C;4BACdX;4BACA4B;4BACAI;4BACAC;4BACAc;4BACAX;wBACF;oBACF,EAAE,OAAOyB,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7CzC,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,gBAC3BF,UAAU,KAAK,EACf;4BAEFxC,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAawC,UAAU,WAAW;gCAClC,kBAAkBA,UAAU,gBAAgB;4BAC9C;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBR,QAAQ;oBAC7B;oBACAhE,MAAM,cAAc4E,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN5D,KAAK,EACL6D,KAAK,EACLC,WAAW,EACXC,gBAAgB,EAChBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGjB;oBACJb,eAAe4B;oBAEfrD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCiD;wBACAC;oBACF;oBACAlD,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,gBAAgBM,OAAO;oBACpDhD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGmD;oBACzCnD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS4C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQe;wBACR,wBAAwBf,WAAW,sBAAsB;wBACzDgB;wBACAC;oBACF;oBACAvD,gBAAgB,SAAS,GAAGC;oBAE5BC,OAAO,CAACf,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAE2D,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBACFlD,OACE,OACA,CAAC,aAAa,EAAEmD,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAalC,QAAQ;YAG3B,MAAMf,QAAQiD,YAAY,WAAW,EAAE;YACvChB,SAAS,IAAI,IAAKgB,YAAY,YAAY,EAAE;YAE5C,IAAIkB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CnE,OACAC,eACAC,cACA;oBACEkB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO5B,OAAO;gBACd,OAAOQ,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAER,MAAM,SAAS,EAAEwD,KAAK,SAAS,CAC5EtD,QACC;YAEP;YACA,IAAIV,oBAAoB,sBAAsB,EAC5CwD,QAAQ,IAAI,CACV,8FACAxD,oBAAoB,sBAAsB;YAK9C,MAAM8E,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqBvD,OAAO,KAAK,CAAC,MAAM;YAC9C,IAAI;gBACF,MAAMR,QAAQ,YAAY,CAAC6D,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5B7E,qBACA8E,mBACA,IAAI,CAAC,uBAAuB,CAACtD,OAAO,KAAK,CAAC,KAAK,CAACuD;YAEpD,EAAE,OAAOvE,OAAY;gBAEnBqC;gBACA,MAAM5C,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEO,OAAO,WAAWwE,OAAOxE,QAAQ;gBAErEpB,MACE,yFACAoB,iBAAiByE,QAAQzE,MAAM,OAAO,GAAGwE,OAAOxE,QAChD,6CACAqC;YAEJ;YAEA,IAAIA,8BAA8BtD,uCAChC,OAAOyB,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACuB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAACnC,QAAQuD;YAG7C,EAAErC;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMsC,WAAW,CAAC,UAAU,EAAEtC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO5B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAClF,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACN0C;gBACA,QAAQG;YACV;YACAtB;QACF;IACF;IAEQ,oBACN2D,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAEC,EACD;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASN;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOhE,OAAOsE;gBACtB,MAAM,EAAEvE,IAAI,EAAE,GAAGuE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ1E,KAAK,GAAG,GAAG;wBACT0E;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA1E,KAAK,KAAK,GAAG4C,gBAAgB8B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1E,KAAK,iBAAiB,GAAG0E,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMvE,YAAYoE,YAAY,SAAS;gBACvCnE,OAAOD,WAAW;gBAElB,MAAMwE,mBAAmBX,AAAS,YAATA;gBACzB,IAAIY,cAAcX;gBAClB,IAAIY,cAAc;gBAClB,IAAIF,oBAAqBX,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEa,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF,OAAO,IAAIU,kBAAkB;oBAC3BE,cAAcb;oBACdY,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF;gBAEA,IAAIc;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIb,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DlG,MAAM;oBACN,MAAMgH,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAd,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAV,cACAC,KACAa,sBACAZ,kBACAjE,WACAkE;gBAEJ,EAAE,OAAOhF,OAAO;oBACd,IAAIA,iBAAiB8F,cACnBV,UAAUpF,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE+F,IAAI,EAAErC,OAAO,EAAE2B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAIpB,AAAS,cAATA,MAEPqB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAK1F,QAC1BkG,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAWjG,QAC1BkG,eAAgBD,KAAa,MAAM;qBAEnChF,OAAO,OAAO;gBAKpB,IAAI4D,AAAS,aAATA,QAAqB,CAACqB,cAAc;oBACtCrF,KAAK,OAAO,GAAG+C;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQsC;oBACR,KAAKb;oBACLzB;gBACF;YACF;QACF;QAEA,OAAOuB;IACT;IACA,MAAM,yBACJN,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAEC,EAC4B;QAC7B,MAAMxE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMK,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CN,MACAC,QACAC,cACAC,KACAC,kBACAC;QAGF,MAAMhE,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACyE;QAE1C,IAAI,CAAChE,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEwC,OAAO,EAAE,GAAGzC;QAE5B,OAAO;YACLC;YACAwC;YACA1C;QACF;IACF;IAEA,MAAM,QACJiF,SAAsB,EACtBnB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEqB,UAAU,EAAEnB,gBAAgB,EAAE,GAAGoB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM1F,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW2F;QAE1B,MAAMpF,SAASR,QAAQ,SAAS;QAChC,MAAM,EACJ6F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG3B;QACJ,MAAM4B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA1F,OAAOkF,WAAW;QAClBlF,OAAOsF,WAAW;QAClBtF,OAAOuF,iBAAiB;QAExBvF,OACEuF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACArB,cACA6B,mBACA3B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACyE;YAO3C,IAAIhE,QAAQ,QACV,OAAO;gBACL,QAAQnB;gBACRkB;YACF;YAGF8F,eACE7F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEiF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMvD,UAAU,CAAC,kBAAkB,EAAE4C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGxD;oBAAQ;iBAAE,EAC9DmB,cACAA;gBAEF,IAAIsC,UAAU,CAAC,EAAE,EACf,MAAM3G,QAAQ,YAAY,CAAC2G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO3G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEsG,cAAc;IACnE;IAt0BA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AA0yBF;AAEO,eAAezF,gBACpBuF,iBAAoC,EACpC1F,iBAAuC,EACvCoB,MAAwB;IAExB,IAAI,CAACpB,mBAAmB,QACtB,OAAOoB;IAGT,IAAI,CAACsE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI3C,MACR,CAAC,gCAAgC,EAAE2C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC/F;IACvB;IAEA,MAAM,EAAEgG,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMvG,SAAS,MAAM6B;QAErB,MAAM9C,QAAQ,MAAM2H;QACpB,IAAI3H,OACF,MAAMA;QAER,OAAOiB;IACT,SAAU;QACRyG;IACF;AACF"}