@midscene/core 1.12.3 → 1.12.4-beta-20260902093735.0

package/dist/es/agent/prompt-context.mjs

@@ -1,25 +1,61 @@
+const INTERNAL_AI_CONTEXT_METADATA_KEY = '_internalAIContextMetadata';
+const resolvedAIContextFromOptions = (options)=>{
+    if (options?.context === void 0) return;
+    return {
+        value: options.context,
+        metadata: options[INTERNAL_AI_CONTEXT_METADATA_KEY] ?? {
+            source: 'call'
+        }
+    };
+};
+const renderAIContext = (resolvedContext, workflowHistory)=>{
+    const blocks = [];
+    const context = resolvedContext?.value.trim();
+    if (context && resolvedContext) {
+        const metadata = resolvedContext.metadata;
+        if ('default' === metadata.source) blocks.push(`<GLOBAL_CONTEXT>\n${context}\n</GLOBAL_CONTEXT>`);
+        else {
+            const apiAttribute = 'api' === metadata.source ? ` api="${metadata.apiName}"` : '';
+            blocks.push(`<REQUEST_CONTEXT source="${metadata.source}"${apiAttribute}>\n${context}\n</REQUEST_CONTEXT>`);
+        }
+    }
+    const history = workflowHistory?.trim();
+    if (history) blocks.push(`<WORKFLOW_HISTORY read_only="true">\n${history}\n</WORKFLOW_HISTORY>`);
+    if (blocks.length > 0) return blocks.join('\n\n');
+    return void 0 !== resolvedContext || void 0 !== workflowHistory ? '' : void 0;
+};
+const mergeAIContexts = (...contexts)=>{
+    const definedContexts = contexts.filter((context)=>void 0 !== context);
+    if (0 === definedContexts.length) return;
+    return definedContexts.filter(Boolean).join('\n\n');
+};
 const buildPromptWithContext = (prompt, context)=>{
     const trimmedContext = context?.trim();
     if (!trimmedContext) return prompt;
     const promptText = 'string' == typeof prompt ? prompt : prompt.prompt;
-    const promptWithContext = `Context for this request:\n${trimmedContext}\n\n${promptText}`;
+    const promptWithContext = `${trimmedContext}\n\n${promptText}`;
     if ('string' == typeof prompt) return promptWithContext;
     return {
         ...prompt,
         prompt: promptWithContext
     };
 };
-const buildLocatePromptWithContext = (prompt, context)=>{
-    const trimmedContext = context?.trim();
-    if (!trimmedContext) return prompt;
+const buildLocatePromptWithContext = (prompt, context, metadata)=>{
+    const renderedContext = renderAIContext(void 0 === context ? void 0 : {
+        value: context,
+        metadata: metadata ?? {
+            source: 'call'
+        }
+    });
+    if (!renderedContext) return prompt;
     const promptText = 'string' == typeof prompt ? prompt : prompt.prompt;
-    const promptWithContext = `<CONTEXT>\n${trimmedContext}\n</CONTEXT>\n\n<LOCATE_TARGET>\n${promptText}\n</LOCATE_TARGET>`;
+    const promptWithContext = `${renderedContext}\n\n<LOCATE_TARGET>\n${promptText}\n</LOCATE_TARGET>`;
     if ('string' == typeof prompt) return promptWithContext;
     return {
         ...prompt,
         prompt: promptWithContext
     };
 };
-export { buildLocatePromptWithContext, buildPromptWithContext };
+export { INTERNAL_AI_CONTEXT_METADATA_KEY, buildLocatePromptWithContext, buildPromptWithContext, mergeAIContexts, renderAIContext, resolvedAIContextFromOptions };
 
 //# sourceMappingURL=prompt-context.mjs.map

package/dist/es/agent/prompt-context.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/prompt-context.mjs","sources":["../../../src/agent/prompt-context.ts"],"sourcesContent":["import type { TUserPrompt } from '@/ai-model';\n\nexport const buildPromptWithContext = (\n  prompt: TUserPrompt,\n  context: string | undefined,\n): TUserPrompt => {\n  const trimmedContext = context?.trim();\n  if (!trimmedContext) {\n    return prompt;\n  }\n\n  const promptText = typeof prompt === 'string' ? prompt : prompt.prompt;\n  const promptWithContext = `Context for this request:\\n${trimmedContext}\\n\\n${promptText}`;\n\n  if (typeof prompt === 'string') {\n    return promptWithContext;\n  }\n\n  return {\n    ...prompt,\n    prompt: promptWithContext,\n  };\n};\n\nexport const buildLocatePromptWithContext = (\n  prompt: TUserPrompt,\n  context: string | undefined,\n): TUserPrompt => {\n  const trimmedContext = context?.trim();\n  if (!trimmedContext) {\n    return prompt;\n  }\n\n  const promptText = typeof prompt === 'string' ? prompt : prompt.prompt;\n  const promptWithContext = `<CONTEXT>\\n${trimmedContext}\\n</CONTEXT>\\n\\n<LOCATE_TARGET>\\n${promptText}\\n</LOCATE_TARGET>`;\n\n  if (typeof prompt === 'string') {\n    return promptWithContext;\n  }\n\n  return {\n    ...prompt,\n    prompt: promptWithContext,\n  };\n};\n"],"names":["buildPromptWithContext","prompt","context","trimmedContext","promptText","promptWithContext","buildLocatePromptWithContext"],"mappings":"AAEO,MAAMA,yBAAyB,CACpCC,QACAC;IAEA,MAAMC,iBAAiBD,SAAS;IAChC,IAAI,CAACC,gBACH,OAAOF;IAGT,MAAMG,aAAa,AAAkB,YAAlB,OAAOH,SAAsBA,SAASA,OAAO,MAAM;IACtE,MAAMI,oBAAoB,CAAC,2BAA2B,EAAEF,eAAe,IAAI,EAAEC,YAAY;IAEzF,IAAI,AAAkB,YAAlB,OAAOH,QACT,OAAOI;IAGT,OAAO;QACL,GAAGJ,MAAM;QACT,QAAQI;IACV;AACF;AAEO,MAAMC,+BAA+B,CAC1CL,QACAC;IAEA,MAAMC,iBAAiBD,SAAS;IAChC,IAAI,CAACC,gBACH,OAAOF;IAGT,MAAMG,aAAa,AAAkB,YAAlB,OAAOH,SAAsBA,SAASA,OAAO,MAAM;IACtE,MAAMI,oBAAoB,CAAC,WAAW,EAAEF,eAAe,iCAAiC,EAAEC,WAAW,kBAAkB,CAAC;IAExH,IAAI,AAAkB,YAAlB,OAAOH,QACT,OAAOI;IAGT,OAAO;QACL,GAAGJ,MAAM;QACT,QAAQI;IACV;AACF"}
+{"version":3,"file":"agent/prompt-context.mjs","sources":["../../../src/agent/prompt-context.ts"],"sourcesContent":["import type { TUserPrompt } from '@/ai-model';\nimport type { AiApiName } from '@midscene/shared/agent-tools/agent-context';\n\nexport type AIContextMetadata =\n  | { source: 'default' }\n  | { source: 'api'; apiName: AiApiName }\n  | { source: 'call' };\n\nexport interface ResolvedAIContext {\n  value: string;\n  metadata: AIContextMetadata;\n}\n\nexport const INTERNAL_AI_CONTEXT_METADATA_KEY =\n  '_internalAIContextMetadata' as const;\n\nexport interface InternalAIContextOptions {\n  [INTERNAL_AI_CONTEXT_METADATA_KEY]?: AIContextMetadata;\n  /** Framework-owned workflow history appended after the selected context. */\n  _internalAdditionalContext?: string;\n}\n\nexport const resolvedAIContextFromOptions = (\n  options:\n    | ({ context?: string } & Partial<InternalAIContextOptions>)\n    | undefined,\n): ResolvedAIContext | undefined => {\n  if (options?.context === undefined) {\n    return undefined;\n  }\n\n  return {\n    value: options.context,\n    metadata: options[INTERNAL_AI_CONTEXT_METADATA_KEY] ?? { source: 'call' },\n  };\n};\n\n/** Render the selected user context and framework history as separate blocks. */\nexport const renderAIContext = (\n  resolvedContext: ResolvedAIContext | undefined,\n  workflowHistory?: string,\n): string | undefined => {\n  const blocks: string[] = [];\n  const context = resolvedContext?.value.trim();\n\n  if (context && resolvedContext) {\n    const metadata = resolvedContext.metadata;\n    if (metadata.source === 'default') {\n      blocks.push(`<GLOBAL_CONTEXT>\\n${context}\\n</GLOBAL_CONTEXT>`);\n    } else {\n      const apiAttribute =\n        metadata.source === 'api' ? ` api=\"${metadata.apiName}\"` : '';\n      blocks.push(\n        `<REQUEST_CONTEXT source=\"${metadata.source}\"${apiAttribute}>\\n${context}\\n</REQUEST_CONTEXT>`,\n      );\n    }\n  }\n\n  const history = workflowHistory?.trim();\n  if (history) {\n    blocks.push(\n      `<WORKFLOW_HISTORY read_only=\"true\">\\n${history}\\n</WORKFLOW_HISTORY>`,\n    );\n  }\n\n  if (blocks.length > 0) {\n    return blocks.join('\\n\\n');\n  }\n\n  return resolvedContext !== undefined || workflowHistory !== undefined\n    ? ''\n    : undefined;\n};\n\n/** Explicitly concatenate context fragments in the order provided. */\nexport const mergeAIContexts = (\n  ...contexts: Array<string | undefined>\n): string | undefined => {\n  const definedContexts = contexts.filter(\n    (context): context is string => context !== undefined,\n  );\n  if (definedContexts.length === 0) {\n    return undefined;\n  }\n\n  return definedContexts.filter(Boolean).join('\\n\\n');\n};\n\nexport const buildPromptWithContext = (\n  prompt: TUserPrompt,\n  context: string | undefined,\n): TUserPrompt => {\n  const trimmedContext = context?.trim();\n  if (!trimmedContext) {\n    return prompt;\n  }\n\n  const promptText = typeof prompt === 'string' ? prompt : prompt.prompt;\n  const promptWithContext = `${trimmedContext}\\n\\n${promptText}`;\n\n  if (typeof prompt === 'string') {\n    return promptWithContext;\n  }\n\n  return {\n    ...prompt,\n    prompt: promptWithContext,\n  };\n};\n\nexport const buildLocatePromptWithContext = (\n  prompt: TUserPrompt,\n  context: string | undefined,\n  metadata?: AIContextMetadata,\n): TUserPrompt => {\n  const renderedContext = renderAIContext(\n    context === undefined\n      ? undefined\n      : { value: context, metadata: metadata ?? { source: 'call' } },\n  );\n  if (!renderedContext) {\n    return prompt;\n  }\n\n  const promptText = typeof prompt === 'string' ? prompt : prompt.prompt;\n  const promptWithContext = `${renderedContext}\\n\\n<LOCATE_TARGET>\\n${promptText}\\n</LOCATE_TARGET>`;\n\n  if (typeof prompt === 'string') {\n    return promptWithContext;\n  }\n\n  return {\n    ...prompt,\n    prompt: promptWithContext,\n  };\n};\n"],"names":["INTERNAL_AI_CONTEXT_METADATA_KEY","resolvedAIContextFromOptions","options","undefined","renderAIContext","resolvedContext","workflowHistory","blocks","context","metadata","apiAttribute","history","mergeAIContexts","contexts","definedContexts","Boolean","buildPromptWithContext","prompt","trimmedContext","promptText","promptWithContext","buildLocatePromptWithContext","renderedContext"],"mappings":"AAaO,MAAMA,mCACX;AAQK,MAAMC,+BAA+B,CAC1CC;IAIA,IAAIA,SAAS,YAAYC,QACvB;IAGF,OAAO;QACL,OAAOD,QAAQ,OAAO;QACtB,UAAUA,OAAO,CAACF,iCAAiC,IAAI;YAAE,QAAQ;QAAO;IAC1E;AACF;AAGO,MAAMI,kBAAkB,CAC7BC,iBACAC;IAEA,MAAMC,SAAmB,EAAE;IAC3B,MAAMC,UAAUH,iBAAiB,MAAM;IAEvC,IAAIG,WAAWH,iBAAiB;QAC9B,MAAMI,WAAWJ,gBAAgB,QAAQ;QACzC,IAAII,AAAoB,cAApBA,SAAS,MAAM,EACjBF,OAAO,IAAI,CAAC,CAAC,kBAAkB,EAAEC,QAAQ,mBAAmB,CAAC;aACxD;YACL,MAAME,eACJD,AAAoB,UAApBA,SAAS,MAAM,GAAa,CAAC,MAAM,EAAEA,SAAS,OAAO,CAAC,CAAC,CAAC,GAAG;YAC7DF,OAAO,IAAI,CACT,CAAC,yBAAyB,EAAEE,SAAS,MAAM,CAAC,CAAC,EAAEC,aAAa,GAAG,EAAEF,QAAQ,oBAAoB,CAAC;QAElG;IACF;IAEA,MAAMG,UAAUL,iBAAiB;IACjC,IAAIK,SACFJ,OAAO,IAAI,CACT,CAAC,qCAAqC,EAAEI,QAAQ,qBAAqB,CAAC;IAI1E,IAAIJ,OAAO,MAAM,GAAG,GAClB,OAAOA,OAAO,IAAI,CAAC;IAGrB,OAAOF,AAAoBF,WAApBE,mBAAiCC,AAAoBH,WAApBG,kBACpC,KACAH;AACN;AAGO,MAAMS,kBAAkB,CAC7B,GAAGC;IAEH,MAAMC,kBAAkBD,SAAS,MAAM,CACrC,CAACL,UAA+BA,AAAYL,WAAZK;IAElC,IAAIM,AAA2B,MAA3BA,gBAAgB,MAAM,EACxB;IAGF,OAAOA,gBAAgB,MAAM,CAACC,SAAS,IAAI,CAAC;AAC9C;AAEO,MAAMC,yBAAyB,CACpCC,QACAT;IAEA,MAAMU,iBAAiBV,SAAS;IAChC,IAAI,CAACU,gBACH,OAAOD;IAGT,MAAME,aAAa,AAAkB,YAAlB,OAAOF,SAAsBA,SAASA,OAAO,MAAM;IACtE,MAAMG,oBAAoB,GAAGF,eAAe,IAAI,EAAEC,YAAY;IAE9D,IAAI,AAAkB,YAAlB,OAAOF,QACT,OAAOG;IAGT,OAAO;QACL,GAAGH,MAAM;QACT,QAAQG;IACV;AACF;AAEO,MAAMC,+BAA+B,CAC1CJ,QACAT,SACAC;IAEA,MAAMa,kBAAkBlB,gBACtBI,AAAYL,WAAZK,UACIL,SACA;QAAE,OAAOK;QAAS,UAAUC,YAAY;YAAE,QAAQ;QAAO;IAAE;IAEjE,IAAI,CAACa,iBACH,OAAOL;IAGT,MAAME,aAAa,AAAkB,YAAlB,OAAOF,SAAsBA,SAASA,OAAO,MAAM;IACtE,MAAMG,oBAAoB,GAAGE,gBAAgB,qBAAqB,EAAEH,WAAW,kBAAkB,CAAC;IAElG,IAAI,AAAkB,YAAlB,OAAOF,QACT,OAAOG;IAGT,OAAO;QACL,GAAGH,MAAM;QACT,QAAQG;IACV;AACF"}

package/dist/es/agent/run-gherkin-scenario.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/run-gherkin-scenario.mjs","sources":["../../../src/agent/run-gherkin-scenario.ts"],"sourcesContent":["import type { AiActOptions } from './agent';\n\nexport type GherkinStepKeyword = 'Given' | 'When' | 'Then' | 'And' | 'But';\n\ntype RunGherkinScenarioStepAction = 'aiAct' | 'aiAssert';\n\nexport type RunGherkinScenarioOptions = AiActOptions & {\n  context?: string;\n};\n\ntype GherkinPrimaryKeyword = 'Given' | 'When' | 'Then';\n\ntype ParsedGherkinStep = {\n  keyword: GherkinStepKeyword;\n  effectiveKeyword: GherkinPrimaryKeyword;\n  text: string;\n  lineNumber: number;\n};\n\ntype RawGherkinStep = {\n  keyword: GherkinStepKeyword;\n  text: string;\n  lineNumber: number;\n};\n\ntype ParsedGherkinScenario = {\n  scenario?: string;\n  steps: ParsedGherkinStep[];\n};\n\ntype GherkinScenarioAgent = {\n  aiAct: (taskPrompt: string, opt?: AiActOptions) => Promise<unknown>;\n  aiAssert: (\n    assertion: string,\n    msg?: string,\n    opt?: { context?: string; abortSignal?: AbortSignal },\n  ) => Promise<unknown>;\n};\n\nconst stepKeywordPattern = /^(Given|When|Then|And|But)\\s+(.+)$/i;\nconst headerPattern = /^(Scenario):\\s*(.*)$/i;\nconst unsupportedHeaderPattern =\n  /^(Feature|Background|Scenario Outline|Scenario Template|Examples|Rule):/i;\n\nconst normalizeStepKeyword = (keyword: string): GherkinStepKeyword => {\n  const lowerKeyword = keyword.toLowerCase();\n  if (lowerKeyword === 'given') return 'Given';\n  if (lowerKeyword === 'when') return 'When';\n  if (lowerKeyword === 'then') return 'Then';\n  if (lowerKeyword === 'and') return 'And';\n  return 'But';\n};\n\nconst isPrimaryKeyword = (\n  keyword: GherkinStepKeyword,\n): keyword is GherkinPrimaryKeyword => {\n  return keyword === 'Given' || keyword === 'When' || keyword === 'Then';\n};\n\nconst resolveSteps = (steps: RawGherkinStep[]): ParsedGherkinStep[] => {\n  let previousPrimaryKeyword: GherkinPrimaryKeyword | undefined;\n\n  return steps.map((step) => {\n    const effectiveKeyword = isPrimaryKeyword(step.keyword)\n      ? step.keyword\n      : previousPrimaryKeyword;\n\n    if (!effectiveKeyword) {\n      throw new Error(\n        `runGherkinScenario cannot resolve \"${step.keyword}\" at line ${step.lineNumber}; use Given, When, or Then before ${step.keyword}.`,\n      );\n    }\n\n    previousPrimaryKeyword = effectiveKeyword;\n\n    return {\n      ...step,\n      effectiveKeyword,\n    };\n  });\n};\n\nconst throwIfAborted = (abortSignal: AbortSignal | undefined) => {\n  if (!abortSignal?.aborted) {\n    return;\n  }\n\n  if (typeof abortSignal.throwIfAborted === 'function') {\n    abortSignal.throwIfAborted();\n  }\n\n  throw new Error(\n    `runGherkinScenario aborted: ${abortSignal.reason || 'signal already aborted'}`,\n  );\n};\n\nexport const parseGherkinScenario = (\n  scenarioText: string,\n): ParsedGherkinScenario => {\n  const lines = scenarioText.split(/\\r?\\n/);\n  const scenarioSteps: RawGherkinStep[] = [];\n  const anonymousSteps: RawGherkinStep[] = [];\n  let scenario: string | undefined;\n  let scenarioCount = 0;\n  let section: 'prelude' | 'scenario' = 'prelude';\n\n  for (const [lineIndex, rawLine] of lines.entries()) {\n    const lineNumber = lineIndex + 1;\n    const line = rawLine.trim();\n\n    if (!line || line.startsWith('#') || line.startsWith('@')) {\n      continue;\n    }\n\n    if (line === '\"\"\"' || line === \"'''\") {\n      throw new Error(\n        `runGherkinScenario does not support doc strings; found one at line ${lineNumber}.`,\n      );\n    }\n\n    if (line.startsWith('|')) {\n      throw new Error(\n        `runGherkinScenario does not support data tables; found one at line ${lineNumber}.`,\n      );\n    }\n\n    if (unsupportedHeaderPattern.test(line)) {\n      throw new Error(\n        `runGherkinScenario does not support \"${line}\" at line ${lineNumber}.`,\n      );\n    }\n\n    const headerMatch = line.match(headerPattern);\n    if (headerMatch) {\n      scenarioCount += 1;\n      scenario = headerMatch[2].trim() || undefined;\n      section = 'scenario';\n      continue;\n    }\n\n    const stepMatch = line.match(stepKeywordPattern);\n    if (stepMatch) {\n      const step = {\n        keyword: normalizeStepKeyword(stepMatch[1]),\n        text: stepMatch[2].trim(),\n        lineNumber,\n      };\n\n      if (section === 'scenario') {\n        scenarioSteps.push(step);\n      } else {\n        anonymousSteps.push(step);\n      }\n\n      continue;\n    }\n\n    throw new Error(\n      `runGherkinScenario does not support content at line ${lineNumber}: ${line}`,\n    );\n  }\n\n  if (scenarioCount > 1) {\n    throw new Error(\n      `runGherkinScenario expects exactly one Scenario, but found ${scenarioCount}.`,\n    );\n  }\n\n  if (scenarioCount === 1 && anonymousSteps.length > 0) {\n    throw new Error(\n      'runGherkinScenario cannot mix anonymous steps with a Scenario block.',\n    );\n  }\n\n  const rawSteps = scenarioCount === 1 ? scenarioSteps : anonymousSteps;\n\n  if (rawSteps.length === 0) {\n    throw new Error('runGherkinScenario requires at least one Gherkin step.');\n  }\n\n  return {\n    scenario,\n    steps: resolveSteps(rawSteps),\n  };\n};\n\nconst buildStepPrompt = (step: ParsedGherkinStep) => {\n  if (step.effectiveKeyword === 'Given') {\n    return `Set up this precondition: ${step.text}`;\n  }\n\n  if (step.effectiveKeyword === 'When') {\n    return `Perform this user action: ${step.text}`;\n  }\n\n  return `Verify that ${step.text}`;\n};\n\nconst describeStepExecution = (\n  step: ParsedGherkinStep,\n  action: RunGherkinScenarioStepAction,\n) => {\n  const keywordMapping =\n    step.keyword === step.effectiveKeyword\n      ? step.keyword\n      : `${step.keyword} as ${step.effectiveKeyword}`;\n\n  if (step.effectiveKeyword === 'Given') {\n    return `setting up the precondition (${keywordMapping} -> ${action})`;\n  }\n\n  if (step.effectiveKeyword === 'When') {\n    return `performing the user action (${keywordMapping} -> ${action})`;\n  }\n\n  return `verifying the expected result (${keywordMapping} -> ${action})`;\n};\n\nconst describeErrorCause = (error: unknown) => {\n  if (error instanceof Error && error.message) {\n    return ` Original error: ${error.message}`;\n  }\n\n  return '';\n};\n\nexport const runGherkinScenario = async (\n  agent: GherkinScenarioAgent,\n  scenarioText: string,\n  opt?: RunGherkinScenarioOptions,\n): Promise<void> => {\n  const parsedScenario = parseGherkinScenario(scenarioText);\n  const aiActOptions: AiActOptions = {\n    ...opt,\n    cacheable: false,\n  };\n\n  for (const step of parsedScenario.steps) {\n    throwIfAborted(opt?.abortSignal);\n\n    const action: RunGherkinScenarioStepAction =\n      step.effectiveKeyword === 'Then' ? 'aiAssert' : 'aiAct';\n    const prompt = buildStepPrompt(step);\n\n    try {\n      if (action === 'aiAct') {\n        await agent.aiAct(prompt, aiActOptions);\n      } else if (opt?.context || opt?.abortSignal) {\n        await agent.aiAssert(prompt, undefined, {\n          context: opt.context,\n          abortSignal: opt.abortSignal,\n        });\n      } else {\n        await agent.aiAssert(prompt);\n      }\n    } catch (error) {\n      throw new Error(\n        `runGherkinScenario failed while ${describeStepExecution(\n          step,\n          action,\n        )} at line ${step.lineNumber}: ${step.keyword} ${step.text}.${describeErrorCause(\n          error,\n        )}`,\n        {\n          cause: error,\n        },\n      );\n    }\n  }\n};\n"],"names":["stepKeywordPattern","headerPattern","unsupportedHeaderPattern","normalizeStepKeyword","keyword","lowerKeyword","isPrimaryKeyword","resolveSteps","steps","previousPrimaryKeyword","step","effectiveKeyword","Error","throwIfAborted","abortSignal","parseGherkinScenario","scenarioText","lines","scenarioSteps","anonymousSteps","scenario","scenarioCount","section","lineIndex","rawLine","lineNumber","line","headerMatch","undefined","stepMatch","rawSteps","buildStepPrompt","describeStepExecution","action","keywordMapping","describeErrorCause","error","runGherkinScenario","agent","opt","parsedScenario","aiActOptions","prompt"],"mappings":"AAuCA,MAAMA,qBAAqB;AAC3B,MAAMC,gBAAgB;AACtB,MAAMC,2BACJ;AAEF,MAAMC,uBAAuB,CAACC;IAC5B,MAAMC,eAAeD,QAAQ,WAAW;IACxC,IAAIC,AAAiB,YAAjBA,cAA0B,OAAO;IACrC,IAAIA,AAAiB,WAAjBA,cAAyB,OAAO;IACpC,IAAIA,AAAiB,WAAjBA,cAAyB,OAAO;IACpC,IAAIA,AAAiB,UAAjBA,cAAwB,OAAO;IACnC,OAAO;AACT;AAEA,MAAMC,mBAAmB,CACvBF,UAEOA,AAAY,YAAZA,WAAuBA,AAAY,WAAZA,WAAsBA,AAAY,WAAZA;AAGtD,MAAMG,eAAe,CAACC;IACpB,IAAIC;IAEJ,OAAOD,MAAM,GAAG,CAAC,CAACE;QAChB,MAAMC,mBAAmBL,iBAAiBI,KAAK,OAAO,IAClDA,KAAK,OAAO,GACZD;QAEJ,IAAI,CAACE,kBACH,MAAM,IAAIC,MACR,CAAC,mCAAmC,EAAEF,KAAK,OAAO,CAAC,UAAU,EAAEA,KAAK,UAAU,CAAC,kCAAkC,EAAEA,KAAK,OAAO,CAAC,CAAC,CAAC;QAItID,yBAAyBE;QAEzB,OAAO;YACL,GAAGD,IAAI;YACPC;QACF;IACF;AACF;AAEA,MAAME,iBAAiB,CAACC;IACtB,IAAI,CAACA,aAAa,SAChB;IAGF,IAAI,AAAsC,cAAtC,OAAOA,YAAY,cAAc,EACnCA,YAAY,cAAc;IAG5B,MAAM,IAAIF,MACR,CAAC,4BAA4B,EAAEE,YAAY,MAAM,IAAI,0BAA0B;AAEnF;AAEO,MAAMC,uBAAuB,CAClCC;IAEA,MAAMC,QAAQD,aAAa,KAAK,CAAC;IACjC,MAAME,gBAAkC,EAAE;IAC1C,MAAMC,iBAAmC,EAAE;IAC3C,IAAIC;IACJ,IAAIC,gBAAgB;IACpB,IAAIC,UAAkC;IAEtC,KAAK,MAAM,CAACC,WAAWC,QAAQ,IAAIP,MAAM,OAAO,GAAI;QAClD,MAAMQ,aAAaF,YAAY;QAC/B,MAAMG,OAAOF,QAAQ,IAAI;QAEzB,IAAI,CAACE,QAAQA,KAAK,UAAU,CAAC,QAAQA,KAAK,UAAU,CAAC,MACnD;QAGF,IAAIA,AAAS,UAATA,QAAkBA,AAAS,UAATA,MACpB,MAAM,IAAId,MACR,CAAC,mEAAmE,EAAEa,WAAW,CAAC,CAAC;QAIvF,IAAIC,KAAK,UAAU,CAAC,MAClB,MAAM,IAAId,MACR,CAAC,mEAAmE,EAAEa,WAAW,CAAC,CAAC;QAIvF,IAAIvB,yBAAyB,IAAI,CAACwB,OAChC,MAAM,IAAId,MACR,CAAC,qCAAqC,EAAEc,KAAK,UAAU,EAAED,WAAW,CAAC,CAAC;QAI1E,MAAME,cAAcD,KAAK,KAAK,CAACzB;QAC/B,IAAI0B,aAAa;YACfN,iBAAiB;YACjBD,WAAWO,WAAW,CAAC,EAAE,CAAC,IAAI,MAAMC;YACpCN,UAAU;YACV;QACF;QAEA,MAAMO,YAAYH,KAAK,KAAK,CAAC1B;QAC7B,IAAI6B,WAAW;YACb,MAAMnB,OAAO;gBACX,SAASP,qBAAqB0B,SAAS,CAAC,EAAE;gBAC1C,MAAMA,SAAS,CAAC,EAAE,CAAC,IAAI;gBACvBJ;YACF;YAEA,IAAIH,AAAY,eAAZA,SACFJ,cAAc,IAAI,CAACR;iBAEnBS,eAAe,IAAI,CAACT;YAGtB;QACF;QAEA,MAAM,IAAIE,MACR,CAAC,oDAAoD,EAAEa,WAAW,EAAE,EAAEC,MAAM;IAEhF;IAEA,IAAIL,gBAAgB,GAClB,MAAM,IAAIT,MACR,CAAC,2DAA2D,EAAES,cAAc,CAAC,CAAC;IAIlF,IAAIA,AAAkB,MAAlBA,iBAAuBF,eAAe,MAAM,GAAG,GACjD,MAAM,IAAIP,MACR;IAIJ,MAAMkB,WAAWT,AAAkB,MAAlBA,gBAAsBH,gBAAgBC;IAEvD,IAAIW,AAAoB,MAApBA,SAAS,MAAM,EACjB,MAAM,IAAIlB,MAAM;IAGlB,OAAO;QACLQ;QACA,OAAOb,aAAauB;IACtB;AACF;AAEA,MAAMC,kBAAkB,CAACrB;IACvB,IAAIA,AAA0B,YAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,0BAA0B,EAAEA,KAAK,IAAI,EAAE;IAGjD,IAAIA,AAA0B,WAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,0BAA0B,EAAEA,KAAK,IAAI,EAAE;IAGjD,OAAO,CAAC,YAAY,EAAEA,KAAK,IAAI,EAAE;AACnC;AAEA,MAAMsB,wBAAwB,CAC5BtB,MACAuB;IAEA,MAAMC,iBACJxB,KAAK,OAAO,KAAKA,KAAK,gBAAgB,GAClCA,KAAK,OAAO,GACZ,GAAGA,KAAK,OAAO,CAAC,IAAI,EAAEA,KAAK,gBAAgB,EAAE;IAEnD,IAAIA,AAA0B,YAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,6BAA6B,EAAEwB,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;IAGvE,IAAIvB,AAA0B,WAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,4BAA4B,EAAEwB,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;IAGtE,OAAO,CAAC,+BAA+B,EAAEC,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;AACzE;AAEA,MAAME,qBAAqB,CAACC;IAC1B,IAAIA,iBAAiBxB,SAASwB,MAAM,OAAO,EACzC,OAAO,CAAC,iBAAiB,EAAEA,MAAM,OAAO,EAAE;IAG5C,OAAO;AACT;AAEO,MAAMC,qBAAqB,OAChCC,OACAtB,cACAuB;IAEA,MAAMC,iBAAiBzB,qBAAqBC;IAC5C,MAAMyB,eAA6B;QACjC,GAAGF,GAAG;QACN,WAAW;IACb;IAEA,KAAK,MAAM7B,QAAQ8B,eAAe,KAAK,CAAE;QACvC3B,eAAe0B,KAAK;QAEpB,MAAMN,SACJvB,AAA0B,WAA1BA,KAAK,gBAAgB,GAAc,aAAa;QAClD,MAAMgC,SAASX,gBAAgBrB;QAE/B,IAAI;YACF,IAAIuB,AAAW,YAAXA,QACF,MAAMK,MAAM,KAAK,CAACI,QAAQD;iBACrB,IAAIF,KAAK,WAAWA,KAAK,aAC9B,MAAMD,MAAM,QAAQ,CAACI,QAAQd,QAAW;gBACtC,SAASW,IAAI,OAAO;gBACpB,aAAaA,IAAI,WAAW;YAC9B;iBAEA,MAAMD,MAAM,QAAQ,CAACI;QAEzB,EAAE,OAAON,OAAO;YACd,MAAM,IAAIxB,MACR,CAAC,gCAAgC,EAAEoB,sBACjCtB,MACAuB,QACA,SAAS,EAAEvB,KAAK,UAAU,CAAC,EAAE,EAAEA,KAAK,OAAO,CAAC,CAAC,EAAEA,KAAK,IAAI,CAAC,CAAC,EAAEyB,mBAC5DC,QACC,EACH;gBACE,OAAOA;YACT;QAEJ;IACF;AACF"}
+{"version":3,"file":"agent/run-gherkin-scenario.mjs","sources":["../../../src/agent/run-gherkin-scenario.ts"],"sourcesContent":["import type { AiActOptions } from './agent';\n\nexport type GherkinStepKeyword = 'Given' | 'When' | 'Then' | 'And' | 'But';\n\ntype RunGherkinScenarioStepAction = 'aiAct' | 'aiAssert';\n\nexport type RunGherkinScenarioOptions = AiActOptions & {\n  /** Additional AI guidance applied to every step in this scenario run. */\n  context?: string;\n};\n\ntype GherkinPrimaryKeyword = 'Given' | 'When' | 'Then';\n\ntype ParsedGherkinStep = {\n  keyword: GherkinStepKeyword;\n  effectiveKeyword: GherkinPrimaryKeyword;\n  text: string;\n  lineNumber: number;\n};\n\ntype RawGherkinStep = {\n  keyword: GherkinStepKeyword;\n  text: string;\n  lineNumber: number;\n};\n\ntype ParsedGherkinScenario = {\n  scenario?: string;\n  steps: ParsedGherkinStep[];\n};\n\ntype GherkinScenarioAgent = {\n  aiAct: (taskPrompt: string, opt?: AiActOptions) => Promise<unknown>;\n  aiAssert: (\n    assertion: string,\n    msg?: string,\n    opt?: { context?: string; abortSignal?: AbortSignal },\n  ) => Promise<unknown>;\n};\n\nconst stepKeywordPattern = /^(Given|When|Then|And|But)\\s+(.+)$/i;\nconst headerPattern = /^(Scenario):\\s*(.*)$/i;\nconst unsupportedHeaderPattern =\n  /^(Feature|Background|Scenario Outline|Scenario Template|Examples|Rule):/i;\n\nconst normalizeStepKeyword = (keyword: string): GherkinStepKeyword => {\n  const lowerKeyword = keyword.toLowerCase();\n  if (lowerKeyword === 'given') return 'Given';\n  if (lowerKeyword === 'when') return 'When';\n  if (lowerKeyword === 'then') return 'Then';\n  if (lowerKeyword === 'and') return 'And';\n  return 'But';\n};\n\nconst isPrimaryKeyword = (\n  keyword: GherkinStepKeyword,\n): keyword is GherkinPrimaryKeyword => {\n  return keyword === 'Given' || keyword === 'When' || keyword === 'Then';\n};\n\nconst resolveSteps = (steps: RawGherkinStep[]): ParsedGherkinStep[] => {\n  let previousPrimaryKeyword: GherkinPrimaryKeyword | undefined;\n\n  return steps.map((step) => {\n    const effectiveKeyword = isPrimaryKeyword(step.keyword)\n      ? step.keyword\n      : previousPrimaryKeyword;\n\n    if (!effectiveKeyword) {\n      throw new Error(\n        `runGherkinScenario cannot resolve \"${step.keyword}\" at line ${step.lineNumber}; use Given, When, or Then before ${step.keyword}.`,\n      );\n    }\n\n    previousPrimaryKeyword = effectiveKeyword;\n\n    return {\n      ...step,\n      effectiveKeyword,\n    };\n  });\n};\n\nconst throwIfAborted = (abortSignal: AbortSignal | undefined) => {\n  if (!abortSignal?.aborted) {\n    return;\n  }\n\n  if (typeof abortSignal.throwIfAborted === 'function') {\n    abortSignal.throwIfAborted();\n  }\n\n  throw new Error(\n    `runGherkinScenario aborted: ${abortSignal.reason || 'signal already aborted'}`,\n  );\n};\n\nexport const parseGherkinScenario = (\n  scenarioText: string,\n): ParsedGherkinScenario => {\n  const lines = scenarioText.split(/\\r?\\n/);\n  const scenarioSteps: RawGherkinStep[] = [];\n  const anonymousSteps: RawGherkinStep[] = [];\n  let scenario: string | undefined;\n  let scenarioCount = 0;\n  let section: 'prelude' | 'scenario' = 'prelude';\n\n  for (const [lineIndex, rawLine] of lines.entries()) {\n    const lineNumber = lineIndex + 1;\n    const line = rawLine.trim();\n\n    if (!line || line.startsWith('#') || line.startsWith('@')) {\n      continue;\n    }\n\n    if (line === '\"\"\"' || line === \"'''\") {\n      throw new Error(\n        `runGherkinScenario does not support doc strings; found one at line ${lineNumber}.`,\n      );\n    }\n\n    if (line.startsWith('|')) {\n      throw new Error(\n        `runGherkinScenario does not support data tables; found one at line ${lineNumber}.`,\n      );\n    }\n\n    if (unsupportedHeaderPattern.test(line)) {\n      throw new Error(\n        `runGherkinScenario does not support \"${line}\" at line ${lineNumber}.`,\n      );\n    }\n\n    const headerMatch = line.match(headerPattern);\n    if (headerMatch) {\n      scenarioCount += 1;\n      scenario = headerMatch[2].trim() || undefined;\n      section = 'scenario';\n      continue;\n    }\n\n    const stepMatch = line.match(stepKeywordPattern);\n    if (stepMatch) {\n      const step = {\n        keyword: normalizeStepKeyword(stepMatch[1]),\n        text: stepMatch[2].trim(),\n        lineNumber,\n      };\n\n      if (section === 'scenario') {\n        scenarioSteps.push(step);\n      } else {\n        anonymousSteps.push(step);\n      }\n\n      continue;\n    }\n\n    throw new Error(\n      `runGherkinScenario does not support content at line ${lineNumber}: ${line}`,\n    );\n  }\n\n  if (scenarioCount > 1) {\n    throw new Error(\n      `runGherkinScenario expects exactly one Scenario, but found ${scenarioCount}.`,\n    );\n  }\n\n  if (scenarioCount === 1 && anonymousSteps.length > 0) {\n    throw new Error(\n      'runGherkinScenario cannot mix anonymous steps with a Scenario block.',\n    );\n  }\n\n  const rawSteps = scenarioCount === 1 ? scenarioSteps : anonymousSteps;\n\n  if (rawSteps.length === 0) {\n    throw new Error('runGherkinScenario requires at least one Gherkin step.');\n  }\n\n  return {\n    scenario,\n    steps: resolveSteps(rawSteps),\n  };\n};\n\nconst buildStepPrompt = (step: ParsedGherkinStep) => {\n  if (step.effectiveKeyword === 'Given') {\n    return `Set up this precondition: ${step.text}`;\n  }\n\n  if (step.effectiveKeyword === 'When') {\n    return `Perform this user action: ${step.text}`;\n  }\n\n  return `Verify that ${step.text}`;\n};\n\nconst describeStepExecution = (\n  step: ParsedGherkinStep,\n  action: RunGherkinScenarioStepAction,\n) => {\n  const keywordMapping =\n    step.keyword === step.effectiveKeyword\n      ? step.keyword\n      : `${step.keyword} as ${step.effectiveKeyword}`;\n\n  if (step.effectiveKeyword === 'Given') {\n    return `setting up the precondition (${keywordMapping} -> ${action})`;\n  }\n\n  if (step.effectiveKeyword === 'When') {\n    return `performing the user action (${keywordMapping} -> ${action})`;\n  }\n\n  return `verifying the expected result (${keywordMapping} -> ${action})`;\n};\n\nconst describeErrorCause = (error: unknown) => {\n  if (error instanceof Error && error.message) {\n    return ` Original error: ${error.message}`;\n  }\n\n  return '';\n};\n\nexport const runGherkinScenario = async (\n  agent: GherkinScenarioAgent,\n  scenarioText: string,\n  opt?: RunGherkinScenarioOptions,\n): Promise<void> => {\n  const parsedScenario = parseGherkinScenario(scenarioText);\n  const aiActOptions: AiActOptions = {\n    ...opt,\n    cacheable: false,\n  };\n\n  for (const step of parsedScenario.steps) {\n    throwIfAborted(opt?.abortSignal);\n\n    const action: RunGherkinScenarioStepAction =\n      step.effectiveKeyword === 'Then' ? 'aiAssert' : 'aiAct';\n    const prompt = buildStepPrompt(step);\n\n    try {\n      if (action === 'aiAct') {\n        await agent.aiAct(prompt, aiActOptions);\n      } else if (opt?.context || opt?.abortSignal) {\n        await agent.aiAssert(prompt, undefined, {\n          context: opt.context,\n          abortSignal: opt.abortSignal,\n        });\n      } else {\n        await agent.aiAssert(prompt);\n      }\n    } catch (error) {\n      throw new Error(\n        `runGherkinScenario failed while ${describeStepExecution(\n          step,\n          action,\n        )} at line ${step.lineNumber}: ${step.keyword} ${step.text}.${describeErrorCause(\n          error,\n        )}`,\n        {\n          cause: error,\n        },\n      );\n    }\n  }\n};\n"],"names":["stepKeywordPattern","headerPattern","unsupportedHeaderPattern","normalizeStepKeyword","keyword","lowerKeyword","isPrimaryKeyword","resolveSteps","steps","previousPrimaryKeyword","step","effectiveKeyword","Error","throwIfAborted","abortSignal","parseGherkinScenario","scenarioText","lines","scenarioSteps","anonymousSteps","scenario","scenarioCount","section","lineIndex","rawLine","lineNumber","line","headerMatch","undefined","stepMatch","rawSteps","buildStepPrompt","describeStepExecution","action","keywordMapping","describeErrorCause","error","runGherkinScenario","agent","opt","parsedScenario","aiActOptions","prompt"],"mappings":"AAwCA,MAAMA,qBAAqB;AAC3B,MAAMC,gBAAgB;AACtB,MAAMC,2BACJ;AAEF,MAAMC,uBAAuB,CAACC;IAC5B,MAAMC,eAAeD,QAAQ,WAAW;IACxC,IAAIC,AAAiB,YAAjBA,cAA0B,OAAO;IACrC,IAAIA,AAAiB,WAAjBA,cAAyB,OAAO;IACpC,IAAIA,AAAiB,WAAjBA,cAAyB,OAAO;IACpC,IAAIA,AAAiB,UAAjBA,cAAwB,OAAO;IACnC,OAAO;AACT;AAEA,MAAMC,mBAAmB,CACvBF,UAEOA,AAAY,YAAZA,WAAuBA,AAAY,WAAZA,WAAsBA,AAAY,WAAZA;AAGtD,MAAMG,eAAe,CAACC;IACpB,IAAIC;IAEJ,OAAOD,MAAM,GAAG,CAAC,CAACE;QAChB,MAAMC,mBAAmBL,iBAAiBI,KAAK,OAAO,IAClDA,KAAK,OAAO,GACZD;QAEJ,IAAI,CAACE,kBACH,MAAM,IAAIC,MACR,CAAC,mCAAmC,EAAEF,KAAK,OAAO,CAAC,UAAU,EAAEA,KAAK,UAAU,CAAC,kCAAkC,EAAEA,KAAK,OAAO,CAAC,CAAC,CAAC;QAItID,yBAAyBE;QAEzB,OAAO;YACL,GAAGD,IAAI;YACPC;QACF;IACF;AACF;AAEA,MAAME,iBAAiB,CAACC;IACtB,IAAI,CAACA,aAAa,SAChB;IAGF,IAAI,AAAsC,cAAtC,OAAOA,YAAY,cAAc,EACnCA,YAAY,cAAc;IAG5B,MAAM,IAAIF,MACR,CAAC,4BAA4B,EAAEE,YAAY,MAAM,IAAI,0BAA0B;AAEnF;AAEO,MAAMC,uBAAuB,CAClCC;IAEA,MAAMC,QAAQD,aAAa,KAAK,CAAC;IACjC,MAAME,gBAAkC,EAAE;IAC1C,MAAMC,iBAAmC,EAAE;IAC3C,IAAIC;IACJ,IAAIC,gBAAgB;IACpB,IAAIC,UAAkC;IAEtC,KAAK,MAAM,CAACC,WAAWC,QAAQ,IAAIP,MAAM,OAAO,GAAI;QAClD,MAAMQ,aAAaF,YAAY;QAC/B,MAAMG,OAAOF,QAAQ,IAAI;QAEzB,IAAI,CAACE,QAAQA,KAAK,UAAU,CAAC,QAAQA,KAAK,UAAU,CAAC,MACnD;QAGF,IAAIA,AAAS,UAATA,QAAkBA,AAAS,UAATA,MACpB,MAAM,IAAId,MACR,CAAC,mEAAmE,EAAEa,WAAW,CAAC,CAAC;QAIvF,IAAIC,KAAK,UAAU,CAAC,MAClB,MAAM,IAAId,MACR,CAAC,mEAAmE,EAAEa,WAAW,CAAC,CAAC;QAIvF,IAAIvB,yBAAyB,IAAI,CAACwB,OAChC,MAAM,IAAId,MACR,CAAC,qCAAqC,EAAEc,KAAK,UAAU,EAAED,WAAW,CAAC,CAAC;QAI1E,MAAME,cAAcD,KAAK,KAAK,CAACzB;QAC/B,IAAI0B,aAAa;YACfN,iBAAiB;YACjBD,WAAWO,WAAW,CAAC,EAAE,CAAC,IAAI,MAAMC;YACpCN,UAAU;YACV;QACF;QAEA,MAAMO,YAAYH,KAAK,KAAK,CAAC1B;QAC7B,IAAI6B,WAAW;YACb,MAAMnB,OAAO;gBACX,SAASP,qBAAqB0B,SAAS,CAAC,EAAE;gBAC1C,MAAMA,SAAS,CAAC,EAAE,CAAC,IAAI;gBACvBJ;YACF;YAEA,IAAIH,AAAY,eAAZA,SACFJ,cAAc,IAAI,CAACR;iBAEnBS,eAAe,IAAI,CAACT;YAGtB;QACF;QAEA,MAAM,IAAIE,MACR,CAAC,oDAAoD,EAAEa,WAAW,EAAE,EAAEC,MAAM;IAEhF;IAEA,IAAIL,gBAAgB,GAClB,MAAM,IAAIT,MACR,CAAC,2DAA2D,EAAES,cAAc,CAAC,CAAC;IAIlF,IAAIA,AAAkB,MAAlBA,iBAAuBF,eAAe,MAAM,GAAG,GACjD,MAAM,IAAIP,MACR;IAIJ,MAAMkB,WAAWT,AAAkB,MAAlBA,gBAAsBH,gBAAgBC;IAEvD,IAAIW,AAAoB,MAApBA,SAAS,MAAM,EACjB,MAAM,IAAIlB,MAAM;IAGlB,OAAO;QACLQ;QACA,OAAOb,aAAauB;IACtB;AACF;AAEA,MAAMC,kBAAkB,CAACrB;IACvB,IAAIA,AAA0B,YAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,0BAA0B,EAAEA,KAAK,IAAI,EAAE;IAGjD,IAAIA,AAA0B,WAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,0BAA0B,EAAEA,KAAK,IAAI,EAAE;IAGjD,OAAO,CAAC,YAAY,EAAEA,KAAK,IAAI,EAAE;AACnC;AAEA,MAAMsB,wBAAwB,CAC5BtB,MACAuB;IAEA,MAAMC,iBACJxB,KAAK,OAAO,KAAKA,KAAK,gBAAgB,GAClCA,KAAK,OAAO,GACZ,GAAGA,KAAK,OAAO,CAAC,IAAI,EAAEA,KAAK,gBAAgB,EAAE;IAEnD,IAAIA,AAA0B,YAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,6BAA6B,EAAEwB,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;IAGvE,IAAIvB,AAA0B,WAA1BA,KAAK,gBAAgB,EACvB,OAAO,CAAC,4BAA4B,EAAEwB,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;IAGtE,OAAO,CAAC,+BAA+B,EAAEC,eAAe,IAAI,EAAED,OAAO,CAAC,CAAC;AACzE;AAEA,MAAME,qBAAqB,CAACC;IAC1B,IAAIA,iBAAiBxB,SAASwB,MAAM,OAAO,EACzC,OAAO,CAAC,iBAAiB,EAAEA,MAAM,OAAO,EAAE;IAG5C,OAAO;AACT;AAEO,MAAMC,qBAAqB,OAChCC,OACAtB,cACAuB;IAEA,MAAMC,iBAAiBzB,qBAAqBC;IAC5C,MAAMyB,eAA6B;QACjC,GAAGF,GAAG;QACN,WAAW;IACb;IAEA,KAAK,MAAM7B,QAAQ8B,eAAe,KAAK,CAAE;QACvC3B,eAAe0B,KAAK;QAEpB,MAAMN,SACJvB,AAA0B,WAA1BA,KAAK,gBAAgB,GAAc,aAAa;QAClD,MAAMgC,SAASX,gBAAgBrB;QAE/B,IAAI;YACF,IAAIuB,AAAW,YAAXA,QACF,MAAMK,MAAM,KAAK,CAACI,QAAQD;iBACrB,IAAIF,KAAK,WAAWA,KAAK,aAC9B,MAAMD,MAAM,QAAQ,CAACI,QAAQd,QAAW;gBACtC,SAASW,IAAI,OAAO;gBACpB,aAAaA,IAAI,WAAW;YAC9B;iBAEA,MAAMD,MAAM,QAAQ,CAACI;QAEzB,EAAE,OAAON,OAAO;YACd,MAAM,IAAIxB,MACR,CAAC,gCAAgC,EAAEoB,sBACjCtB,MACAuB,QACA,SAAS,EAAEvB,KAAK,UAAU,CAAC,EAAE,EAAEA,KAAK,OAAO,CAAC,CAAC,EAAEA,KAAK,IAAI,CAAC,CAAC,EAAEyB,mBAC5DC,QACC,EACH;gBACE,OAAOA;YACT;QAEJ;IACF;AACF"}

package/dist/es/agent/utils.mjs

@@ -154,7 +154,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.12.3";
+const getMidsceneVersion = ()=>"1.12.4-beta-20260902093735.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/prompt/insight/user-prompt.mjs

@@ -7,7 +7,7 @@ function buildTypeQueryDemandValue(type, demand) {
 const extractDataQueryPrompt = (pageDescription, dataQuery, context)=>{
     const dataQueryText = 'string' == typeof dataQuery ? dataQuery : JSON.stringify(dataQuery, null, 2);
     const trimmedContext = context?.trim();
-    const contextSection = trimmedContext ? `\n<CONTEXT>\n${trimmedContext}\n</CONTEXT>\n` : '';
+    const contextSection = trimmedContext ? `\n${trimmedContext}\n` : '';
     return `
 <PageDescription>
 ${pageDescription}

package/dist/es/ai-model/prompt/insight/user-prompt.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/insight/user-prompt.mjs","sources":["../../../../../src/ai-model/prompt/insight/user-prompt.ts"],"sourcesContent":["import type { ServiceExtractParam } from '@/types';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n  context?: string,\n) => {\n  const dataQueryText =\n    typeof dataQuery === 'string'\n      ? dataQuery\n      : JSON.stringify(dataQuery, null, 2);\n\n  const trimmedContext = context?.trim();\n  const contextSection = trimmedContext\n    ? `\\n<CONTEXT>\\n${trimmedContext}\\n</CONTEXT>\\n`\n    : '';\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n${contextSection}\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","extractDataQueryPrompt","pageDescription","dataQuery","context","dataQueryText","JSON","trimmedContext","contextSection"],"mappings":"AAEO,SAASA,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAEO,MAAME,yBAAyB,CACpCC,iBACAC,WACAC;IAEA,MAAMC,gBACJ,AAAqB,YAArB,OAAOF,YACHA,YACAG,KAAK,SAAS,CAACH,WAAW,MAAM;IAEtC,MAAMI,iBAAiBH,SAAS;IAChC,MAAMI,iBAAiBD,iBACnB,CAAC,aAAa,EAAEA,eAAe,cAAc,CAAC,GAC9C;IAEJ,OAAO,CAAC;;AAEV,EAAEL,gBAAgB;;AAElB,EAAEM,eAAe;;AAEjB,EAAEH,cAAc;;EAEd,CAAC;AACH"}
+{"version":3,"file":"ai-model/prompt/insight/user-prompt.mjs","sources":["../../../../../src/ai-model/prompt/insight/user-prompt.ts"],"sourcesContent":["import type { ServiceExtractParam } from '@/types';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n  context?: string,\n) => {\n  const dataQueryText =\n    typeof dataQuery === 'string'\n      ? dataQuery\n      : JSON.stringify(dataQuery, null, 2);\n\n  const trimmedContext = context?.trim();\n  const contextSection = trimmedContext ? `\\n${trimmedContext}\\n` : '';\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n${contextSection}\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","extractDataQueryPrompt","pageDescription","dataQuery","context","dataQueryText","JSON","trimmedContext","contextSection"],"mappings":"AAEO,SAASA,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAEO,MAAME,yBAAyB,CACpCC,iBACAC,WACAC;IAEA,MAAMC,gBACJ,AAAqB,YAArB,OAAOF,YACHA,YACAG,KAAK,SAAS,CAACH,WAAW,MAAM;IAEtC,MAAMI,iBAAiBH,SAAS;IAChC,MAAMI,iBAAiBD,iBAAiB,CAAC,EAAE,EAAEA,eAAe,EAAE,CAAC,GAAG;IAElE,OAAO,CAAC;;AAEV,EAAEL,gBAAgB;;AAElB,EAAEM,eAAe;;AAEjB,EAAEH,cAAc;;EAEd,CAAC;AACH"}

package/dist/es/ai-model/workflows/insight/extraction.mjs

@@ -1,4 +1,5 @@
 import { getDebug } from "@midscene/shared/logger";
+import { renderAIContext, resolvedAIContextFromOptions } from "../../../agent/prompt-context.mjs";
 import { buildInsightSystemPrompt, extractDataQueryPrompt } from "../../prompt/insight/index.mjs";
 import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
@@ -14,7 +15,9 @@ async function AiExtractElementInfo(options) {
         insightProtocol
     });
     const screenshotBase64 = context.screenshot.base64;
-    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery, extractOption?.context);
+    const internalContextOptions = extractOption;
+    const renderedContext = renderAIContext(resolvedAIContextFromOptions(internalContextOptions), internalContextOptions?._internalAdditionalContext);
+    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery, renderedContext);
     const userContent = [];
     if (extractOption?.screenshotIncluded !== false) {
         const screenshotSequence = context.screenshotSequence;

package/dist/es/ai-model/workflows/insight/extraction.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/extraction.mjs","sources":["../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["import type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToChatMessages } from '../../shared/multimodal-prompt';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    extractOption?.context,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["debugInsight","getDebug","AiExtractElementInfo","options","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToChatMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawChoiceMessage","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;;;;AAyBA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,qBAAwBC,OAQ7C;IACC,MAAM,EAAEC,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEL;IACF,MAAMM,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,yBAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAMQ,wBAAwBC,uBAC5BX,QAAQ,eAAe,IAAI,IAC3BC,WACAE,eAAe;IAGjB,MAAMS,cAAyD,EAAE;IAEjE,IAAIT,eAAe,uBAAuB,OAAO;QAC/C,MAAMU,qBAAqBX,QAAQ,kBAAkB;QACrD,IAAIW,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKH;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAG,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAAST;QAAa;QACxC;YACE,MAAM;YACN,SAASK;QACX;KACD;IAED,IAAIR,kBAAkB;QACpB,MAAMa,SAAS,MAAMC,+BAA+Bd;QACpDY,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,MAAMK,qBAChChB,cACA;gBACE,aAAaL,QAAQ,WAAW;gBAChC,sBAAsBoB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAGJ;YACJ,MAAMK,cAAcC,qBAClBL,aACAnB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACLwB;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBnB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaL,QAAQ,WAAW;QAChC,cAAc,CAACoC;YACbvC,aACE,iDACAuC,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/extraction.mjs","sources":["../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["import type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  type InternalAIContextOptions,\n  renderAIContext,\n  resolvedAIContextFromOptions,\n} from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToChatMessages } from '../../shared/multimodal-prompt';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const internalContextOptions = extractOption as\n    | (ServiceExtractOption & InternalAIContextOptions)\n    | undefined;\n  const renderedContext = renderAIContext(\n    resolvedAIContextFromOptions(internalContextOptions),\n    internalContextOptions?._internalAdditionalContext,\n  );\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["debugInsight","getDebug","AiExtractElementInfo","options","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","internalContextOptions","renderedContext","renderAIContext","resolvedAIContextFromOptions","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToChatMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawChoiceMessage","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;;;;;AA8BA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,qBAAwBC,OAQ7C;IACC,MAAM,EAAEC,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEL;IACF,MAAMM,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,yBAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,yBAAyBP;IAG/B,MAAMQ,kBAAkBC,gBACtBC,6BAA6BH,yBAC7BA,wBAAwB;IAE1B,MAAMI,wBAAwBC,uBAC5Bf,QAAQ,eAAe,IAAI,IAC3BC,WACAU;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIb,eAAe,uBAAuB,OAAO;QAC/C,MAAMc,qBAAqBf,QAAQ,kBAAkB;QACrD,IAAIe,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAO,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASb;QAAa;QACxC;YACE,MAAM;YACN,SAASS;QACX;KACD;IAED,IAAIZ,kBAAkB;QACpB,MAAMiB,SAAS,MAAMC,+BAA+BlB;QACpDgB,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,MAAMK,qBAChCpB,cACA;gBACE,aAAaL,QAAQ,WAAW;gBAChC,sBAAsBwB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAGJ;YACJ,MAAMK,cAAcC,qBAClBL,aACAvB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL4B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBvB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaL,QAAQ,WAAW;QAChC,cAAc,CAACwC;YACb3C,aACE,iDACA2C,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}

package/dist/es/ai-model/workflows/planning/custom-planning.mjs

@@ -3,13 +3,13 @@ import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
 import { AIResponseParseError, callAIWithStringResponse } from "../../service-caller/index.mjs";
 import { preparedReferenceImagesToChatMessages } from "../../shared/multimodal-prompt.mjs";
 import { normalizePlanningActionLocateFields } from "./locate-normalization.mjs";
-function appendHighPriorityKnowledge(systemPrompt, actionContext) {
-    return systemPrompt + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n` : '');
+function appendActionContext(systemPrompt, actionContext) {
+    return systemPrompt + (actionContext ? `${actionContext}\n` : '');
 }
 function buildCustomPlanningMessages(input, config) {
     const { options } = input;
     const { conversationHistory, context, actionContext } = options;
-    const systemPrompt = appendHighPriorityKnowledge(config.buildSystemPrompt(), actionContext);
+    const systemPrompt = appendActionContext(config.buildSystemPrompt(), actionContext);
     const userInstructionText = input.userInstruction.text;
     const userInstruction = config.buildUserInstruction ? config.buildUserInstruction(userInstructionText) : userInstructionText;
     const referenceImageMessages = preparedReferenceImagesToChatMessages(input.userInstruction.referenceImages);

package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendHighPriorityKnowledge(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return (\n    systemPrompt +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\\n`\n      : '')\n  );\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ChatCompletionMessageParam[] {\n  const { options } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendHighPriorityKnowledge(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstructionText = input.userInstruction.text;\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    input.userInstruction.referenceImages,\n  );\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.append({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: { url: context.screenshot.base64 },\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...referenceImageMessages,\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...referenceImageMessages,\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: PreparedUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultCodec: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n      acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,\n      parseRawLocateParameter: (value) =>\n        value as ParsedPlanningLocateParameter,\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  // UI-TARS and Auto-GLM define their own assistant history format.\n  if (assistantContent) {\n    options.conversationHistory.append({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawChoiceMessage,\n  };\n}\n"],"names":["appendHighPriorityKnowledge","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToChatMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawChoiceMessage","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","value","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAqBA,SAASA,4BACPC,YAAoB,EACpBC,aAAsB;IAEtB,OACED,eACCC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,4BAA4B,CAAC,GACvE,EAAC;AAET;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,4BACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,sCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBAAE,KAAKC,QAAQ,UAAU,CAAC,MAAM;gBAAC;YAC9C;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,gBAAgB,EAAE,GAAG,MAAMC,yBACjDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;YACA,mBAAmBE,gBAAgB,YAAY,CAAC,OAAO,CAAC,iBAAiB;YACzE,yBAAyB,CAACW,QACxBA;QACJ;QACAF,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOI,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACd,SAASe,QAAW,IACnCd,OACAC;IAEJ;IAEA,MAAMc,mBAAmB9B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IAAI+B,kBACF7B,QAAQ,mBAAmB,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,SAAS6B;IACX;IAGF,OAAO;QACLX;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaQ,KAAK,SAAS,CAACd,SAASe,QAAW;QAChDb;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendActionContext(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return systemPrompt + (actionContext ? `${actionContext}\\n` : '');\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ChatCompletionMessageParam[] {\n  const { options } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendActionContext(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstructionText = input.userInstruction.text;\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    input.userInstruction.referenceImages,\n  );\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.append({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: { url: context.screenshot.base64 },\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...referenceImageMessages,\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...referenceImageMessages,\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: PreparedUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultCodec: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n      acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,\n      parseRawLocateParameter: (value) =>\n        value as ParsedPlanningLocateParameter,\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  // UI-TARS and Auto-GLM define their own assistant history format.\n  if (assistantContent) {\n    options.conversationHistory.append({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawChoiceMessage,\n  };\n}\n"],"names":["appendActionContext","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToChatMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawChoiceMessage","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","value","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAqBA,SAASA,oBACPC,YAAoB,EACpBC,aAAsB;IAEtB,OAAOD,eAAgBC,CAAAA,gBAAgB,GAAGA,cAAc,EAAE,CAAC,GAAG,EAAC;AACjE;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,oBACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,sCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBAAE,KAAKC,QAAQ,UAAU,CAAC,MAAM;gBAAC;YAC9C;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,gBAAgB,EAAE,GAAG,MAAMC,yBACjDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;YACA,mBAAmBE,gBAAgB,YAAY,CAAC,OAAO,CAAC,iBAAiB;YACzE,yBAAyB,CAACW,QACxBA;QACJ;QACAF,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOI,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACd,SAASe,QAAW,IACnCd,OACAC;IAEJ;IAEA,MAAMc,mBAAmB9B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IAAI+B,kBACF7B,QAAQ,mBAAmB,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,SAAS6B;IACX;IAGF,OAAO;QACLX;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaQ,KAAK,SAAS,CAACd,SAASe,QAAW;QAChDb;IACF;AACF"}

package/dist/es/ai-model/workflows/planning/standard-planning.mjs

@@ -101,7 +101,7 @@ async function standardPlan(userInstruction, opts) {
         policy: adapter.imagePreprocess
     });
     const imagePayload = preparedImage.imageBase64;
-    const actionContext = opts.actionContext ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\n` : '';
+    const actionContext = opts.actionContext ? `${opts.actionContext}\n` : '';
     const referenceImageMessages = preparedReferenceImagesToChatMessages(userInstruction.referenceImages);
     const instruction = [
         {

package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n        acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n      });\n      // dumpActionParam keeps only the locator prompt, so runtime-only\n      // locatedPixelBbox fields added during normalization are not serialized.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","actions","normalizePlanningActionLocateFields","yamlFlow","buildYamlFlowFromPlans","parseError","errorMessage","Error","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IAAIS,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAMC,UAAUH,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DI,oCAAoCD,SAAS;gBAC3ChB;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;gBACzD,yBAAyBS,qBAAqB,uBAAuB;YACvE;YAGA,MAAMY,WAAWC,uBAAuBH,SAAShB;YACjD,OAAO;gBAAEY;gBAAUC;gBAAYG;gBAASE;YAAS;QACnD;QACA,cAAc,CAACE,YAAYR;YACzB,MAAMS,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACsB;YACb9B,MACE,mDACA8B,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;QAE9D;IACF;AACF;AAEO,eAAeK,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM9B,eAAe8B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGjC;IACpB,MAAM,EAAEkC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDvB,OACEyB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC9B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIyB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrDiC,QAAQ,QAAQ,CAAC,iBAAiB,GAClCf;IAGJ,MAAMoB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMxB,iBAAiBwB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMvB,aAAauB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAAC1B,mBACnC,MAAM,IAAIqB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMuC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BxB;QACAC;QACA+B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAI1B,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMqC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpC,uBAAuB;IAGpC,MAAMuD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACD1C,UAAU,EACVG,OAAO,EACPE,QAAQ,EACT,GAAG,MAAMxB,6BAA6B;QACrC,UAAUyD;QACVtD;QACA,aAAa8B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7B1B;QACA,qBAAqB;YACnB,cAAcqC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAnC;QACAC;IACF;IAEA,IAAIoD,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIrB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAG5C,UAAU;QACbG;QACAoC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIsB,iBAAiB;QACnB,IAAItB,WAAW,cAAc,EAAE,QAC7BgB,oBAAoB,aAAa,CAAChB,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDgB,oBAAoB,mBAAmB,CAAC6B;QAI5C,IAAI7C,WAAW,GAAG,EAChBgB,oBAAoB,gBAAgB,CAAChB,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBgB,oBAAoB,mBAAmB,CAAChB,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBgB,oBAAoB,YAAY,CAAChB,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7D0D,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOK;AACT"}
+{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n        acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n      });\n      // dumpActionParam keeps only the locator prompt, so runtime-only\n      // locatedPixelBbox fields added during normalization are not serialized.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","actions","normalizePlanningActionLocateFields","yamlFlow","buildYamlFlowFromPlans","parseError","errorMessage","Error","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IAAIS,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAMC,UAAUH,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DI,oCAAoCD,SAAS;gBAC3ChB;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;gBACzD,yBAAyBS,qBAAqB,uBAAuB;YACvE;YAGA,MAAMY,WAAWC,uBAAuBH,SAAShB;YACjD,OAAO;gBAAEY;gBAAUC;gBAAYG;gBAASE;YAAS;QACnD;QACA,cAAc,CAACE,YAAYR;YACzB,MAAMS,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACsB;YACb9B,MACE,mDACA8B,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;QAE9D;IACF;AACF;AAEO,eAAeK,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM9B,eAAe8B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGjC;IACpB,MAAM,EAAEkC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDvB,OACEyB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC9B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIyB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrDiC,QAAQ,QAAQ,CAAC,iBAAiB,GAClCf;IAGJ,MAAMoB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMxB,iBAAiBwB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMvB,aAAauB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAAC1B,mBACnC,MAAM,IAAIqB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMuC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BxB;QACAC;QACA+B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAI1B,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMqC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpC,uBAAuB;IAGpC,MAAMuD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACD1C,UAAU,EACVG,OAAO,EACPE,QAAQ,EACT,GAAG,MAAMxB,6BAA6B;QACrC,UAAUyD;QACVtD;QACA,aAAa8B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7B1B;QACA,qBAAqB;YACnB,cAAcqC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAnC;QACAC;IACF;IAEA,IAAIoD,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIrB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAG5C,UAAU;QACbG;QACAoC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIsB,iBAAiB;QACnB,IAAItB,WAAW,cAAc,EAAE,QAC7BgB,oBAAoB,aAAa,CAAChB,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDgB,oBAAoB,mBAAmB,CAAC6B;QAI5C,IAAI7C,WAAW,GAAG,EAChBgB,oBAAoB,gBAAgB,CAAChB,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBgB,oBAAoB,mBAAmB,CAAChB,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBgB,oBAAoB,YAAY,CAAChB,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7D0D,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOK;AACT"}

package/dist/es/index.mjs

@@ -5,7 +5,7 @@ import { getVersion } from "./utils.mjs";
 import { AiLocateElement, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, getMidsceneLocationSchema, runConnectivityTest, standardPlan } from "./ai-model/index.mjs";
 import { MIDSCENE_MODEL_NAME } from "@midscene/shared/env";
 import { ExecutionDump, GroupedActionDump, ReportActionDump, ServiceError } from "./types.mjs";
-import { Agent, createAgent } from "./agent/index.mjs";
+import { Agent, createAgent, mergeAIContexts } from "./agent/index.mjs";
 import { describeElementAtPoint, verifyElementDescriptionAtPoint, verifyLocator } from "./element-describer.mjs";
 import { createInlineImageResolver, deriveCaseStatus, deriveTaskStatus, escapeContent, generateDumpScriptTag, generateImageScriptTag, parseDumpScript, parseDumpScriptAttributes, parseImageScripts, restoreImageReferences, restoreReportImageReferences, unescapeContent } from "./dump/index.mjs";
 import { getTaskSearchArea, getTaskServiceDump } from "./dump/task-service-dump.mjs";
@@ -16,6 +16,6 @@ import { ScreenshotItem } from "./screenshot-item.mjs";
 import { ReportImageStore, ScreenshotStore } from "./dump/screenshot-store.mjs";
 import { executionToMarkdown, reportToMarkdown } from "./report-markdown.mjs";
 const src = service;
-export { Agent, AiLocateElement, ExecutionDump, GroupedActionDump, MIDSCENE_MODEL_NAME, PointSchema, RectSchema, ReportActionDump, ReportGenerator, ReportImageStore, ReportMergingTool, ScreenshotItem, ScreenshotStore, service as Service, ServiceError, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, TaskRunner, collectDedupedExecutions, createAgent, createInlineImageResolver, createReportCliCommands, dedupeExecutionsKeepLatest, src as default, deriveCaseStatus, deriveTaskStatus, describeElementAtPoint, escapeContent, executionToMarkdown, generateDumpScriptTag, generateImageScriptTag, getMidsceneLocationSchema, getTaskSearchArea, getTaskServiceDump, getVersion, mergeReportFiles, nullReportGenerator, parseDumpScript, parseDumpScriptAttributes, parseImageScripts, reportFileToMarkdown, reportToMarkdown, restoreImageReferences, restoreReportImageReferences, runConnectivityTest, splitReportFile, splitReportHtmlByExecution, standardPlan, unescapeContent, verifyElementDescriptionAtPoint, verifyLocator, z };
+export { Agent, AiLocateElement, ExecutionDump, GroupedActionDump, MIDSCENE_MODEL_NAME, PointSchema, RectSchema, ReportActionDump, ReportGenerator, ReportImageStore, ReportMergingTool, ScreenshotItem, ScreenshotStore, service as Service, ServiceError, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, TaskRunner, collectDedupedExecutions, createAgent, createInlineImageResolver, createReportCliCommands, dedupeExecutionsKeepLatest, src as default, deriveCaseStatus, deriveTaskStatus, describeElementAtPoint, escapeContent, executionToMarkdown, generateDumpScriptTag, generateImageScriptTag, getMidsceneLocationSchema, getTaskSearchArea, getTaskServiceDump, getVersion, mergeAIContexts, mergeReportFiles, nullReportGenerator, parseDumpScript, parseDumpScriptAttributes, parseImageScripts, reportFileToMarkdown, reportToMarkdown, restoreImageReferences, restoreReportImageReferences, runConnectivityTest, splitReportFile, splitReportHtmlByExecution, standardPlan, unescapeContent, verifyElementDescriptionAtPoint, verifyLocator, z };
 
 //# sourceMappingURL=index.mjs.map

package/dist/es/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","sources":["../../src/index.ts"],"sourcesContent":["import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n  standardPlan,\n  AiLocateElement,\n  runConnectivityTest,\n  getMidsceneLocationSchema,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  type ConnectivityTestConfig,\n  type ConnectivityTestResult,\n} from './ai-model/index';\n\nexport {\n  MIDSCENE_MODEL_NAME,\n  type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport {\n  ServiceError,\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n  type IExecutionDump,\n  type IReportActionDump,\n  type IGroupedActionDump,\n  type ReportMeta,\n  type GroupMeta,\n} from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  MidsceneYamlConfig,\n  MidsceneYamlScriptWebEnv,\n  MidsceneYamlScriptAndroidEnv,\n  MidsceneYamlScriptIOSEnv,\n  MidsceneYamlScriptEnv,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport {\n  Agent,\n  type AgentOpt,\n  type AiActOptions,\n  type GherkinStepKeyword,\n  type MidsceneUsageMetrics,\n  type RunGherkinScenarioOptions,\n  type UsageBucket,\n  createAgent,\n  type UIObservation,\n  type UIObserver,\n  type UIObserverOption,\n} from './agent';\nexport {\n  describeElementAtPoint,\n  verifyElementDescriptionAtPoint,\n  verifyLocator,\n  type DescribeElementAtPointOptions,\n  type DescribeElementCoordinateSpace,\n  type ElementDescriberRuntime,\n  type VerifyElementDescriptionAtPointOptions,\n} from './element-describer';\n\n// Dump utilities\nexport {\n  createInlineImageResolver,\n  restoreImageReferences,\n  restoreReportImageReferences,\n  escapeContent,\n  unescapeContent,\n  parseImageScripts,\n  parseDumpScript,\n  parseDumpScriptAttributes,\n  generateImageScriptTag,\n  generateDumpScriptTag,\n  deriveTaskStatus,\n  deriveCaseStatus,\n} from './dump';\nexport type {\n  TaskStatusFields,\n  DerivedTaskStatus,\n  RestoredScreenshotReference,\n  StoredImageReferenceResolver,\n} from './dump';\nexport {\n  getTaskSearchArea,\n  getTaskServiceDump,\n} from './dump/task-service-dump';\n\n// Report generator\nexport type { IReportGenerator } from './report-generator';\nexport { ReportGenerator, nullReportGenerator } from './report-generator';\nexport {\n  collectDedupedExecutions,\n  ReportMergingTool,\n  dedupeExecutionsKeepLatest,\n  splitReportHtmlByExecution,\n} from './report';\nexport {\n  createReportCliCommands,\n  reportFileToMarkdown,\n  splitReportFile,\n  mergeReportFiles,\n  type ConsumeReportFileAction,\n  type ReportFileToMarkdownOptions,\n  type ReportCliCommandDefinition,\n  type ReportCliCommandEntry,\n  type SplitReportFileOptions,\n  type MergeReportFilesOptions,\n  type MergeReportFilesResult,\n} from './report-cli';\n\n// ScreenshotItem\nexport { ScreenshotItem } from './screenshot-item';\nexport type {\n  ImageUrlRef,\n  ScreenshotRef,\n  StoredImageRef,\n} from './dump/image-reference';\nexport {\n  ReportImageStore,\n  ScreenshotStore,\n} from './dump/screenshot-store';\n\nexport {\n  executionToMarkdown,\n  reportToMarkdown,\n  type ExecutionMarkdownOptions,\n  type ExecutionMarkdownResult,\n  type ReportMarkdownOptions,\n  type ReportMarkdownResult,\n  type MarkdownAttachment,\n} from './report-markdown';\n"],"names":["Service"],"mappings":";;;;;;;;;;;;;;;;;AAyCA,YAAeA"}
+{"version":3,"file":"index.mjs","sources":["../../src/index.ts"],"sourcesContent":["import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n  standardPlan,\n  AiLocateElement,\n  runConnectivityTest,\n  getMidsceneLocationSchema,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  type ConnectivityTestConfig,\n  type ConnectivityTestResult,\n} from './ai-model/index';\n\nexport {\n  MIDSCENE_MODEL_NAME,\n  type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport {\n  ServiceError,\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n  type IExecutionDump,\n  type IReportActionDump,\n  type IGroupedActionDump,\n  type ReportMeta,\n  type GroupMeta,\n} from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  MidsceneYamlConfig,\n  MidsceneYamlScriptWebEnv,\n  MidsceneYamlScriptAndroidEnv,\n  MidsceneYamlScriptIOSEnv,\n  MidsceneYamlScriptEnv,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport {\n  Agent,\n  mergeAIContexts,\n  type AgentOpt,\n  type AiActOptions,\n  type GherkinStepKeyword,\n  type MidsceneUsageMetrics,\n  type RunGherkinScenarioOptions,\n  type UsageBucket,\n  createAgent,\n  type UIObservation,\n  type UIObserver,\n  type UIObserverOption,\n} from './agent';\nexport {\n  describeElementAtPoint,\n  verifyElementDescriptionAtPoint,\n  verifyLocator,\n  type DescribeElementAtPointOptions,\n  type DescribeElementCoordinateSpace,\n  type ElementDescriberRuntime,\n  type VerifyElementDescriptionAtPointOptions,\n} from './element-describer';\n\n// Dump utilities\nexport {\n  createInlineImageResolver,\n  restoreImageReferences,\n  restoreReportImageReferences,\n  escapeContent,\n  unescapeContent,\n  parseImageScripts,\n  parseDumpScript,\n  parseDumpScriptAttributes,\n  generateImageScriptTag,\n  generateDumpScriptTag,\n  deriveTaskStatus,\n  deriveCaseStatus,\n} from './dump';\nexport type {\n  TaskStatusFields,\n  DerivedTaskStatus,\n  RestoredScreenshotReference,\n  StoredImageReferenceResolver,\n} from './dump';\nexport {\n  getTaskSearchArea,\n  getTaskServiceDump,\n} from './dump/task-service-dump';\n\n// Report generator\nexport type { IReportGenerator } from './report-generator';\nexport { ReportGenerator, nullReportGenerator } from './report-generator';\nexport {\n  collectDedupedExecutions,\n  ReportMergingTool,\n  dedupeExecutionsKeepLatest,\n  splitReportHtmlByExecution,\n} from './report';\nexport {\n  createReportCliCommands,\n  reportFileToMarkdown,\n  splitReportFile,\n  mergeReportFiles,\n  type ConsumeReportFileAction,\n  type ReportFileToMarkdownOptions,\n  type ReportCliCommandDefinition,\n  type ReportCliCommandEntry,\n  type SplitReportFileOptions,\n  type MergeReportFilesOptions,\n  type MergeReportFilesResult,\n} from './report-cli';\n\n// ScreenshotItem\nexport { ScreenshotItem } from './screenshot-item';\nexport type {\n  ImageUrlRef,\n  ScreenshotRef,\n  StoredImageRef,\n} from './dump/image-reference';\nexport {\n  ReportImageStore,\n  ScreenshotStore,\n} from './dump/screenshot-store';\n\nexport {\n  executionToMarkdown,\n  reportToMarkdown,\n  type ExecutionMarkdownOptions,\n  type ExecutionMarkdownResult,\n  type ReportMarkdownOptions,\n  type ReportMarkdownResult,\n  type MarkdownAttachment,\n} from './report-markdown';\n"],"names":["Service"],"mappings":";;;;;;;;;;;;;;;;;AAyCA,YAAeA"}