@midscene/core 1.4.3 → 1.4.4

package/dist/es/agent/utils.mjs

@@ -106,7 +106,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.4.3";
+const getMidsceneVersion = ()=>"1.4.4";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/llm-planning.mjs

@@ -3,7 +3,7 @@ import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { buildYamlFlowFromPlans, fillBboxParam, findAllMidsceneLocatorField } from "../common.mjs";
 import { systemPromptToTaskPlanning } from "./prompt/llm-planning.mjs";
-import { extractXMLTag, parseMarkFinishedIndexes, parseSubGoalsFromXML } from "./prompt/util.mjs";
+import { extractXMLTag } from "./prompt/util.mjs";
 import { AIResponseParseError, callAI, safeParseJson } from "./service-caller/index.mjs";
 const debug = getDebug('planning');
 const warnLog = getDebug('planning', {
@@ -24,10 +24,6 @@ function parseXMLPlanningResponse(xmlString, modelFamily) {
         finalizeSuccess = 'true' === completeGoalMatch[1];
         finalizeMessage = completeGoalMatch[2]?.trim() || void 0;
     }
-    const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');
-    const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');
-    const updateSubGoals = updatePlanContent ? parseSubGoalsFromXML(updatePlanContent) : void 0;
-    const markFinishedIndexes = markSubGoalDone ? parseMarkFinishedIndexes(markSubGoalDone) : void 0;
     let action = null;
     if (actionType && 'null' !== actionType.toLowerCase()) {
         const type = actionType.trim();
@@ -61,12 +57,6 @@ function parseXMLPlanningResponse(xmlString, modelFamily) {
         } : {},
         ...void 0 !== finalizeSuccess ? {
             finalizeSuccess
-        } : {},
-        ...updateSubGoals?.length ? {
-            updateSubGoals
-        } : {},
-        ...markFinishedIndexes?.length ? {
-            markFinishedIndexes
         } : {}
     };
 }
@@ -75,13 +65,12 @@ async function plan(userInstruction, opts) {
     const { size } = context;
     const screenshotBase64 = context.screenshot.base64;
     const { modelFamily } = modelConfig;
-    const includeSubGoals = true === opts.deepThink;
     const systemPrompt = await systemPromptToTaskPlanning({
         actionSpace: opts.actionSpace,
         modelFamily,
         includeBbox: opts.includeBbox,
         includeThought: true,
-        includeSubGoals
+        deepThink: true === opts.deepThink
     });
     let imagePayload = screenshotBase64;
     let imageWidth = size.width;
@@ -105,8 +94,8 @@ async function plan(userInstruction, opts) {
         }
     ];
     let latestFeedbackMessage;
-    const subGoalsText = includeSubGoals ? conversationHistory.subGoalsToText() : conversationHistory.historicalLogsToText();
-    const subGoalsSection = subGoalsText ? `\n\n${subGoalsText}` : '';
+    const historicalLogsText = conversationHistory.historicalLogsToText();
+    const historicalLogsSection = historicalLogsText ? `\n\n${historicalLogsText}` : '';
     const memoriesText = conversationHistory.memoriesToText();
     const memoriesSection = memoriesText ? `\n\n${memoriesText}` : '';
     if (conversationHistory.pendingFeedbackMessage) {
@@ -115,7 +104,7 @@ async function plan(userInstruction, opts) {
             content: [
                 {
                     type: 'text',
-                    text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${subGoalsSection}`
+                    text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${historicalLogsSection}`
                 },
                 {
                     type: 'image_url',
@@ -132,7 +121,7 @@ async function plan(userInstruction, opts) {
         content: [
             {
                 type: 'text',
-                text: `this is the latest screenshot${memoriesSection}${subGoalsSection}`
+                text: `this is the latest screenshot${memoriesSection}${historicalLogsSection}`
             },
             {
                 type: 'image_url',
@@ -172,7 +161,6 @@ async function plan(userInstruction, opts) {
         if (void 0 !== planFromAI.finalizeSuccess) {
             debug('task completed via <complete> tag, stop planning');
             shouldContinuePlanning = false;
-            if (includeSubGoals) conversationHistory.markAllSubGoalsFinished();
         }
         const returnValue = {
             ...planFromAI,
@@ -195,11 +183,7 @@ async function plan(userInstruction, opts) {
                 if (locateResult && void 0 !== modelFamily) action.param[field] = fillBboxParam(locateResult, imageWidth, imageHeight, modelFamily);
             });
         });
-        if (includeSubGoals) {
-            if (planFromAI.updateSubGoals?.length) conversationHistory.setSubGoals(planFromAI.updateSubGoals);
-            if (planFromAI.markFinishedIndexes?.length) for (const index of planFromAI.markFinishedIndexes)conversationHistory.markSubGoalFinished(index);
-            if (planFromAI.log) conversationHistory.appendSubGoalLog(planFromAI.log);
-        } else if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
+        if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
         if (planFromAI.memory) conversationHistory.appendMemory(planFromAI.memory);
         conversationHistory.append({
             role: 'assistant',

package/dist/es/ai-model/llm-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n  DeepThinkOption,\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n} from '../common';\nimport type { ConversationHistory } from './conversation-history';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport {\n  AIResponseParseError,\n  callAI,\n  safeParseJson,\n} from './service-caller/index';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  modelFamily: TModelFamily | undefined,\n): RawResponsePlanningAIResponse {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    const type = actionType.trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = safeParseJson(actionParamStr, modelFamily);\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    actionContext?: string;\n    modelConfig: IModelConfig;\n    conversationHistory: ConversationHistory;\n    includeBbox: boolean;\n    imagesIncludeCount?: number;\n    deepThink?: DeepThinkOption;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig, conversationHistory } = opts;\n  const { size } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const { modelFamily } = modelConfig;\n\n  // Only enable sub-goals when deepThink is true\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    modelFamily,\n    includeBbox: opts.includeBbox,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = size.width;\n  let imageHeight = size.height;\n  const rightLimit = imageWidth;\n  const bottomLimit = imageHeight;\n\n  // Process image based on VL mode requirements\n  if (modelFamily === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`,\n        },\n      ],\n    },\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In deepThink mode: show full sub-goals with logs\n  // In non-deepThink mode: show historical execution logs\n  const subGoalsText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const subGoalsSection = subGoalsText ? `\\n\\n${subGoalsText}` : '';\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${subGoalsSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `this is the latest screenshot${memoriesSection}${subGoalsSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n  } = await callAI(msgs, modelConfig, {\n    deepThink: opts.deepThink === 'unset' ? undefined : opts.deepThink,\n  });\n\n  // Parse XML response to JSON object, capture parsing errors\n  let planFromAI: RawResponsePlanningAIResponse;\n  try {\n    planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);\n\n    if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n      warnLog(\n        'Planning response included both an action and <complete>; ignoring <complete> output.',\n      );\n      planFromAI.finalizeMessage = undefined;\n      planFromAI.finalizeSuccess = undefined;\n    }\n\n    const actions = planFromAI.action ? [planFromAI.action] : [];\n    let shouldContinuePlanning = true;\n\n    // Check if task is completed via <complete> tag\n    if (planFromAI.finalizeSuccess !== undefined) {\n      debug('task completed via <complete> tag, stop planning');\n      shouldContinuePlanning = false;\n      // Mark all sub-goals as finished when goal is completed (only when deepThink is enabled)\n      if (includeSubGoals) {\n        conversationHistory.markAllSubGoalsFinished();\n      }\n    }\n\n    const returnValue: PlanningAIResponse = {\n      ...planFromAI,\n      actions,\n      rawResponse,\n      usage,\n      reasoning_content,\n      yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),\n      shouldContinuePlanning,\n    };\n\n    assert(planFromAI, \"can't get plans from AI\");\n\n    actions.forEach((action) => {\n      const type = action.type;\n      const actionInActionSpace = opts.actionSpace.find(\n        (action) => action.name === type,\n      );\n\n      debug('actionInActionSpace matched', actionInActionSpace);\n      const locateFields = actionInActionSpace\n        ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n        : [];\n\n      debug('locateFields', locateFields);\n\n      locateFields.forEach((field) => {\n        const locateResult = action.param[field];\n        if (locateResult && modelFamily !== undefined) {\n          // Always use model family to fill bbox parameters\n          action.param[field] = fillBboxParam(\n            locateResult,\n            imageWidth,\n            imageHeight,\n            modelFamily,\n          );\n        }\n      });\n    });\n\n    // Update sub-goals in conversation history based on response (only when deepThink is enabled)\n    if (includeSubGoals) {\n      if (planFromAI.updateSubGoals?.length) {\n        conversationHistory.setSubGoals(planFromAI.updateSubGoals);\n      }\n      if (planFromAI.markFinishedIndexes?.length) {\n        for (const index of planFromAI.markFinishedIndexes) {\n          conversationHistory.markSubGoalFinished(index);\n        }\n      }\n      // Append the planning log to the currently running sub-goal\n      if (planFromAI.log) {\n        conversationHistory.appendSubGoalLog(planFromAI.log);\n      }\n    } else {\n      // In non-deepThink mode, accumulate logs as historical execution steps\n      if (planFromAI.log) {\n        conversationHistory.appendHistoricalLog(planFromAI.log);\n      }\n    }\n\n    // Append memory to conversation history if present\n    if (planFromAI.memory) {\n      conversationHistory.appendMemory(planFromAI.memory);\n    }\n\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n\n    return returnValue;\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n    );\n  }\n}\n"],"names":["debug","getDebug","warnLog","parseXMLPlanningResponse","xmlString","modelFamily","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","safeParseJson","e","Error","plan","userInstruction","opts","context","modelConfig","conversationHistory","size","screenshotBase64","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","imagePayload","imageWidth","imageHeight","paddedResult","paddingToMatchBlockByBase64","actionContext","instruction","latestFeedbackMessage","subGoalsText","subGoalsSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","callAI","planFromAI","actions","shouldContinuePlanning","returnValue","buildYamlFlowFromPlans","assert","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","index","parseError","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;AA+BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAK9C,SAASE,yBACdC,SAAiB,EACjBC,WAAqC;IAErC,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,cAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,cAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,qBAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,yBAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QACrD,MAAMc,OAAOd,WAAW,IAAI;QAC5B,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQC,cAAcf,gBAAgBP;QACxC,EAAE,OAAOuB,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFJ,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAEO,eAAeQ,KACpBC,eAAuB,EACvBC,IAUC;IAED,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGH;IACtD,MAAM,EAAEI,IAAI,EAAE,GAAGH;IACjB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM,EAAE5B,WAAW,EAAE,GAAG6B;IAGxB,MAAMI,kBAAkBN,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMO,eAAe,MAAMC,2BAA2B;QACpD,aAAaR,KAAK,WAAW;QAC7B3B;QACA,aAAa2B,KAAK,WAAW;QAC7B,gBAAgB;QAChBM;IACF;IAEA,IAAIG,eAAeJ;IACnB,IAAIK,aAAaN,KAAK,KAAK;IAC3B,IAAIO,cAAcP,KAAK,MAAM;IAK7B,IAAI/B,AAAgB,iBAAhBA,aAA8B;QAChC,MAAMuC,eAAe,MAAMC,4BAA4BJ;QACvDC,aAAaE,aAAa,KAAK;QAC/BD,cAAcC,aAAa,MAAM;QACjCH,eAAeG,aAAa,WAAW;IACzC;IAEA,MAAME,gBAAgBd,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMe,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGD,cAAc,kBAAkB,EAAEf,gBAAgB,mBAAmB,CAAC;gBACjF;aACD;QACH;KACD;IAED,IAAIiB;IAKJ,MAAMC,eAAeX,kBACjBH,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMe,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAG/D,MAAME,eAAehB,oBAAoB,cAAc;IACvD,MAAMiB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIhB,oBAAoB,sBAAsB,EAAE;QAC9Ca,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGb,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEiB,kBAAkBF,iBAAiB;gBAChN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKT;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAN,oBAAoB,mCAAmC;IACzD,OACEa,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,6BAA6B,EAAEI,kBAAkBF,iBAAiB;YAC3E;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKT;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFN,oBAAoB,MAAM,CAACa;IAG3Bb,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMkB,aAAalB,oBAAoB,QAAQ,CAACH,KAAK,kBAAkB;IAEvE,MAAMsB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,OAAOJ,MAAMpB,aAAa;QAClC,WAAWF,AAAmB,YAAnBA,KAAK,SAAS,GAAef,SAAYe,KAAK,SAAS;IACpE;IAGA,IAAI2B;IACJ,IAAI;QACFA,aAAaxD,yBAAyBoD,aAAalD;QAEnD,IAAIsD,WAAW,MAAM,IAAIA,AAA+B1C,WAA/B0C,WAAW,eAAe,EAAgB;YACjEzD,QACE;YAEFyD,WAAW,eAAe,GAAG1C;YAC7B0C,WAAW,eAAe,GAAG1C;QAC/B;QAEA,MAAM2C,UAAUD,WAAW,MAAM,GAAG;YAACA,WAAW,MAAM;SAAC,GAAG,EAAE;QAC5D,IAAIE,yBAAyB;QAG7B,IAAIF,AAA+B1C,WAA/B0C,WAAW,eAAe,EAAgB;YAC5C3D,MAAM;YACN6D,yBAAyB;YAEzB,IAAIvB,iBACFH,oBAAoB,uBAAuB;QAE/C;QAEA,MAAM2B,cAAkC;YACtC,GAAGH,UAAU;YACbC;YACAL;YACAC;YACAC;YACA,UAAUM,uBAAuBH,SAAS5B,KAAK,WAAW;YAC1D6B;QACF;QAEAG,OAAOL,YAAY;QAEnBC,QAAQ,OAAO,CAAC,CAACpC;YACf,MAAMC,OAAOD,OAAO,IAAI;YACxB,MAAMyC,sBAAsBjC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACR,SAAWA,OAAO,IAAI,KAAKC;YAG9BzB,MAAM,+BAA+BiE;YACrC,MAAMC,eAAeD,sBACjBE,4BAA4BF,oBAAoB,WAAW,IAC3D,EAAE;YAENjE,MAAM,gBAAgBkE;YAEtBA,aAAa,OAAO,CAAC,CAACE;gBACpB,MAAMC,eAAe7C,OAAO,KAAK,CAAC4C,MAAM;gBACxC,IAAIC,gBAAgBhE,AAAgBY,WAAhBZ,aAElBmB,OAAO,KAAK,CAAC4C,MAAM,GAAGE,cACpBD,cACA3B,YACAC,aACAtC;YAGN;QACF;QAGA,IAAIiC,iBAAiB;YACnB,IAAIqB,WAAW,cAAc,EAAE,QAC7BxB,oBAAoB,WAAW,CAACwB,WAAW,cAAc;YAE3D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAMY,SAASZ,WAAW,mBAAmB,CAChDxB,oBAAoB,mBAAmB,CAACoC;YAI5C,IAAIZ,WAAW,GAAG,EAChBxB,oBAAoB,gBAAgB,CAACwB,WAAW,GAAG;QAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBxB,oBAAoB,mBAAmB,CAACwB,WAAW,GAAG;QAK1D,IAAIA,WAAW,MAAM,EACnBxB,oBAAoB,YAAY,CAACwB,WAAW,MAAM;QAGpDxB,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMoB;gBACR;aACD;QACH;QAEA,OAAOO;IACT,EAAE,OAAOU,YAAY;QAEnB,MAAMC,eACJD,sBAAsB3C,QAAQ2C,WAAW,OAAO,GAAGE,OAAOF;QAC5D,MAAM,IAAIG,qBACR,CAAC,iBAAiB,EAAEF,cAAc,EAClClB,aACAC;IAEJ;AACF"}
+{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n  DeepThinkOption,\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n} from '../common';\nimport type { ConversationHistory } from './conversation-history';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport { extractXMLTag } from './prompt/util';\nimport {\n  AIResponseParseError,\n  callAI,\n  safeParseJson,\n} from './service-caller/index';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  modelFamily: TModelFamily | undefined,\n): RawResponsePlanningAIResponse {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    const type = actionType.trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = safeParseJson(actionParamStr, modelFamily);\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n  };\n}\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    actionContext?: string;\n    modelConfig: IModelConfig;\n    conversationHistory: ConversationHistory;\n    includeBbox: boolean;\n    imagesIncludeCount?: number;\n    deepThink?: DeepThinkOption;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig, conversationHistory } = opts;\n  const { size } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const { modelFamily } = modelConfig;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    modelFamily,\n    includeBbox: opts.includeBbox,\n    includeThought: true, // always include thought\n    deepThink: opts.deepThink === true,\n  });\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = size.width;\n  let imageHeight = size.height;\n  const rightLimit = imageWidth;\n  const bottomLimit = imageHeight;\n\n  // Process image based on VL mode requirements\n  if (modelFamily === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`,\n        },\n      ],\n    },\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build historical execution logs text to include in the message\n  const historicalLogsText = conversationHistory.historicalLogsToText();\n  const historicalLogsSection = historicalLogsText\n    ? `\\n\\n${historicalLogsText}`\n    : '';\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${historicalLogsSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `this is the latest screenshot${memoriesSection}${historicalLogsSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n  } = await callAI(msgs, modelConfig, {\n    deepThink: opts.deepThink === 'unset' ? undefined : opts.deepThink,\n  });\n\n  // Parse XML response to JSON object, capture parsing errors\n  let planFromAI: RawResponsePlanningAIResponse;\n  try {\n    planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);\n\n    if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n      warnLog(\n        'Planning response included both an action and <complete>; ignoring <complete> output.',\n      );\n      planFromAI.finalizeMessage = undefined;\n      planFromAI.finalizeSuccess = undefined;\n    }\n\n    const actions = planFromAI.action ? [planFromAI.action] : [];\n    let shouldContinuePlanning = true;\n\n    // Check if task is completed via <complete> tag\n    if (planFromAI.finalizeSuccess !== undefined) {\n      debug('task completed via <complete> tag, stop planning');\n      shouldContinuePlanning = false;\n    }\n\n    const returnValue: PlanningAIResponse = {\n      ...planFromAI,\n      actions,\n      rawResponse,\n      usage,\n      reasoning_content,\n      yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),\n      shouldContinuePlanning,\n    };\n\n    assert(planFromAI, \"can't get plans from AI\");\n\n    actions.forEach((action) => {\n      const type = action.type;\n      const actionInActionSpace = opts.actionSpace.find(\n        (action) => action.name === type,\n      );\n\n      debug('actionInActionSpace matched', actionInActionSpace);\n      const locateFields = actionInActionSpace\n        ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n        : [];\n\n      debug('locateFields', locateFields);\n\n      locateFields.forEach((field) => {\n        const locateResult = action.param[field];\n        if (locateResult && modelFamily !== undefined) {\n          // Always use model family to fill bbox parameters\n          action.param[field] = fillBboxParam(\n            locateResult,\n            imageWidth,\n            imageHeight,\n            modelFamily,\n          );\n        }\n      });\n    });\n\n    // Accumulate logs as historical execution steps\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n\n    // Append memory to conversation history if present\n    if (planFromAI.memory) {\n      conversationHistory.appendMemory(planFromAI.memory);\n    }\n\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n\n    return returnValue;\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n    );\n  }\n}\n"],"names":["debug","getDebug","warnLog","parseXMLPlanningResponse","xmlString","modelFamily","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","action","type","param","safeParseJson","e","Error","plan","userInstruction","opts","context","modelConfig","conversationHistory","size","screenshotBase64","systemPrompt","systemPromptToTaskPlanning","imagePayload","imageWidth","imageHeight","paddedResult","paddingToMatchBlockByBase64","actionContext","instruction","latestFeedbackMessage","historicalLogsText","historicalLogsSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","callAI","planFromAI","actions","shouldContinuePlanning","returnValue","buildYamlFlowFromPlans","assert","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","parseError","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;AA2BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAK9C,SAASE,yBACdC,SAAiB,EACjBC,WAAqC;IAErC,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,IAAIC,SAAc;IAClB,IAAIP,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QACrD,MAAMQ,OAAOR,WAAW,IAAI;QAC5B,IAAIS;QAEJ,IAAIR,gBACF,IAAI;YAEFQ,QAAQC,cAAcT,gBAAgBP;QACxC,EAAE,OAAOiB,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFJ,SAAS;YACPC;YACA,GAAIC,AAAUH,WAAVG,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAId,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1BQ;QACA,GAAIH,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;IAC9D;AACF;AAEO,eAAeQ,KACpBC,eAAuB,EACvBC,IAUC;IAED,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGH;IACtD,MAAM,EAAEI,IAAI,EAAE,GAAGH;IACjB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM,EAAEtB,WAAW,EAAE,GAAGuB;IAExB,MAAMI,eAAe,MAAMC,2BAA2B;QACpD,aAAaP,KAAK,WAAW;QAC7BrB;QACA,aAAaqB,KAAK,WAAW;QAC7B,gBAAgB;QAChB,WAAWA,AAAmB,SAAnBA,KAAK,SAAS;IAC3B;IAEA,IAAIQ,eAAeH;IACnB,IAAII,aAAaL,KAAK,KAAK;IAC3B,IAAIM,cAAcN,KAAK,MAAM;IAK7B,IAAIzB,AAAgB,iBAAhBA,aAA8B;QAChC,MAAMgC,eAAe,MAAMC,4BAA4BJ;QACvDC,aAAaE,aAAa,KAAK;QAC/BD,cAAcC,aAAa,MAAM;QACjCH,eAAeG,aAAa,WAAW;IACzC;IAEA,MAAME,gBAAgBb,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMc,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGD,cAAc,kBAAkB,EAAEd,gBAAgB,mBAAmB,CAAC;gBACjF;aACD;QACH;KACD;IAED,IAAIgB;IAGJ,MAAMC,qBAAqBb,oBAAoB,oBAAoB;IACnE,MAAMc,wBAAwBD,qBAC1B,CAAC,IAAI,EAAEA,oBAAoB,GAC3B;IAGJ,MAAME,eAAef,oBAAoB,cAAc;IACvD,MAAMgB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIf,oBAAoB,sBAAsB,EAAE;QAC9CY,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGZ,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEgB,kBAAkBF,uBAAuB;gBACtN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKT;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAL,oBAAoB,mCAAmC;IACzD,OACEY,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,6BAA6B,EAAEI,kBAAkBF,uBAAuB;YACjF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKT;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFL,oBAAoB,MAAM,CAACY;IAG3BZ,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMiB,aAAajB,oBAAoB,QAAQ,CAACH,KAAK,kBAAkB;IAEvE,MAAMqB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,OAAOJ,MAAMnB,aAAa;QAClC,WAAWF,AAAmB,YAAnBA,KAAK,SAAS,GAAeT,SAAYS,KAAK,SAAS;IACpE;IAGA,IAAI0B;IACJ,IAAI;QACFA,aAAajD,yBAAyB6C,aAAa3C;QAEnD,IAAI+C,WAAW,MAAM,IAAIA,AAA+BnC,WAA/BmC,WAAW,eAAe,EAAgB;YACjElD,QACE;YAEFkD,WAAW,eAAe,GAAGnC;YAC7BmC,WAAW,eAAe,GAAGnC;QAC/B;QAEA,MAAMoC,UAAUD,WAAW,MAAM,GAAG;YAACA,WAAW,MAAM;SAAC,GAAG,EAAE;QAC5D,IAAIE,yBAAyB;QAG7B,IAAIF,AAA+BnC,WAA/BmC,WAAW,eAAe,EAAgB;YAC5CpD,MAAM;YACNsD,yBAAyB;QAC3B;QAEA,MAAMC,cAAkC;YACtC,GAAGH,UAAU;YACbC;YACAL;YACAC;YACAC;YACA,UAAUM,uBAAuBH,SAAS3B,KAAK,WAAW;YAC1D4B;QACF;QAEAG,OAAOL,YAAY;QAEnBC,QAAQ,OAAO,CAAC,CAACnC;YACf,MAAMC,OAAOD,OAAO,IAAI;YACxB,MAAMwC,sBAAsBhC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACR,SAAWA,OAAO,IAAI,KAAKC;YAG9BnB,MAAM,+BAA+B0D;YACrC,MAAMC,eAAeD,sBACjBE,4BAA4BF,oBAAoB,WAAW,IAC3D,EAAE;YAEN1D,MAAM,gBAAgB2D;YAEtBA,aAAa,OAAO,CAAC,CAACE;gBACpB,MAAMC,eAAe5C,OAAO,KAAK,CAAC2C,MAAM;gBACxC,IAAIC,gBAAgBzD,AAAgBY,WAAhBZ,aAElBa,OAAO,KAAK,CAAC2C,MAAM,GAAGE,cACpBD,cACA3B,YACAC,aACA/B;YAGN;QACF;QAGA,IAAI+C,WAAW,GAAG,EAChBvB,oBAAoB,mBAAmB,CAACuB,WAAW,GAAG;QAIxD,IAAIA,WAAW,MAAM,EACnBvB,oBAAoB,YAAY,CAACuB,WAAW,MAAM;QAGpDvB,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMmB;gBACR;aACD;QACH;QAEA,OAAOO;IACT,EAAE,OAAOS,YAAY;QAEnB,MAAMC,eACJD,sBAAsBzC,QAAQyC,WAAW,OAAO,GAAGE,OAAOF;QAC5D,MAAM,IAAIG,qBACR,CAAC,iBAAiB,EAAEF,cAAc,EAClCjB,aACAC;IAEJ;AACF"}

package/dist/es/ai-model/prompt/llm-planning.mjs

@@ -63,12 +63,11 @@ const descriptionForAction = (action, locatorSchemaTypeDescription)=>{
 ${tab}${fields.join(`\n${tab}`)}
 `.trim();
 };
-async function systemPromptToTaskPlanning({ actionSpace, modelFamily, includeBbox, includeThought, includeSubGoals }) {
+async function systemPromptToTaskPlanning({ actionSpace, modelFamily, includeBbox, includeThought, deepThink }) {
     const preferredLanguage = getPreferredLanguage();
     if (includeBbox && !modelFamily) throw new Error('modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.');
     const actionDescriptionList = actionSpace.map((action)=>descriptionForAction(action, vlLocateParam(includeBbox ? modelFamily : void 0)));
     const actionList = actionDescriptionList.join('\n');
-    const shouldIncludeSubGoals = includeSubGoals ?? false;
     const locateExample1 = includeBbox ? `{
     "prompt": "Add to cart button for Sauce Labs Backpack",
     "bbox": [345, 442, 458, 483]
@@ -87,98 +86,26 @@ async function systemPromptToTaskPlanning({ actionSpace, modelFamily, includeBbo
   }` : `{
     "prompt": "Email input field in the registration form"
   }`;
-    const step1Title = shouldIncludeSubGoals ? '## Step 1: Observe and Plan (related tags: <thought>, <update-plan-content>, <mark-sub-goal-done>)' : '## Step 1: Observe (related tags: <thought>)';
-    const step1Description = shouldIncludeSubGoals ? "First, observe the current screenshot and previous logs, then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot." : 'First, observe the current screenshot and previous logs to understand the current state.';
     const explicitInstructionRule = 'CRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: "fill out the form" means only fill fields, do NOT submit; "click the button" means only click, do NOT wait for page load or verify results; "type \'hello\'" means only type, do NOT press Enter.';
-    const thoughtTagDescription = shouldIncludeSubGoals ? `REQUIRED: You MUST always output the <thought> tag. Never skip it.
+    const thoughtTagDescription = `REQUIRED: You MUST always output the <thought> tag. Never skip it.
 
-Include your thought process in the <thought> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action? Write your thoughts naturally without numbering or section headers.
-
-${explicitInstructionRule}` : `REQUIRED: You MUST always output the <thought> tag. Never skip it.
-
-Include your thought process in the <thought> tag. It should answer: What is the current state based on the screenshot? What should be the next action? Write your thoughts naturally without numbering or section headers.
+Include your thought process in the <thought> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? What should be the next action? Write your thoughts naturally without numbering or section headers.
 
 ${explicitInstructionRule}`;
-    const subGoalTags = shouldIncludeSubGoals ? `
-
-* <update-plan-content> tag
-
-Use this structure to give or update your plan:
-
-<update-plan-content>
-  <sub-goal index="1" status="finished|pending">sub goal description</sub-goal>
-  <sub-goal index="2" status="finished|pending">sub goal description</sub-goal>
-  ...
-</update-plan-content>
-
-* <mark-sub-goal-done> tag
-
-Use this structure to mark a sub-goal as done:
-
-<mark-sub-goal-done>
-  <sub-goal index="1" status="finished" />
-</mark-sub-goal-done>
-
-IMPORTANT: You MUST only mark a sub-goal as "finished" AFTER you have confirmed the task is actually completed by observing the result in the screenshot. Do NOT mark a sub-goal as done just because you expect the next action will complete it. Wait until you see visual confirmation in the screenshot that the sub-goal has been achieved.
-
-* Note
-
-During execution, you can call <update-plan-content> at any time to update the plan based on the latest screenshot and completed sub-goals.
-
-### Example
-
-If the user wants to "log in to a system using username and password, complete all to-do items, and submit a registration form", you can break it down into the following sub-goals:
-
-<thought>...</thought>
-<update-plan-content>
-  <sub-goal index="1" status="pending">Log in to the system</sub-goal>
-  <sub-goal index="2" status="pending">Complete all to-do items</sub-goal>
-  <sub-goal index="3" status="pending">Submit the registration form</sub-goal>
-</update-plan-content>
-
-After logging in and seeing the to-do items, you can mark the sub-goal as done:
-
-<mark-sub-goal-done>
-  <sub-goal index="1" status="finished" />
-</mark-sub-goal-done>
-
-At this point, the status of all sub-goals is:
-
-<update-plan-content>
-  <sub-goal index="1" status="finished" />
-  <sub-goal index="2" status="pending" />
-  <sub-goal index="3" status="pending" />
-</update-plan-content>
-
-After some time, when the last sub-goal is also completed, you can mark it as done as well:
-
-<mark-sub-goal-done>
-  <sub-goal index="3" status="finished" />
-</mark-sub-goal-done>` : '';
-    const memoryStepNumber = 2;
-    const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;
-    const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;
     return `
 Target: You are an expert to manipulate the UI to accomplish the user's instruction. User will give you an instruction, some screenshots, background knowledge and previous logs indicating what have been done. Your task is to accomplish the instruction by thinking through the path to complete the task and give the next action to execute.
 
-${step1Title}
+## Step 1: Observe (related tags: <thought>)
 
-${step1Description}
+First, observe the current screenshot and previous logs to understand the current state.
 
 * <thought> tag (REQUIRED)
 
 ${thoughtTagDescription}
-${subGoalTags}
-${shouldIncludeSubGoals ? `
-## Step ${memoryStepNumber}: Memory Data from Current Screenshot (related tags: <memory>)
 
-While observing the current screenshot, if you notice any information that might be needed in follow-up actions, record it here. The current screenshot will NOT be available in subsequent steps, so this memory is your only way to preserve essential information. Examples: extracted data, element states, content that needs to be referenced.
+## Step 2: Check if the Instruction is Fulfilled (related tags: <complete>)
 
-Don't use this tag if no information needs to be preserved.
-` : ''}
-## Step ${checkGoalStepNumber}: ${shouldIncludeSubGoals ? 'Check if Goal is Accomplished' : 'Check if the Instruction is Fulfilled'} (related tags: <complete>)
-
-${shouldIncludeSubGoals ? 'Based on the current screenshot and the status of all sub-goals, determine' : 'Determine'} if the entire task is completed.
+Determine if the entire task is completed.
 
 ### CRITICAL: The User's Instruction is the Supreme Authority
 
@@ -188,36 +115,36 @@ The user's instruction defines the EXACT scope of what you must accomplish. You
 - If the user gives you **explicit operation steps** (e.g., "click X", "type Y", "fill out the form"), treat them as exact commands. Execute ONLY those steps, nothing more.
 - If the user gives you a **high-level goal** (e.g., "log in to the system", "complete the purchase"), you may determine the necessary steps to achieve it.
 
-**What "${shouldIncludeSubGoals ? 'goal accomplished' : 'instruction fulfilled'}" means:**
-- The ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} when you have done EXACTLY what the user asked - no extra steps, no assumptions.
+**What "instruction fulfilled" means:**
+- The instruction is fulfilled when you have done EXACTLY what the user asked - no extra steps, no assumptions.
 - Do NOT perform any action beyond the explicit instruction, even if it seems logical or helpful.
 
 **Examples - Explicit instructions (execute exactly, no extra steps):**
-- "fill out the form" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when all fields are filled. Do NOT submit the form.
-- "click the login button" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} once clicked. Do NOT wait for page load or verify login success.
-- "type 'hello' in the search box" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when 'hello' is typed. Do NOT press Enter or trigger search.
-- "select the first item" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when selected. Do NOT proceed to checkout.
+- "fill out the form" → Instruction fulfilled when all fields are filled. Do NOT submit the form.
+- "click the login button" → Instruction fulfilled once clicked. Do NOT wait for page load or verify login success.
+- "type 'hello' in the search box" → Instruction fulfilled when 'hello' is typed. Do NOT press Enter or trigger search.
+- "select the first item" → Instruction fulfilled when selected. Do NOT proceed to checkout.
 
 **Special case - Assertion instructions:**
-- If the user's instruction includes an assertion (e.g., "verify that...", "check that...", "assert..."), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark ${shouldIncludeSubGoals ? 'the goal' : 'it'} as failed (success="false").
+- If the user's instruction includes an assertion (e.g., "verify that...", "check that...", "assert..."), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark it as failed (success="false").
 - If the page is still loading (e.g., you see a loading spinner, skeleton screen, or progress bar), do NOT assert yet. Wait for the page to finish loading before evaluating the assertion.
-${!shouldIncludeSubGoals ? `
+
 **Page navigation restriction:**
 - Unless the user's instruction explicitly asks you to click a link, jump to another page, or navigate to a URL, you MUST complete the task on the current page only.
 - Do NOT navigate away from the current page on your own initiative (e.g., do not click links that lead to other pages, do not use browser back/forward, do not open new URLs).
 - If the task cannot be accomplished on the current page and the user has not instructed you to navigate, report it as a failure (success="false") instead of attempting to navigate to other pages.
-` : ''}
+
 ### Output Rules
 
-- If the task is NOT complete, skip this section and continue to Step ${actionStepNumber}.
+- If the task is NOT complete, skip this section and continue to Step 3.
 - Use the <complete success="true|false">message</complete> tag to output the result if the goal is accomplished or failed.
-  - the 'success' attribute is required. ${shouldIncludeSubGoals ? 'It means whether the expected goal is accomplished based on what you observe in the current screenshot. ' : ''}No matter what actions were executed or what errors occurred during execution, if the ${shouldIncludeSubGoals ? 'expected goal is accomplished' : 'instruction is fulfilled'}, set success="true". If the ${shouldIncludeSubGoals ? 'expected goal is not accomplished and cannot be accomplished' : 'instruction is not fulfilled and cannot be fulfilled'}, set success="false".
+  - the 'success' attribute is required. No matter what actions were executed or what errors occurred during execution, if the instruction is fulfilled, set success="true". If the instruction is not fulfilled and cannot be fulfilled, set success="false".
   - the 'message' is the information that will be provided to the user. If the user asks for a specific format, strictly follow that.
 - If you output <complete>, do NOT output <action-type> or <action-param-json>. The task ends here.
 
-## Step ${actionStepNumber}: Determine Next Action (related tags: <log>, <action-type>, <action-param-json>, <error>)
+## Step 3: Determine Next Action (related tags: <log>, <action-type>, <action-param-json>, <error>)
 
-ONLY if the task is not complete: Think what the next action is according to the current screenshot${shouldIncludeSubGoals ? ' and the plan' : ''}.
+ONLY if the task is not complete: Think what the next action is according to the current screenshot.
 
 - Don't give extra actions or plans beyond the instruction or the plan. For example, don't try to submit the form if the instruction is only to fill something.
 - Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.
@@ -272,26 +199,15 @@ For example:
 Return in XML format following this decision flow:
 
 **Always include (REQUIRED):**
-<!-- Step 1: Observe${shouldIncludeSubGoals ? ' and Plan' : ''} -->
+<!-- Step 1: Observe -->
 <thought>Your thought process here. NEVER skip this tag.</thought>
-${shouldIncludeSubGoals ? `
-<!-- required when no update-plan-content is provided in the previous response -->
-<update-plan-content>...</update-plan-content>
-
-<!-- required when any sub-goal is completed -->
-<mark-sub-goal-done>
-  <sub-goal index="1" status="finished" />
-</mark-sub-goal-done>
-` : ''}${shouldIncludeSubGoals ? `
-<!-- Step ${memoryStepNumber}: Memory data from current screenshot if needed -->
-<memory>...</memory>
-` : ''}
+
 **Then choose ONE of the following paths:**
 
-**Path A: If the ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} or failed (Step ${checkGoalStepNumber})**
+**Path A: If the instruction is fulfilled or failed (Step 2)**
 <complete success="true|false">...</complete>
 
-**Path B: If the ${shouldIncludeSubGoals ? 'goal is NOT complete' : 'instruction is NOT fulfilled'} yet (Step ${actionStepNumber})**
+**Path B: If the instruction is NOT fulfilled yet (Step 3)**
 <!-- Determine next action -->
 <log>...</log>
 <action-type>...</action-type>
@@ -299,137 +215,7 @@ ${shouldIncludeSubGoals ? `
 
 <!-- OR if there's an error -->
 <error>...</error>
-${shouldIncludeSubGoals ? `
-## Multi-turn Conversation Example
-
-Below is an example of a multi-turn conversation for "fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address":
-
-### Turn 1 - Initial instruction
-
-**User message:**
-<user_instruction>fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address</user_instruction>
-
-**Screenshot:** [Shows a registration form with empty Name and Email fields]
-
-**Your response:**
-<thought>The user wants me to fill out the registration form with specific values and return the email address. I can see the form has two fields: Name and Email. Both are currently empty. I'll break this down into sub-goals and start with the Name field. Note: The instruction is to fill the form only (not submit), and return the email at the end.</thought>
-<update-plan-content>
-  <sub-goal index="1" status="pending">Fill in the Name field with 'John'</sub-goal>
-  <sub-goal index="2" status="pending">Fill in the Email field with 'john@example.com'</sub-goal>
-  <sub-goal index="3" status="pending">Return the filled email address</sub-goal>
-</update-plan-content>
-<log>Click on the Name field to start filling the form</log>
-<action-type>Tap</action-type>
-<action-param-json>
-{
-  "locate": ${locateNameField}
-}
-</action-param-json>
 
-### Turn 2 - After clicking Name field
-
-**User message:**
-The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.
-
-Sub-goals:
-1. Fill in the Name field with 'John' (running)
-2. Fill in the Email field with 'john@example.com' (pending)
-3. Return the filled email address (pending)
-Current sub-goal is: Fill in the Name field with 'John'
-Actions performed for current sub-goal:
-- Click on the Name field to start filling the form
-
-**Screenshot:** [Shows the form with Name field now focused/active]
-
-**Your response:**
-<thought>The Name field is now focused. I need to type 'John' into this field. Current sub-goal is running, will be completed after input.</thought>
-<log>Typing 'John' into the Name field</log>
-<action-type>Input</action-type>
-<action-param-json>
-{
-  "value": "John"
-}
-</action-param-json>
-
-### Turn 3 - After entering name
-
-**User message:**
-The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.
-
-Sub-goals:
-1. Fill in the Name field with 'John' (running)
-2. Fill in the Email field with 'john@example.com' (pending)
-3. Return the filled email address (pending)
-Current sub-goal is: Fill in the Name field with 'John'
-Actions performed for current sub-goal:
-- Click on the Name field to start filling the form
-- Typing 'John' into the Name field
-
-**Screenshot:** [Shows the form with Name field containing 'John']
-
-**Your response:**
-<thought>The Name field now shows 'John'. Sub-goal 1 is complete. Next, I need to fill the Email field for sub-goal 2.</thought>
-<mark-sub-goal-done>
-  <sub-goal index="1" status="finished" />
-</mark-sub-goal-done>
-<memory>Name field has been filled with 'John'</memory>
-<log>Moving to the Email field</log>
-<action-type>Tap</action-type>
-<action-param-json>
-{
-  "locate": ${locateEmailField}
-}
-</action-param-json>
-
-### Turn 4 - After clicking Email field
-
-**User message:**
-The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.
-
-Sub-goals:
-1. Fill in the Name field with 'John' (finished)
-2. Fill in the Email field with 'john@example.com' (running)
-3. Return the filled email address (pending)
-Current sub-goal is: Fill in the Email field with 'john@example.com'
-Actions performed for current sub-goal:
-- Moving to the Email field
-
-**Screenshot:** [Shows the form with Name='John' and Email field focused]
-
-**Your response:**
-<thought>The Email field is now focused. I'll enter 'john@example.com'. After this, sub-goal 2 will be complete and my task will be done.</thought>
-<log>Typing email address into the Email field</log>
-<action-type>Input</action-type>
-<action-param-json>
-{
-  "value": "john@example.com"
-}
-</action-param-json>
-
-### Turn 5 - After entering email (Goal accomplished)
-
-**User message:**
-The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.
-
-Sub-goals:
-1. Fill in the Name field with 'John' (finished)
-2. Fill in the Email field with 'john@example.com' (running)
-3. Return the filled email address (pending)
-Current sub-goal is: Fill in the Email field with 'john@example.com'
-Actions performed for current sub-goal:
-- Moving to the Email field
-- Typing email address into the Email field
-
-**Screenshot:** [Shows the form with Name='John' and Email='john@example.com']
-
-**Your response:**
-<thought>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. Sub-goal 2 is complete. The user asked me to return the filled email address, so I need to include 'john@example.com' in my response. All sub-goals are now finished.</thought>
-<mark-sub-goal-done>
-  <sub-goal index="2" status="finished" />
-  <sub-goal index="3" status="finished" />
-</mark-sub-goal-done>
-<complete success="true">john@example.com</complete>
-` : `
 ## Multi-turn Conversation Example
 
 Below is an example of a multi-turn conversation for "fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address":
@@ -512,7 +298,7 @@ The previous action has been executed, here is the latest screenshot. Please con
 **Your response:**
 <thought>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. The user asked me to return the filled email address, so I should include 'john@example.com' in my response. The instruction has been fulfilled.</thought>
 <complete success="true">john@example.com</complete>
-`}`;
+`;
 }
 export { descriptionForAction, systemPromptToTaskPlanning };