@midscene/core 1.13.4-beta-20260928072823.0 → 1.14.0

package/dist/es/ai-model/workflows/planning/conversation-history.mjs

@@ -16,14 +16,21 @@ class ConversationHistory {
     resetPendingFeedbackMessageIfExists() {
         if (this.pendingFeedbackMessage) this.pendingFeedbackMessage = '';
     }
-    append(message) {
-        this.messages.push(message);
+    appendMessage(message) {
+        this.messages.push({
+            type: 'input-message',
+            message
+        });
+    }
+    appendModelOutput(output) {
+        this.messages.push({
+            type: 'model-output',
+            output
+        });
     }
     seed(messages) {
         this.reset();
-        messages.forEach((message)=>{
-            this.append(message);
-        });
+        this.messages.push(...messages);
     }
     reset() {
         this.messages.length = 0;
@@ -39,11 +46,12 @@ class ConversationHistory {
         const clonedMessages = structuredClone(this.messages);
         let imageCount = 0;
         for(let i = clonedMessages.length - 1; i >= 0; i--){
-            const message = clonedMessages[i];
-            const content = message.content;
+            const entry = clonedMessages[i];
+            if ('input-message' !== entry.type) continue;
+            const content = entry.message.content;
             if (Array.isArray(content)) for(let j = 0; j < content.length; j++){
                 const item = content[j];
-                if ('image_url' === item.type) {
+                if ('image' === item.type) {
                     imageCount++;
                     if (imageCount > maxImages) content[j] = {
                         type: 'text',
@@ -166,7 +174,7 @@ class ConversationHistory {
         };
         const recentMessages = this.messages.slice(-keepCount);
         this.messages.length = 0;
-        this.messages.push(omittedPlaceholder);
+        this.appendMessage(omittedPlaceholder);
         for (const msg of recentMessages)this.messages.push(msg);
         return true;
     }

package/dist/es/ai-model/workflows/planning/conversation-history.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/conversation-history.mjs","sources":["../../../../../src/ai-model/workflows/planning/conversation-history.ts"],"sourcesContent":["import type { SubGoal } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildSubGoalsText } from '../../prompt/planning/sub-goals-text';\n\nexport interface ConversationHistoryOptions {\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly messages: ChatCompletionMessageParam[] = [];\n  private subGoals: SubGoal[] = [];\n  private memories: string[] = [];\n  private historicalLogs: string[] = [];\n\n  public pendingFeedbackMessage: string;\n\n  constructor(options?: ConversationHistoryOptions) {\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n    this.pendingFeedbackMessage = '';\n  }\n\n  resetPendingFeedbackMessageIfExists() {\n    if (this.pendingFeedbackMessage) {\n      this.pendingFeedbackMessage = '';\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n    this.memories.length = 0;\n    this.subGoals.length = 0;\n    this.historicalLogs.length = 0;\n    this.pendingFeedbackMessage = '';\n  }\n\n  /**\n   * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n   * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n   * @returns The snapshot of the conversation history.\n   */\n  snapshot(maxImages?: number): ChatCompletionMessageParam[] {\n    if (maxImages === undefined) {\n      return [...this.messages];\n    }\n\n    const clonedMessages = structuredClone(this.messages);\n    let imageCount = 0;\n\n    // Traverse from the end to the beginning\n    for (let i = clonedMessages.length - 1; i >= 0; i--) {\n      const message = clonedMessages[i];\n      const content = message.content;\n\n      // Only process if content is an array\n      if (Array.isArray(content)) {\n        for (let j = 0; j < content.length; j++) {\n          const item = content[j];\n\n          // Check if this is an image\n          if (item.type === 'image_url') {\n            imageCount++;\n\n            // If we've exceeded the limit, replace with text\n            if (imageCount > maxImages) {\n              content[j] = {\n                type: 'text',\n                text: '(image ignored due to size optimization)',\n              };\n            }\n          }\n        }\n      }\n    }\n\n    return clonedMessages;\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n\n  // Planning sub-goal management methods\n\n  /**\n   * Set all sub-goals, replacing any existing ones.\n   * Automatically marks the first pending goal as running.\n   */\n  setSubGoals(subGoals: SubGoal[]): void {\n    this.subGoals = subGoals.map((goal) => ({ ...goal }));\n    this.markFirstPendingAsRunning();\n  }\n\n  /**\n   * Merge sub-goals from update-plan-content.\n   * Preserves existing descriptions when incoming description is empty.\n   *\n   * This handles compact XML updates like:\n   * <sub-goal index=\"1\" status=\"finished\" />\n   */\n  mergeSubGoals(subGoals: SubGoal[]): void {\n    if (this.subGoals.length === 0) {\n      this.setSubGoals(subGoals);\n      return;\n    }\n\n    const existingByIndex = new Map(\n      this.subGoals.map((goal) => [goal.index, goal] as const),\n    );\n\n    const mergedSubGoals = subGoals.map((goal) => {\n      const existingGoal = existingByIndex.get(goal.index);\n      const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n      if (!existingGoal && !hasNonEmptyDescription) {\n        return null;\n      }\n\n      return {\n        ...goal,\n        description:\n          hasNonEmptyDescription || !existingGoal\n            ? goal.description\n            : existingGoal.description,\n      };\n    });\n\n    const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n    if (validSubGoals.length === 0) {\n      return;\n    }\n\n    this.setSubGoals(validSubGoals);\n  }\n\n  /**\n   * Update a single sub-goal by index.\n   * Clears logs if status or description actually changes.\n   * @returns true if the sub-goal was found and updated, false otherwise\n   */\n  updateSubGoal(\n    index: number,\n    updates: Partial<Omit<SubGoal, 'index'>>,\n  ): boolean {\n    const goal = this.subGoals.find((g) => g.index === index);\n    if (!goal) {\n      return false;\n    }\n\n    let changed = false;\n\n    if (updates.status !== undefined && updates.status !== goal.status) {\n      goal.status = updates.status;\n      changed = true;\n    }\n    if (\n      updates.description !== undefined &&\n      updates.description !== goal.description\n    ) {\n      goal.description = updates.description;\n      changed = true;\n    }\n\n    if (changed) {\n      goal.logs = [];\n    }\n\n    return true;\n  }\n\n  /**\n   * Mark the first pending sub-goal as running.\n   * Clears logs since status changes.\n   */\n  markFirstPendingAsRunning(): void {\n    const firstPending = this.subGoals.find((g) => g.status === 'pending');\n    if (firstPending) {\n      firstPending.status = 'running';\n      firstPending.logs = [];\n    }\n  }\n\n  /**\n   * Mark a sub-goal as finished.\n   * Automatically marks the next pending goal as running.\n   * @returns true if the sub-goal was found and updated, false otherwise\n   */\n  markSubGoalFinished(index: number): boolean {\n    const result = this.updateSubGoal(index, { status: 'finished' });\n    if (result) {\n      this.markFirstPendingAsRunning();\n    }\n    return result;\n  }\n\n  /**\n   * Mark all sub-goals as finished.\n   * Clears logs for any goal whose status actually changes.\n   */\n  markAllSubGoalsFinished(): void {\n    for (const goal of this.subGoals) {\n      if (goal.status !== 'finished') {\n        goal.logs = [];\n      }\n      goal.status = 'finished';\n    }\n  }\n\n  /**\n   * Append a log entry to the currently running sub-goal.\n   * The log describes an action performed while working on the sub-goal.\n   */\n  appendSubGoalLog(log: string): void {\n    if (!log) {\n      return;\n    }\n    const runningGoal = this.subGoals.find((g) => g.status === 'running');\n    if (runningGoal) {\n      if (!runningGoal.logs) {\n        runningGoal.logs = [];\n      }\n      runningGoal.logs.push(log);\n    }\n  }\n\n  /**\n   * Convert sub-goals to text representation.\n   * Includes actions performed (logs) for the current sub-goal.\n   */\n  subGoalsToText(): string {\n    return buildSubGoalsText(this.subGoals);\n  }\n\n  // Historical log management methods (used in non-deepThink mode)\n\n  /**\n   * Append a log entry to the historical logs list.\n   * Used in non-deepThink mode to track executed steps across planning rounds.\n   */\n  appendHistoricalLog(log: string): void {\n    if (log) {\n      this.historicalLogs.push(log);\n    }\n  }\n\n  /**\n   * Convert historical logs to text representation.\n   * Provides context about previously executed steps to the model.\n   */\n  historicalLogsToText(): string {\n    if (this.historicalLogs.length === 0) {\n      return '';\n    }\n\n    const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n    return `Here are the steps that have been executed:\\n${logLines}`;\n  }\n\n  // Memory management methods\n\n  /**\n   * Append a memory to the memories list\n   */\n  appendMemory(memory: string): void {\n    if (memory) {\n      this.memories.push(memory);\n    }\n  }\n\n  /**\n   * Get all memories\n   */\n  getMemories(): string[] {\n    return [...this.memories];\n  }\n\n  /**\n   * Convert memories to text representation\n   */\n  memoriesToText(): string {\n    if (this.memories.length === 0) {\n      return '';\n    }\n\n    return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n  }\n\n  /**\n   * Clear all memories\n   */\n  clearMemories(): void {\n    this.memories.length = 0;\n  }\n\n  /**\n   * Compress the conversation history if it exceeds the threshold.\n   * Removes the oldest messages and replaces them with a single placeholder message.\n   * @param threshold - The number of messages that triggers compression.\n   * @param keepCount - The number of recent messages to keep after compression.\n   * @returns true if compression was performed, false otherwise.\n   */\n  compressHistory(threshold: number, keepCount: number): boolean {\n    if (this.messages.length <= threshold) {\n      return false;\n    }\n\n    const omittedCount = this.messages.length - keepCount;\n    const omittedPlaceholder: ChatCompletionMessageParam = {\n      role: 'user',\n      content: `(${omittedCount} previous conversation messages have been omitted)`,\n    };\n\n    // Keep only the last `keepCount` messages\n    const recentMessages = this.messages.slice(-keepCount);\n\n    // Reset and rebuild with placeholder + recent messages\n    this.messages.length = 0;\n    this.messages.push(omittedPlaceholder);\n    for (const msg of recentMessages) {\n      this.messages.push(msg);\n    }\n\n    return true;\n  }\n}\n"],"names":["Symbol","ConversationHistory","message","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","buildSubGoalsText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options"],"mappings":";;;;;;;;;;;;eA8FGA,OAAO,QAAQ;;AAtFX,MAAMC;IAeX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASE,SAAkB,EAAgC;QACzD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMP,UAAUI,cAAc,CAACG,EAAE;YACjC,MAAMC,UAAUR,QAAQ,OAAO;YAG/B,IAAIS,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,gBAAdA,KAAK,IAAI,EAAkB;oBAC7BL;oBAGA,IAAIA,aAAaJ,WACfM,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAON;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACN,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYc,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBlB,WAAnBkB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBlB,WAAxBkB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,OAAOE,kBAAkB,IAAI,CAAC,QAAQ;IACxC;IAQA,oBAAoBF,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMG,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEG,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAAiD;YACrD,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IAvUA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAyC,EAAE;QAC5D,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAmUF"}
+{"version":3,"file":"ai-model/workflows/planning/conversation-history.mjs","sources":["../../../../../src/ai-model/workflows/planning/conversation-history.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type { RawAssistantOutput, SubGoal } from '@/types';\nimport { buildSubGoalsText } from '../../prompt/planning/sub-goals-text';\nimport type { ConversationEntry } from '../../service-caller/types';\n\nexport interface ConversationHistoryOptions {\n  initialMessages?: ConversationEntry[];\n}\n\nexport class ConversationHistory {\n  private readonly messages: ConversationEntry[] = [];\n  private subGoals: SubGoal[] = [];\n  private memories: string[] = [];\n  private historicalLogs: string[] = [];\n\n  public pendingFeedbackMessage: string;\n\n  constructor(options?: ConversationHistoryOptions) {\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n    this.pendingFeedbackMessage = '';\n  }\n\n  resetPendingFeedbackMessageIfExists() {\n    if (this.pendingFeedbackMessage) {\n      this.pendingFeedbackMessage = '';\n    }\n  }\n\n  appendMessage(message: ConversationMessage) {\n    this.messages.push({ type: 'input-message', message });\n  }\n\n  appendModelOutput(output: RawAssistantOutput) {\n    this.messages.push({ type: 'model-output', output });\n  }\n\n  seed(messages: ConversationEntry[]) {\n    this.reset();\n    this.messages.push(...messages);\n  }\n\n  reset() {\n    this.messages.length = 0;\n    this.memories.length = 0;\n    this.subGoals.length = 0;\n    this.historicalLogs.length = 0;\n    this.pendingFeedbackMessage = '';\n  }\n\n  /**\n   * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n   * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n   * @returns The snapshot of the conversation history.\n   */\n  snapshot(maxImages?: number): ConversationEntry[] {\n    if (maxImages === undefined) {\n      return [...this.messages];\n    }\n\n    const clonedMessages = structuredClone(this.messages);\n    let imageCount = 0;\n\n    // Traverse from the end to the beginning\n    for (let i = clonedMessages.length - 1; i >= 0; i--) {\n      const entry = clonedMessages[i];\n      if (entry.type !== 'input-message') {\n        continue;\n      }\n      const content = entry.message.content;\n\n      // Only process if content is an array\n      if (Array.isArray(content)) {\n        for (let j = 0; j < content.length; j++) {\n          const item = content[j];\n\n          // Check if this is an image\n          if (item.type === 'image') {\n            imageCount++;\n\n            // If we've exceeded the limit, replace with text\n            if (imageCount > maxImages) {\n              content[j] = {\n                type: 'text',\n                text: '(image ignored due to size optimization)',\n              };\n            }\n          }\n        }\n      }\n    }\n\n    return clonedMessages;\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ConversationEntry> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ConversationEntry[] {\n    return this.snapshot();\n  }\n\n  // Planning sub-goal management methods\n\n  /**\n   * Set all sub-goals, replacing any existing ones.\n   * Automatically marks the first pending goal as running.\n   */\n  setSubGoals(subGoals: SubGoal[]): void {\n    this.subGoals = subGoals.map((goal) => ({ ...goal }));\n    this.markFirstPendingAsRunning();\n  }\n\n  /**\n   * Merge sub-goals from update-plan-content.\n   * Preserves existing descriptions when incoming description is empty.\n   *\n   * This handles compact XML updates like:\n   * <sub-goal index=\"1\" status=\"finished\" />\n   */\n  mergeSubGoals(subGoals: SubGoal[]): void {\n    if (this.subGoals.length === 0) {\n      this.setSubGoals(subGoals);\n      return;\n    }\n\n    const existingByIndex = new Map(\n      this.subGoals.map((goal) => [goal.index, goal] as const),\n    );\n\n    const mergedSubGoals = subGoals.map((goal) => {\n      const existingGoal = existingByIndex.get(goal.index);\n      const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n      if (!existingGoal && !hasNonEmptyDescription) {\n        return null;\n      }\n\n      return {\n        ...goal,\n        description:\n          hasNonEmptyDescription || !existingGoal\n            ? goal.description\n            : existingGoal.description,\n      };\n    });\n\n    const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n    if (validSubGoals.length === 0) {\n      return;\n    }\n\n    this.setSubGoals(validSubGoals);\n  }\n\n  /**\n   * Update a single sub-goal by index.\n   * Clears logs if status or description actually changes.\n   * @returns true if the sub-goal was found and updated, false otherwise\n   */\n  updateSubGoal(\n    index: number,\n    updates: Partial<Omit<SubGoal, 'index'>>,\n  ): boolean {\n    const goal = this.subGoals.find((g) => g.index === index);\n    if (!goal) {\n      return false;\n    }\n\n    let changed = false;\n\n    if (updates.status !== undefined && updates.status !== goal.status) {\n      goal.status = updates.status;\n      changed = true;\n    }\n    if (\n      updates.description !== undefined &&\n      updates.description !== goal.description\n    ) {\n      goal.description = updates.description;\n      changed = true;\n    }\n\n    if (changed) {\n      goal.logs = [];\n    }\n\n    return true;\n  }\n\n  /**\n   * Mark the first pending sub-goal as running.\n   * Clears logs since status changes.\n   */\n  markFirstPendingAsRunning(): void {\n    const firstPending = this.subGoals.find((g) => g.status === 'pending');\n    if (firstPending) {\n      firstPending.status = 'running';\n      firstPending.logs = [];\n    }\n  }\n\n  /**\n   * Mark a sub-goal as finished.\n   * Automatically marks the next pending goal as running.\n   * @returns true if the sub-goal was found and updated, false otherwise\n   */\n  markSubGoalFinished(index: number): boolean {\n    const result = this.updateSubGoal(index, { status: 'finished' });\n    if (result) {\n      this.markFirstPendingAsRunning();\n    }\n    return result;\n  }\n\n  /**\n   * Mark all sub-goals as finished.\n   * Clears logs for any goal whose status actually changes.\n   */\n  markAllSubGoalsFinished(): void {\n    for (const goal of this.subGoals) {\n      if (goal.status !== 'finished') {\n        goal.logs = [];\n      }\n      goal.status = 'finished';\n    }\n  }\n\n  /**\n   * Append a log entry to the currently running sub-goal.\n   * The log describes an action performed while working on the sub-goal.\n   */\n  appendSubGoalLog(log: string): void {\n    if (!log) {\n      return;\n    }\n    const runningGoal = this.subGoals.find((g) => g.status === 'running');\n    if (runningGoal) {\n      if (!runningGoal.logs) {\n        runningGoal.logs = [];\n      }\n      runningGoal.logs.push(log);\n    }\n  }\n\n  /**\n   * Convert sub-goals to text representation.\n   * Includes actions performed (logs) for the current sub-goal.\n   */\n  subGoalsToText(): string {\n    return buildSubGoalsText(this.subGoals);\n  }\n\n  // Historical log management methods (used in non-deepThink mode)\n\n  /**\n   * Append a log entry to the historical logs list.\n   * Used in non-deepThink mode to track executed steps across planning rounds.\n   */\n  appendHistoricalLog(log: string): void {\n    if (log) {\n      this.historicalLogs.push(log);\n    }\n  }\n\n  /**\n   * Convert historical logs to text representation.\n   * Provides context about previously executed steps to the model.\n   */\n  historicalLogsToText(): string {\n    if (this.historicalLogs.length === 0) {\n      return '';\n    }\n\n    const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n    return `Here are the steps that have been executed:\\n${logLines}`;\n  }\n\n  // Memory management methods\n\n  /**\n   * Append a memory to the memories list\n   */\n  appendMemory(memory: string): void {\n    if (memory) {\n      this.memories.push(memory);\n    }\n  }\n\n  /**\n   * Get all memories\n   */\n  getMemories(): string[] {\n    return [...this.memories];\n  }\n\n  /**\n   * Convert memories to text representation\n   */\n  memoriesToText(): string {\n    if (this.memories.length === 0) {\n      return '';\n    }\n\n    return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n  }\n\n  /**\n   * Clear all memories\n   */\n  clearMemories(): void {\n    this.memories.length = 0;\n  }\n\n  /**\n   * Compress the conversation history if it exceeds the threshold.\n   * Removes the oldest messages and replaces them with a single placeholder message.\n   * @param threshold - The number of messages that triggers compression.\n   * @param keepCount - The number of recent messages to keep after compression.\n   * @returns true if compression was performed, false otherwise.\n   */\n  compressHistory(threshold: number, keepCount: number): boolean {\n    if (this.messages.length <= threshold) {\n      return false;\n    }\n\n    const omittedCount = this.messages.length - keepCount;\n    const omittedPlaceholder: ConversationMessage = {\n      role: 'user',\n      content: `(${omittedCount} previous conversation messages have been omitted)`,\n    };\n\n    // Keep only the last `keepCount` messages\n    const recentMessages = this.messages.slice(-keepCount);\n\n    // Reset and rebuild with placeholder + recent messages\n    this.messages.length = 0;\n    this.appendMessage(omittedPlaceholder);\n    for (const msg of recentMessages) {\n      this.messages.push(msg);\n    }\n\n    return true;\n  }\n}\n"],"names":["Symbol","ConversationHistory","message","output","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","entry","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","buildSubGoalsText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options"],"mappings":";;;;;;;;;;;;eAoGGA,OAAO,QAAQ;;AA3FX,MAAMC;IAeX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,cAAcC,OAA4B,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM;YAAiBA;QAAQ;IACtD;IAEA,kBAAkBC,MAA0B,EAAE;QAC5C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM;YAAgBA;QAAO;IACpD;IAEA,KAAKC,QAA6B,EAAE;QAClC,IAAI,CAAC,KAAK;QACV,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAIA;IACxB;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASC,SAAkB,EAAuB;QAChD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMC,QAAQJ,cAAc,CAACG,EAAE;YAC/B,IAAIC,AAAe,oBAAfA,MAAM,IAAI,EACZ;YAEF,MAAMC,UAAUD,MAAM,OAAO,CAAC,OAAO;YAGrC,IAAIE,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,YAAdA,KAAK,IAAI,EAAc;oBACzBN;oBAGA,IAAIA,aAAaJ,WACfO,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAOP;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAyD;QACvD,OAAO,IAAI,CAAC,QAAQ,CAACP,OAAO,QAAQ,CAAC;IACvC;IAEA,SAA8B;QAC5B,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYgB,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBnB,WAAnBmB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBnB,WAAxBmB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,OAAOE,kBAAkB,IAAI,CAAC,QAAQ;IACxC;IAQA,oBAAoBF,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMG,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEG,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAA0C;YAC9C,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,aAAa,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IA5UA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAgC,EAAE;QACnD,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAwUF"}

package/dist/es/ai-model/workflows/planning/custom-planning.mjs

@@ -1,7 +1,7 @@
 import { ScreenshotItem } from "../../../screenshot-item.mjs";
 import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
 import { AIResponseParseError, callAIWithStringResponse } from "../../service-caller/index.mjs";
-import { preparedReferenceImagesToChatMessages } from "../../shared/multimodal-prompt.mjs";
+import { preparedReferenceImagesToMessages } from "../../shared/multimodal-prompt.mjs";
 import { normalizePlanningActionLocateFields } from "./locate-normalization.mjs";
 function appendActionContext(systemPrompt, actionContext) {
     return systemPrompt + (actionContext ? `${actionContext}\n` : '');
@@ -12,9 +12,9 @@ function buildCustomPlanningMessages(input, config) {
     const systemPrompt = appendActionContext(config.buildSystemPrompt(), actionContext);
     const userInstructionText = input.userInstruction.text;
     const userInstruction = config.buildUserInstruction ? config.buildUserInstruction(userInstructionText) : userInstructionText;
-    const referenceImageMessages = preparedReferenceImagesToChatMessages(input.userInstruction.referenceImages);
+    const referenceImageMessages = preparedReferenceImagesToMessages(input.userInstruction.referenceImages);
     if (conversationHistory.pendingFeedbackMessage) {
-        conversationHistory.append({
+        conversationHistory.appendMessage({
             role: 'user',
             content: [
                 {
@@ -25,14 +25,12 @@ function buildCustomPlanningMessages(input, config) {
         });
         conversationHistory.resetPendingFeedbackMessageIfExists();
     }
-    conversationHistory.append({
+    conversationHistory.appendMessage({
         role: 'user',
         content: [
             {
-                type: 'image_url',
-                image_url: {
-                    url: context.screenshot.base64
-                }
+                type: 'image',
+                url: context.screenshot.base64
             }
         ]
     });
@@ -84,7 +82,7 @@ async function runCustomPlanning(userInstruction, options, config) {
         coordinateSystem: config.coordinateSystem
     };
     const messages = buildCustomPlanningMessages(input, config.messages);
-    const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(messages, preparedOptions.modelRuntime, {
+    const { content, usage, rawAssistantOutput } = await callAIWithStringResponse(messages, preparedOptions.modelRuntime, {
         abortSignal: preparedOptions.abortSignal,
         requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning
     });
@@ -101,16 +99,16 @@ async function runCustomPlanning(userInstruction, options, config) {
             locateResultContext: {
                 preparedSize: preparedImage.preparedSize,
                 contentSize: preparedImage.contentSize
-            },
-            acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias
+            }
         });
         shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);
     } catch (parseError) {
         const errorMessage = `Parse error: ${parseError instanceof Error ? parseError.message : String(parseError)}`;
-        throw new AIResponseParseError(errorMessage, JSON.stringify(content, void 0, 2), usage, rawChoiceMessage);
+        throw new AIResponseParseError(errorMessage, JSON.stringify(content, void 0, 2), usage, rawAssistantOutput);
     }
     const assistantContent = config.messages.buildAssistantContent?.(parsed, content, input);
-    if (assistantContent) options.conversationHistory.append({
+    if (rawAssistantOutput?.type === 'responses' && options.modelRuntime.adapter.responses.replayRawAssistantOutput) options.conversationHistory.appendModelOutput(rawAssistantOutput);
+    else if (assistantContent) options.conversationHistory.appendMessage({
         role: 'assistant',
         content: assistantContent
     });
@@ -120,7 +118,7 @@ async function runCustomPlanning(userInstruction, options, config) {
         usage,
         shouldContinuePlanning,
         rawResponse: JSON.stringify(content, void 0, 2),
-        rawChoiceMessage
+        rawAssistantOutput
     };
 }
 export { buildCustomPlanningMessages, runCustomPlanning };

package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendActionContext(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return systemPrompt + (actionContext ? `${actionContext}\\n` : '');\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ChatCompletionMessageParam[] {\n  const { options } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendActionContext(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstructionText = input.userInstruction.text;\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    input.userInstruction.referenceImages,\n  );\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.append({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: { url: context.screenshot.base64 },\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...referenceImageMessages,\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...referenceImageMessages,\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: PreparedUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultCodec: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n      acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  // UI-TARS and Auto-GLM define their own assistant history format.\n  if (assistantContent) {\n    options.conversationHistory.append({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawChoiceMessage,\n  };\n}\n"],"names":["appendActionContext","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToChatMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawChoiceMessage","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAoBA,SAASA,oBACPC,YAAoB,EACpBC,aAAsB;IAEtB,OAAOD,eAAgBC,CAAAA,gBAAgB,GAAGA,cAAc,EAAE,CAAC,GAAG,EAAC;AACjE;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,oBACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,sCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBAAE,KAAKC,QAAQ,UAAU,CAAC,MAAM;gBAAC;YAC9C;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,gBAAgB,EAAE,GAAG,MAAMC,yBACjDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;YACA,mBAAmBE,gBAAgB,YAAY,CAAC,OAAO,CAAC,iBAAiB;QAC3E;QACAS,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOG,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACb,SAASc,QAAW,IACnCb,OACAC;IAEJ;IAEA,MAAMa,mBAAmB7B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IAAI8B,kBACF5B,QAAQ,mBAAmB,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,SAAS4B;IACX;IAGF,OAAO;QACLV;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaO,KAAK,SAAS,CAACb,SAASc,QAAW;QAChDZ;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendActionContext(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return systemPrompt + (actionContext ? `${actionContext}\\n` : '');\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ModelCallMessages {\n  const { options } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendActionContext(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstructionText = input.userInstruction.text;\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    input.userInstruction.referenceImages,\n  );\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.appendMessage({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.appendMessage({\n    role: 'user',\n    content: [\n      {\n        type: 'image',\n        url: context.screenshot.base64,\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...referenceImageMessages,\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...referenceImageMessages,\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: PreparedUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawAssistantOutput } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultCodec: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawAssistantOutput,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  // UI-TARS and Auto-GLM define their own assistant history format.\n  if (\n    rawAssistantOutput?.type === 'responses' &&\n    options.modelRuntime.adapter.responses.replayRawAssistantOutput\n  ) {\n    options.conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else if (assistantContent) {\n    options.conversationHistory.appendMessage({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawAssistantOutput,\n  };\n}\n"],"names":["appendActionContext","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawAssistantOutput","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAoBA,SAASA,oBACPC,YAAoB,EACpBC,aAAsB;IAEtB,OAAOD,eAAgBC,CAAAA,gBAAgB,GAAGA,cAAc,EAAE,CAAC,GAAG,EAAC;AACjE;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,oBACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,kCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,aAAa,CAAC;QAChC,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,KAAKC,QAAQ,UAAU,CAAC,MAAM;YAChC;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,kBAAkB,EAAE,GAAG,MAAMC,yBACnDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;QACF;QACAW,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOG,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACb,SAASc,QAAW,IACnCb,OACAC;IAEJ;IAEA,MAAMa,mBAAmB7B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IACEiB,oBAAoB,SAAS,eAC7Bf,QAAQ,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,wBAAwB,EAE/DA,QAAQ,mBAAmB,CAAC,iBAAiB,CAACe;SACzC,IAAIa,kBACT5B,QAAQ,mBAAmB,CAAC,aAAa,CAAC;QACxC,MAAM;QACN,SAAS4B;IACX;IAGF,OAAO;QACLV;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaO,KAAK,SAAS,CAACb,SAASc,QAAW;QAChDZ;IACF;AACF"}

package/dist/es/ai-model/workflows/planning/locate-normalization.mjs

@@ -1,6 +1,7 @@
 import { findAllMidsceneLocatorField } from "../../../common.mjs";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
+import { locateResultKeys, readLocateResultField } from "../../shared/model-locate-result/result-field.mjs";
 const debug = getDebug('planning');
 function normalizePlanningActionLocateFields(actions, { actionSpace, ...options }) {
     actions.forEach((action)=>{
@@ -16,19 +17,14 @@ function normalizePlanningActionLocateFields(actions, { actionSpace, ...options
         });
     });
 }
-function normalizePlanningLocateParameter(locateParameter, { includeLocateInPlanning, locateResultCodec, locateResultContext, acceptBbox2dAlias = false }) {
+function normalizePlanningLocateParameter(locateParameter, { includeLocateInPlanning, locateResultCodec, locateResultContext }) {
     if (!includeLocateInPlanning) return {
         prompt: locateParameter.prompt
     };
     assert(locateResultCodec, 'planning locate normalization requires a locate result codec');
-    const resultKey = locateResultCodec.promptSpec.resultKey;
-    const rawLocateValue = void 0 !== locateParameter[resultKey] ? locateParameter[resultKey] : acceptBbox2dAlias && 'bbox' === resultKey ? locateParameter.bbox_2d : void 0;
-    const rawCoordinateKeys = new Set([
-        resultKey,
-        ...acceptBbox2dAlias && 'bbox' === resultKey ? [
-            'bbox_2d'
-        ] : []
-    ]);
+    const { promptSpec } = locateResultCodec;
+    const rawLocateValue = readLocateResultField(locateParameter, promptSpec);
+    const rawCoordinateKeys = new Set(locateResultKeys(promptSpec));
     const locateParamWithoutRawCoordinates = Object.fromEntries(Object.entries(locateParameter).filter(([key])=>!rawCoordinateKeys.has(key)));
     const result = locateResultCodec.toPixelResult(rawLocateValue, locateResultContext);
     return {

package/dist/es/ai-model/workflows/planning/locate-normalization.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/locate-normalization.mjs","sources":["../../../../../src/ai-model/workflows/planning/locate-normalization.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result/types';\n\nconst debug = getDebug('planning');\n\nexport function normalizePlanningActionLocateFields(\n  actions: PlanningAction[],\n  {\n    actionSpace,\n    ...options\n  }: PlanningLocateNormalizationOptions & {\n    actionSpace: DeviceAction[];\n  },\n): void {\n  actions.forEach((action) => {\n    const actionInActionSpace = actionSpace.find(\n      (actionInSpace) => actionInSpace.name === action.type,\n    );\n    if (!actionInActionSpace) {\n      debug('skip locate normalization for action outside actionSpace', action);\n      return;\n    }\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = findAllMidsceneLocatorField(\n      actionInActionSpace.paramSchema,\n    );\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateParameter = action.param?.[field];\n      if (!locateParameter) {\n        return;\n      }\n\n      action.param[field] = normalizePlanningLocateParameter(\n        locateParameter,\n        options,\n      );\n    });\n  });\n}\n\nexport type PlanningLocateNormalizationOptions = {\n  includeLocateInPlanning: boolean;\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  acceptBbox2dAlias?: boolean;\n};\n\nexport function normalizePlanningLocateParameter(\n  locateParameter: ParsedPlanningLocateParameter,\n  {\n    includeLocateInPlanning,\n    locateResultCodec,\n    locateResultContext,\n    acceptBbox2dAlias = false,\n  }: PlanningLocateNormalizationOptions,\n): Record<string, unknown> {\n  if (!includeLocateInPlanning) {\n    // In prompt-only planning mode, ignore any accidental coordinates from the model.\n    return { prompt: locateParameter.prompt };\n  }\n\n  assert(\n    locateResultCodec,\n    'planning locate normalization requires a locate result codec',\n  );\n\n  const resultKey = locateResultCodec.promptSpec.resultKey;\n  const rawLocateValue =\n    locateParameter[resultKey] !== undefined\n      ? locateParameter[resultKey]\n      : acceptBbox2dAlias && resultKey === 'bbox'\n        ? locateParameter.bbox_2d\n        : undefined;\n\n  // The raw result field is replaced by locatedPixelResult, so it should not\n  // remain in the normalized locate parameter.\n  const rawCoordinateKeys = new Set([\n    resultKey,\n    ...(acceptBbox2dAlias && resultKey === 'bbox' ? ['bbox_2d'] : []),\n  ]);\n\n  const locateParamWithoutRawCoordinates = Object.fromEntries(\n    Object.entries(locateParameter).filter(\n      ([key]) => !rawCoordinateKeys.has(key),\n    ),\n  );\n\n  const result = locateResultCodec.toPixelResult(\n    rawLocateValue,\n    locateResultContext,\n  );\n  return {\n    ...locateParamWithoutRawCoordinates,\n    locatedPixelResult: result,\n  };\n}\n"],"names":["debug","getDebug","normalizePlanningActionLocateFields","actions","actionSpace","options","action","actionInActionSpace","actionInSpace","locateFields","findAllMidsceneLocatorField","field","locateParameter","normalizePlanningLocateParameter","includeLocateInPlanning","locateResultCodec","locateResultContext","acceptBbox2dAlias","assert","resultKey","rawLocateValue","undefined","rawCoordinateKeys","Set","locateParamWithoutRawCoordinates","Object","key","result"],"mappings":";;;AAWA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oCACdC,OAAyB,EACzB,EACEC,WAAW,EACX,GAAGC,SAGJ;IAEDF,QAAQ,OAAO,CAAC,CAACG;QACf,MAAMC,sBAAsBH,YAAY,IAAI,CAC1C,CAACI,gBAAkBA,cAAc,IAAI,KAAKF,OAAO,IAAI;QAEvD,IAAI,CAACC,qBAAqB,YACxBP,MAAM,4DAA4DM;QAIpEN,MAAM,+BAA+BO;QACrC,MAAME,eAAeC,4BACnBH,oBAAoB,WAAW;QAGjCP,MAAM,gBAAgBS;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,kBAAkBN,OAAO,KAAK,EAAE,CAACK,MAAM;YAC7C,IAAI,CAACC,iBACH;YAGFN,OAAO,KAAK,CAACK,MAAM,GAAGE,iCACpBD,iBACAP;QAEJ;IACF;AACF;AASO,SAASQ,iCACdD,eAA8C,EAC9C,EACEE,uBAAuB,EACvBC,iBAAiB,EACjBC,mBAAmB,EACnBC,oBAAoB,KAAK,EACU;IAErC,IAAI,CAACH,yBAEH,OAAO;QAAE,QAAQF,gBAAgB,MAAM;IAAC;IAG1CM,OACEH,mBACA;IAGF,MAAMI,YAAYJ,kBAAkB,UAAU,CAAC,SAAS;IACxD,MAAMK,iBACJR,AAA+BS,WAA/BT,eAAe,CAACO,UAAU,GACtBP,eAAe,CAACO,UAAU,GAC1BF,qBAAqBE,AAAc,WAAdA,YACnBP,gBAAgB,OAAO,GACvBS;IAIR,MAAMC,oBAAoB,IAAIC,IAAI;QAChCJ;WACIF,qBAAqBE,AAAc,WAAdA,YAAuB;YAAC;SAAU,GAAG,EAAE;KACjE;IAED,MAAMK,mCAAmCC,OAAO,WAAW,CACzDA,OAAO,OAAO,CAACb,iBAAiB,MAAM,CACpC,CAAC,CAACc,IAAI,GAAK,CAACJ,kBAAkB,GAAG,CAACI;IAItC,MAAMC,SAASZ,kBAAkB,aAAa,CAC5CK,gBACAJ;IAEF,OAAO;QACL,GAAGQ,gCAAgC;QACnC,oBAAoBG;IACtB;AACF"}
+{"version":3,"file":"ai-model/workflows/planning/locate-normalization.mjs","sources":["../../../../../src/ai-model/workflows/planning/locate-normalization.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport {\n  locateResultKeys,\n  readLocateResultField,\n} from '../../shared/model-locate-result/result-field';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result/types';\n\nconst debug = getDebug('planning');\n\nexport function normalizePlanningActionLocateFields(\n  actions: PlanningAction[],\n  {\n    actionSpace,\n    ...options\n  }: PlanningLocateNormalizationOptions & {\n    actionSpace: DeviceAction[];\n  },\n): void {\n  actions.forEach((action) => {\n    const actionInActionSpace = actionSpace.find(\n      (actionInSpace) => actionInSpace.name === action.type,\n    );\n    if (!actionInActionSpace) {\n      debug('skip locate normalization for action outside actionSpace', action);\n      return;\n    }\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = findAllMidsceneLocatorField(\n      actionInActionSpace.paramSchema,\n    );\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateParameter = action.param?.[field];\n      if (!locateParameter) {\n        return;\n      }\n\n      action.param[field] = normalizePlanningLocateParameter(\n        locateParameter,\n        options,\n      );\n    });\n  });\n}\n\nexport type PlanningLocateNormalizationOptions = {\n  includeLocateInPlanning: boolean;\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n};\n\nexport function normalizePlanningLocateParameter(\n  locateParameter: ParsedPlanningLocateParameter,\n  {\n    includeLocateInPlanning,\n    locateResultCodec,\n    locateResultContext,\n  }: PlanningLocateNormalizationOptions,\n): Record<string, unknown> {\n  if (!includeLocateInPlanning) {\n    // In prompt-only planning mode, ignore any accidental coordinates from the model.\n    return { prompt: locateParameter.prompt };\n  }\n\n  assert(\n    locateResultCodec,\n    'planning locate normalization requires a locate result codec',\n  );\n\n  const { promptSpec } = locateResultCodec;\n  const rawLocateValue = readLocateResultField(locateParameter, promptSpec);\n\n  // Remove primary and alias fields after converting the raw coordinates.\n  const rawCoordinateKeys = new Set(locateResultKeys(promptSpec));\n\n  const locateParamWithoutRawCoordinates = Object.fromEntries(\n    Object.entries(locateParameter).filter(\n      ([key]) => !rawCoordinateKeys.has(key),\n    ),\n  );\n\n  const result = locateResultCodec.toPixelResult(\n    rawLocateValue,\n    locateResultContext,\n  );\n  return {\n    ...locateParamWithoutRawCoordinates,\n    locatedPixelResult: result,\n  };\n}\n"],"names":["debug","getDebug","normalizePlanningActionLocateFields","actions","actionSpace","options","action","actionInActionSpace","actionInSpace","locateFields","findAllMidsceneLocatorField","field","locateParameter","normalizePlanningLocateParameter","includeLocateInPlanning","locateResultCodec","locateResultContext","assert","promptSpec","rawLocateValue","readLocateResultField","rawCoordinateKeys","Set","locateResultKeys","locateParamWithoutRawCoordinates","Object","key","result"],"mappings":";;;;AAeA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oCACdC,OAAyB,EACzB,EACEC,WAAW,EACX,GAAGC,SAGJ;IAEDF,QAAQ,OAAO,CAAC,CAACG;QACf,MAAMC,sBAAsBH,YAAY,IAAI,CAC1C,CAACI,gBAAkBA,cAAc,IAAI,KAAKF,OAAO,IAAI;QAEvD,IAAI,CAACC,qBAAqB,YACxBP,MAAM,4DAA4DM;QAIpEN,MAAM,+BAA+BO;QACrC,MAAME,eAAeC,4BACnBH,oBAAoB,WAAW;QAGjCP,MAAM,gBAAgBS;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,kBAAkBN,OAAO,KAAK,EAAE,CAACK,MAAM;YAC7C,IAAI,CAACC,iBACH;YAGFN,OAAO,KAAK,CAACK,MAAM,GAAGE,iCACpBD,iBACAP;QAEJ;IACF;AACF;AAQO,SAASQ,iCACdD,eAA8C,EAC9C,EACEE,uBAAuB,EACvBC,iBAAiB,EACjBC,mBAAmB,EACgB;IAErC,IAAI,CAACF,yBAEH,OAAO;QAAE,QAAQF,gBAAgB,MAAM;IAAC;IAG1CK,OACEF,mBACA;IAGF,MAAM,EAAEG,UAAU,EAAE,GAAGH;IACvB,MAAMI,iBAAiBC,sBAAsBR,iBAAiBM;IAG9D,MAAMG,oBAAoB,IAAIC,IAAIC,iBAAiBL;IAEnD,MAAMM,mCAAmCC,OAAO,WAAW,CACzDA,OAAO,OAAO,CAACb,iBAAiB,MAAM,CACpC,CAAC,CAACc,IAAI,GAAK,CAACL,kBAAkB,GAAG,CAACK;IAItC,MAAMC,SAASZ,kBAAkB,aAAa,CAC5CI,gBACAH;IAEF,OAAO;QACL,GAAGQ,gCAAgC;QACnC,oBAAoBG;IACtB;AACF"}

package/dist/es/ai-model/workflows/planning/standard-planning.mjs

@@ -6,7 +6,7 @@ import { buildStandardPlanningSystemPrompt } from "../../prompt/planning/index.m
 import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
 import { planningModelFamilyRequiredForLocateMessage } from "../../shared/model-locate-result/errors.mjs";
-import { preparedReferenceImagesToChatMessages } from "../../shared/multimodal-prompt.mjs";
+import { preparedReferenceImagesToMessages } from "../../shared/multimodal-prompt.mjs";
 import { parsePlanningActions } from "./parse-planning-actions.mjs";
 import { parseStandardPlanningResponse } from "./standard-planning-parser.mjs";
 const debug = getDebug('planning');
@@ -45,8 +45,7 @@ async function callAndParsePlanningResponse(options) {
                 actionSpace,
                 includeLocateInPlanning,
                 locateResultCodec,
-                locateResultContext,
-                acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias
+                locateResultContext
             });
             return {
                 response,
@@ -56,7 +55,7 @@ async function callAndParsePlanningResponse(options) {
         },
         toParseError: (parseError, response)=>{
             const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
-            return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
+            return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawAssistantOutput, response.reasoning_content);
         },
         parseRetryTimes: modelRuntime.config.retryCount,
         parseRetryInterval: modelRuntime.config.retryInterval,
@@ -101,7 +100,7 @@ async function standardPlan(userInstruction, opts) {
     });
     const imagePayload = preparedImage.imageBase64;
     const actionContext = opts.actionContext ? `${opts.actionContext}\n` : '';
-    const referenceImageMessages = preparedReferenceImagesToChatMessages(userInstruction.referenceImages);
+    const referenceImageMessages = preparedReferenceImagesToMessages(userInstruction.referenceImages);
     const instruction = [
         {
             role: 'user',
@@ -128,11 +127,8 @@ async function standardPlan(userInstruction, opts) {
                     text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`
                 },
                 {
-                    type: 'image_url',
-                    image_url: {
-                        url: imagePayload,
-                        detail: 'high'
-                    }
+                    type: 'image',
+                    url: imagePayload
                 }
             ]
         };
@@ -145,15 +141,12 @@ async function standardPlan(userInstruction, opts) {
                 text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`
             },
             {
-                type: 'image_url',
-                image_url: {
-                    url: imagePayload,
-                    detail: 'high'
-                }
+                type: 'image',
+                url: imagePayload
             }
         ]
     };
-    conversationHistory.append(latestFeedbackMessage);
+    conversationHistory.appendMessage(latestFeedbackMessage);
     conversationHistory.compressHistory(50, 20);
     const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);
     const msgs = [
@@ -164,7 +157,7 @@ async function standardPlan(userInstruction, opts) {
         ...instruction,
         ...historyLog
     ];
-    const { response: { content: rawResponse, usage, reasoning_content, rawChoiceMessage }, planFromAI, actions } = await callAndParsePlanningResponse({
+    const { response: { content: rawResponse, usage, reasoning_content, rawAssistantOutput }, planFromAI, actions } = await callAndParsePlanningResponse({
         messages: msgs,
         modelRuntime,
         abortSignal: opts.abortSignal,
@@ -189,7 +182,7 @@ async function standardPlan(userInstruction, opts) {
         ...planFromAI,
         actions,
         rawResponse,
-        rawChoiceMessage,
+        rawAssistantOutput,
         usage,
         reasoning_content,
         yamlFlow,
@@ -202,16 +195,20 @@ async function standardPlan(userInstruction, opts) {
         if (planFromAI.log) conversationHistory.appendSubGoalLog(planFromAI.log);
     } else if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
     if (planFromAI.memory) conversationHistory.appendMemory(planFromAI.memory);
-    if (modelRuntime.adapter.chatCompletion.replayRawAssistantMessage && rawChoiceMessage) conversationHistory.append(rawChoiceMessage);
-    else conversationHistory.append({
-        role: 'assistant',
-        content: [
-            {
-                type: 'text',
-                text: rawResponse
-            }
-        ]
-    });
+    const shouldReplayRawOutput = rawAssistantOutput?.type === 'responses' && modelRuntime.adapter.responses.replayRawAssistantOutput || rawAssistantOutput?.type === 'chat-completion' && modelRuntime.adapter.chatCompletion.replayRawAssistantMessage;
+    if (rawAssistantOutput && shouldReplayRawOutput) conversationHistory.appendModelOutput(rawAssistantOutput);
+    else {
+        if (!rawAssistantOutput) debug('Raw assistant output is unavailable; saving only response text in conversation history.');
+        conversationHistory.appendMessage({
+            role: 'assistant',
+            content: [
+                {
+                    type: 'text',
+                    text: rawResponse
+                }
+            ]
+        });
+    }
     return returnValue;
 }
 export { standardPlan };

package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (\n        !planFromAI.action &&\n        planFromAI.finalizeSuccess === undefined &&\n        !planFromAI.error\n      ) {\n        throw new Error(\n          'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n        );\n      }\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      parsePlanningActions(actions, {\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n        acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n      });\n      return { response, planFromAI, actions };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  // YAML is derived from validated actions outside the model-response retry scope.\n  // dumpActionParam omits runtime-only locatedPixelResult fields.\n  const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,qBAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;YAC3D;YACA,OAAO;gBAAEe;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,OACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,uBAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7DwD,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOO;AACT"}
+{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ModelCallMessages;\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (\n        !planFromAI.action &&\n        planFromAI.finalizeSuccess === undefined &&\n        !planFromAI.error\n      ) {\n        throw new Error(\n          'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n        );\n      }\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      parsePlanningActions(actions, {\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ConversationMessage[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ConversationMessage;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n  }\n  conversationHistory.appendMessage(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ModelCallMessages = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawAssistantOutput,\n    },\n    planFromAI,\n    actions,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  // YAML is derived from validated actions outside the model-response retry scope.\n  // dumpActionParam omits runtime-only locatedPixelResult fields.\n  const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  const shouldReplayRawOutput =\n    (rawAssistantOutput?.type === 'responses' &&\n      modelRuntime.adapter.responses.replayRawAssistantOutput) ||\n    (rawAssistantOutput?.type === 'chat-completion' &&\n      modelRuntime.adapter.chatCompletion.replayRawAssistantMessage);\n  if (rawAssistantOutput && shouldReplayRawOutput) {\n    conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else {\n    if (!rawAssistantOutput) {\n      debug(\n        'Raw assistant output is unavailable; saving only response text in conversation history.',\n      );\n    }\n    conversationHistory.appendMessage({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawAssistantOutput","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index","shouldReplayRawOutput"],"mappings":";;;;;;;;;;;AA8BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,qBAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEU;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,OACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,kCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAAqC;QACzC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,KAAKP;gBACP;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,KAAKP;YACP;SACD;IACH;IAEFX,oBAAoB,aAAa,CAACgB;IAGlChB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAA0B;QAC9B;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,uBAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAGpD,MAAM8C,wBACHN,oBAAoB,SAAS,eAC5BxD,aAAa,OAAO,CAAC,SAAS,CAAC,wBAAwB,IACxDwD,oBAAoB,SAAS,qBAC5BxD,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB;IACjE,IAAIwD,sBAAsBM,uBACxBhC,oBAAoB,iBAAiB,CAAC0B;SACjC;QACL,IAAI,CAACA,oBACH/D,MACE;QAGJqC,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMuB;gBACR;aACD;QACH;IACF;IAEA,OAAOO;AACT"}

package/dist/es/ai-model/workflows/recorder-generation/common.mjs

@@ -157,10 +157,8 @@ const createMessageContent = (promptText, screenshots = [], includeScreenshots =
         });
         screenshots.forEach((screenshot)=>{
             messageContent.push({
-                type: 'image_url',
-                image_url: {
-                    url: screenshot
-                }
+                type: 'image',
+                url: screenshot
             });
         });
     }