@midscene/core 1.10.8-beta-20260724035136.0 → 1.10.8

package/dist/es/ai-model/llm-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport type {\n  LocateResultAdapter,\n  LocateResultContext,\n} from './shared/model-locate-result';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport { normalizePlanningActionLocateFields } from './workflows/planning/locate-normalization';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  // Use <planning> instead of <thought> to avoid colliding with Gemini thought\n  // summaries, which may also be emitted as <thought> in OpenAI-compatible\n  // response content.\n  const thought = extractXMLTag(xmlString, 'planning');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultAdapter?: LocateResultAdapter;\n  locateResultContext: LocateResultContext;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultAdapter,\n    locateResultContext,\n  } = options;\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(messages, modelRuntime, {\n        abortSignal,\n        requiresOriginalImageDetail: includeLocateInPlanning,\n      }),\n    parseResponse: (response) => {\n      const planFromAI = parseXMLPlanningResponse(\n        response.content,\n        modelRuntime.adapter.jsonParser,\n      );\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      // Keep yamlFlow based on the model's original action params. Coordinate\n      // normalization adds runtime-only locatedPixelBbox fields afterwards.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultAdapter,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultAdapter,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: [\n      {\n        type: 'text',\n        text: rawResponse,\n      },\n    ],\n  });\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultAdapter","locateResultContext","callAiAndParseWithRetry","callAI","response","planFromAI","actions","yamlFlow","buildYamlFlowFromPlans","normalizePlanningActionLocateFields","parseError","errorMessage","String","AIResponseParseError","plan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","assert","index"],"mappings":";;;;;;;;;;AA4BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAKtB,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,cAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,cAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,qBAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,yBAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAcA,eAAeO,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,mBAAmB,EACnBC,mBAAmB,EACpB,GAAGP;IACJ,OAAOQ,wBAAwB;QAC7B,QAAQ,IACNC,OAAOR,UAAUC,cAAc;gBAC7BC;gBACA,6BAA6BC;YAC/B;QACF,eAAe,CAACM;YACd,MAAMC,aAAatC,yBACjBqC,SAAS,OAAO,EAChBR,aAAa,OAAO,CAAC,UAAU;YAEjC,IAAIS,WAAW,MAAM,IAAIA,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;gBACjExC,QACE;gBAEFwC,WAAW,eAAe,GAAGxB;gBAC7BwB,WAAW,eAAe,GAAGxB;YAC/B;YAEA,MAAMyB,UAAUD,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAG5D,MAAME,WAAWC,uBAAuBF,SAASP;YACjDU,oCAAoCH,SAAS;gBAC3CP;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEG;gBAAUC;gBAAYC;gBAASC;YAAS;QACnD;QACA,cAAc,CAACG,YAAYN;YACzB,MAAMO,eACJD,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBR,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACa;YACb/C,MACE,mDACA+C,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMpB,eAAeoB,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGvB;IACpB,MAAM,EAAEwB,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACpB,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIJ,MACR8B,4CAA4C1B,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,sBACJJ,aAAa,MAAM,CAAC,WAAW,IAAIuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BtC;IAGN,MAAM0C,kBAAkBP,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMQ,eAAe,MAAMC,2BAA2B;QACpD,aAAaT,KAAK,WAAW;QAC7B,kBAAkBhB,qBAAqB;QACvC,yBAAyBgB,KAAK,uBAAuB;QACrD,gBAAgB;QAChBO;IACF;IAEA,MAAMG,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaN;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMS,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,mBAAmBf;IAC/C,MAAMgB,gBAAgBf,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMgB,yBAAyBhB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMiB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BL,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpD,uBAAuB;IAGpC,MAAMuE,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAV,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFV,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDvC,UAAU,EACVC,OAAO,EACPC,QAAQ,EACT,GAAG,MAAMd,6BAA6B;QACrC,UAAU+C;QACV5C;QACA,aAAaoB,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BhB;QACA,qBAAqB;YACnB,cAAc0B,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;IACF;IAEA,IAAImB,yBAAyB;IAG7B,IAAIxC,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;QAC5C1C,MAAM;QACNkF,yBAAyB;QAEzB,IAAItB,iBACFL,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAGzC,UAAU;QACbC;QACAmC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAE,OAAO1C,YAAY;IAGnB,IAAIkB,iBAAiB;QACnB,IAAIlB,WAAW,cAAc,EAAE,QAC7Ba,oBAAoB,aAAa,CAACb,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM2C,SAAS3C,WAAW,mBAAmB,CAChDa,oBAAoB,mBAAmB,CAAC8B;QAI5C,IAAI3C,WAAW,GAAG,EAChBa,oBAAoB,gBAAgB,CAACb,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBa,oBAAoB,mBAAmB,CAACb,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBa,oBAAoB,YAAY,CAACb,WAAW,MAAM;IAGpDa,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAEA,OAAOK;AACT"}
+{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport type {\n  LocateResultAdapter,\n  LocateResultContext,\n} from './shared/model-locate-result';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport { normalizePlanningActionLocateFields } from './workflows/planning/locate-normalization';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  // Use <planning> instead of <thought> to avoid colliding with Gemini thought\n  // summaries, which may also be emitted as <thought> in OpenAI-compatible\n  // response content.\n  const thought = extractXMLTag(xmlString, 'planning');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultAdapter?: LocateResultAdapter;\n  locateResultContext: LocateResultContext;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultAdapter,\n    locateResultContext,\n  } = options;\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(messages, modelRuntime, {\n        abortSignal,\n        requiresOriginalImageDetail: includeLocateInPlanning,\n      }),\n    parseResponse: (response) => {\n      const planFromAI = parseXMLPlanningResponse(\n        response.content,\n        modelRuntime.adapter.jsonParser,\n      );\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      // Keep yamlFlow based on the model's original action params. Coordinate\n      // normalization adds runtime-only locatedPixelBbox fields afterwards.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultAdapter,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultAdapter,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultAdapter","locateResultContext","callAiAndParseWithRetry","callAI","response","planFromAI","actions","yamlFlow","buildYamlFlowFromPlans","normalizePlanningActionLocateFields","parseError","errorMessage","String","AIResponseParseError","plan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","assert","index"],"mappings":";;;;;;;;;;AA4BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAKtB,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,cAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,cAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,qBAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,yBAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAcA,eAAeO,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,mBAAmB,EACnBC,mBAAmB,EACpB,GAAGP;IACJ,OAAOQ,wBAAwB;QAC7B,QAAQ,IACNC,OAAOR,UAAUC,cAAc;gBAC7BC;gBACA,6BAA6BC;YAC/B;QACF,eAAe,CAACM;YACd,MAAMC,aAAatC,yBACjBqC,SAAS,OAAO,EAChBR,aAAa,OAAO,CAAC,UAAU;YAEjC,IAAIS,WAAW,MAAM,IAAIA,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;gBACjExC,QACE;gBAEFwC,WAAW,eAAe,GAAGxB;gBAC7BwB,WAAW,eAAe,GAAGxB;YAC/B;YAEA,MAAMyB,UAAUD,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAG5D,MAAME,WAAWC,uBAAuBF,SAASP;YACjDU,oCAAoCH,SAAS;gBAC3CP;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEG;gBAAUC;gBAAYC;gBAASC;YAAS;QACnD;QACA,cAAc,CAACG,YAAYN;YACzB,MAAMO,eACJD,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBR,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACa;YACb/C,MACE,mDACA+C,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMpB,eAAeoB,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGvB;IACpB,MAAM,EAAEwB,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACpB,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIJ,MACR8B,4CAA4C1B,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,sBACJJ,aAAa,MAAM,CAAC,WAAW,IAAIuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BtC;IAGN,MAAM0C,kBAAkBP,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMQ,eAAe,MAAMC,2BAA2B;QACpD,aAAaT,KAAK,WAAW;QAC7B,kBAAkBhB,qBAAqB;QACvC,yBAAyBgB,KAAK,uBAAuB;QACrD,gBAAgB;QAChBO;IACF;IAEA,MAAMG,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaN;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMS,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,mBAAmBf;IAC/C,MAAMgB,gBAAgBf,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMgB,yBAAyBhB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMiB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BL,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpD,uBAAuB;IAGpC,MAAMuE,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAV,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFV,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDvC,UAAU,EACVC,OAAO,EACPC,QAAQ,EACT,GAAG,MAAMd,6BAA6B;QACrC,UAAU+C;QACV5C;QACA,aAAaoB,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BhB;QACA,qBAAqB;YACnB,cAAc0B,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;IACF;IAEA,IAAImB,yBAAyB;IAG7B,IAAIxC,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;QAC5C1C,MAAM;QACNkF,yBAAyB;QAEzB,IAAItB,iBACFL,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAGzC,UAAU;QACbC;QACAmC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAE,OAAO1C,YAAY;IAGnB,IAAIkB,iBAAiB;QACnB,IAAIlB,WAAW,cAAc,EAAE,QAC7Ba,oBAAoB,aAAa,CAACb,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM2C,SAAS3C,WAAW,mBAAmB,CAChDa,oBAAoB,mBAAmB,CAAC8B;QAI5C,IAAI3C,WAAW,GAAG,EAChBa,oBAAoB,gBAAgB,CAACb,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBa,oBAAoB,mBAAmB,CAACb,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBa,oBAAoB,YAAY,CAACb,WAAW,MAAM;IAMpD,IACET,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7DgD,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOK;AACT"}

package/dist/es/ai-model/model-adapter/chat-completion.mjs

@@ -35,6 +35,7 @@ function resolveChatCompletion(chatCompletion) {
     const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];
     const extractContentAndReasoning = resolveExtractContentAndReasoning(chatCompletion);
     const useReasoningAsContentFallback = chatCompletion?.useReasoningAsContentFallback ?? false;
+    const replayRawAssistantMessage = chatCompletion?.replayRawAssistantMessage ?? false;
     return {
         unsupportedUserConfig,
         buildChatCompletionParams: (input)=>{
@@ -51,7 +52,8 @@ function resolveChatCompletion(chatCompletion) {
                 midsceneDefaults: midsceneChatCompletionDefaults
             }),
         extractContentAndReasoning,
-        useReasoningAsContentFallback
+        useReasoningAsContentFallback,
+        replayRawAssistantMessage
     };
 }
 export { resolveChatCompletion };

package/dist/es/ai-model/model-adapter/chat-completion.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/chat-completion.mjs","sources":["../../../../src/ai-model/model-adapter/chat-completion.ts"],"sourcesContent":["import type {\n  ChatCompletionAdapter,\n  ChatCompletionCallContext,\n  ExtractContentAndReasoning,\n  MidsceneChatCompletionDefaults,\n  ModelAdapterDefinition,\n} from './types';\n\nconst defaultImageDetail = (_input: unknown) => undefined;\n\nconst defaultChatCompletionParams = ({\n  midsceneDefaults,\n  userConfig,\n}: ChatCompletionCallContext) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n  temperature: 0,\n};\n\nfunction createDefaultExtractContentAndReasoning(\n  reasoningContentKeys: string[],\n): ExtractContentAndReasoning {\n  return (message) => {\n    if (!message) {\n      return {\n        content: '',\n        reasoning_content: '',\n      };\n    }\n\n    const messageRecord = message as Record<string, unknown>;\n    const rawReasoning = reasoningContentKeys\n      .map((key) => messageRecord[key])\n      .find((value) => typeof value === 'string');\n    const messageReasoning =\n      typeof rawReasoning === 'string' ? rawReasoning : '';\n\n    return {\n      content: typeof message.content === 'string' ? message.content : '',\n      reasoning_content: messageReasoning,\n    };\n  };\n}\n\nfunction resolveExtractContentAndReasoning(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ExtractContentAndReasoning {\n  const messageExtraction = chatCompletion?.messageExtraction;\n  if (messageExtraction?.kind === 'custom') {\n    return messageExtraction.extractContentAndReasoning;\n  }\n\n  return createDefaultExtractContentAndReasoning(\n    messageExtraction?.reasoningContentKeys ?? ['reasoning_content'],\n  );\n}\n\nexport function resolveChatCompletion(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n  const buildChatCompletionParams =\n    chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n  const resolveImageDetail =\n    chatCompletion?.resolveImageDetail ?? defaultImageDetail;\n  const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n  const extractContentAndReasoning =\n    resolveExtractContentAndReasoning(chatCompletion);\n  const useReasoningAsContentFallback =\n    chatCompletion?.useReasoningAsContentFallback ?? false;\n\n  return {\n    unsupportedUserConfig,\n    buildChatCompletionParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      };\n      return buildChatCompletionParams(context);\n    },\n    resolveImageDetail: (input) =>\n      resolveImageDetail({\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      }),\n    extractContentAndReasoning,\n    useReasoningAsContentFallback,\n  };\n}\n"],"names":["defaultImageDetail","_input","undefined","defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","createDefaultExtractContentAndReasoning","reasoningContentKeys","message","messageRecord","rawReasoning","key","value","messageReasoning","resolveExtractContentAndReasoning","chatCompletion","messageExtraction","resolveChatCompletion","buildChatCompletionParams","resolveImageDetail","unsupportedUserConfig","extractContentAndReasoning","useReasoningAsContentFallback","input","context"],"mappings":"AAQA,MAAMA,qBAAqB,CAACC,SAAoBC;AAEhD,MAAMC,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEA,SAASC,wCACPC,oBAA8B;IAE9B,OAAO,CAACC;QACN,IAAI,CAACA,SACH,OAAO;YACL,SAAS;YACT,mBAAmB;QACrB;QAGF,MAAMC,gBAAgBD;QACtB,MAAME,eAAeH,qBAClB,GAAG,CAAC,CAACI,MAAQF,aAAa,CAACE,IAAI,EAC/B,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA;QAC1B,MAAMC,mBACJ,AAAwB,YAAxB,OAAOH,eAA4BA,eAAe;QAEpD,OAAO;YACL,SAAS,AAA2B,YAA3B,OAAOF,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;YACjE,mBAAmBK;QACrB;IACF;AACF;AAEA,SAASC,kCACPC,cAAwD;IAExD,MAAMC,oBAAoBD,gBAAgB;IAC1C,IAAIC,mBAAmB,SAAS,UAC9B,OAAOA,kBAAkB,0BAA0B;IAGrD,OAAOV,wCACLU,mBAAmB,wBAAwB;QAAC;KAAoB;AAEpE;AAEO,SAASC,sBACdF,cAAwD;IAExD,MAAMG,4BACJH,gBAAgB,6BAA6Bb;IAC/C,MAAMiB,qBACJJ,gBAAgB,sBAAsBhB;IACxC,MAAMqB,wBAAwBL,gBAAgB,yBAAyB,EAAE;IACzE,MAAMM,6BACJP,kCAAkCC;IACpC,MAAMO,gCACJP,gBAAgB,iCAAiC;IAEnD,OAAO;QACLK;QACA,2BAA2B,CAACG;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBlB;YACpB;YACA,OAAOa,0BAA0BM;QACnC;QACA,oBAAoB,CAACD,QACnBJ,mBAAmB;gBACjB,GAAGI,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBlB;YACpB;QACFgB;QACAC;IACF;AACF"}
+{"version":3,"file":"ai-model/model-adapter/chat-completion.mjs","sources":["../../../../src/ai-model/model-adapter/chat-completion.ts"],"sourcesContent":["import type {\n  ChatCompletionAdapter,\n  ChatCompletionCallContext,\n  ExtractContentAndReasoning,\n  MidsceneChatCompletionDefaults,\n  ModelAdapterDefinition,\n} from './types';\n\nconst defaultImageDetail = (_input: unknown) => undefined;\n\nconst defaultChatCompletionParams = ({\n  midsceneDefaults,\n  userConfig,\n}: ChatCompletionCallContext) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n  temperature: 0,\n};\n\nfunction createDefaultExtractContentAndReasoning(\n  reasoningContentKeys: string[],\n): ExtractContentAndReasoning {\n  return (message) => {\n    if (!message) {\n      return {\n        content: '',\n        reasoning_content: '',\n      };\n    }\n\n    const messageRecord = message as Record<string, unknown>;\n    const rawReasoning = reasoningContentKeys\n      .map((key) => messageRecord[key])\n      .find((value) => typeof value === 'string');\n    const messageReasoning =\n      typeof rawReasoning === 'string' ? rawReasoning : '';\n\n    return {\n      content: typeof message.content === 'string' ? message.content : '',\n      reasoning_content: messageReasoning,\n    };\n  };\n}\n\nfunction resolveExtractContentAndReasoning(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ExtractContentAndReasoning {\n  const messageExtraction = chatCompletion?.messageExtraction;\n  if (messageExtraction?.kind === 'custom') {\n    return messageExtraction.extractContentAndReasoning;\n  }\n\n  return createDefaultExtractContentAndReasoning(\n    messageExtraction?.reasoningContentKeys ?? ['reasoning_content'],\n  );\n}\n\nexport function resolveChatCompletion(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n  const buildChatCompletionParams =\n    chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n  const resolveImageDetail =\n    chatCompletion?.resolveImageDetail ?? defaultImageDetail;\n  const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n  const extractContentAndReasoning =\n    resolveExtractContentAndReasoning(chatCompletion);\n  const useReasoningAsContentFallback =\n    chatCompletion?.useReasoningAsContentFallback ?? false;\n  const replayRawAssistantMessage =\n    chatCompletion?.replayRawAssistantMessage ?? false;\n\n  return {\n    unsupportedUserConfig,\n    buildChatCompletionParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      };\n      return buildChatCompletionParams(context);\n    },\n    resolveImageDetail: (input) =>\n      resolveImageDetail({\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      }),\n    extractContentAndReasoning,\n    useReasoningAsContentFallback,\n    replayRawAssistantMessage,\n  };\n}\n"],"names":["defaultImageDetail","_input","undefined","defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","createDefaultExtractContentAndReasoning","reasoningContentKeys","message","messageRecord","rawReasoning","key","value","messageReasoning","resolveExtractContentAndReasoning","chatCompletion","messageExtraction","resolveChatCompletion","buildChatCompletionParams","resolveImageDetail","unsupportedUserConfig","extractContentAndReasoning","useReasoningAsContentFallback","replayRawAssistantMessage","input","context"],"mappings":"AAQA,MAAMA,qBAAqB,CAACC,SAAoBC;AAEhD,MAAMC,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEA,SAASC,wCACPC,oBAA8B;IAE9B,OAAO,CAACC;QACN,IAAI,CAACA,SACH,OAAO;YACL,SAAS;YACT,mBAAmB;QACrB;QAGF,MAAMC,gBAAgBD;QACtB,MAAME,eAAeH,qBAClB,GAAG,CAAC,CAACI,MAAQF,aAAa,CAACE,IAAI,EAC/B,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA;QAC1B,MAAMC,mBACJ,AAAwB,YAAxB,OAAOH,eAA4BA,eAAe;QAEpD,OAAO;YACL,SAAS,AAA2B,YAA3B,OAAOF,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;YACjE,mBAAmBK;QACrB;IACF;AACF;AAEA,SAASC,kCACPC,cAAwD;IAExD,MAAMC,oBAAoBD,gBAAgB;IAC1C,IAAIC,mBAAmB,SAAS,UAC9B,OAAOA,kBAAkB,0BAA0B;IAGrD,OAAOV,wCACLU,mBAAmB,wBAAwB;QAAC;KAAoB;AAEpE;AAEO,SAASC,sBACdF,cAAwD;IAExD,MAAMG,4BACJH,gBAAgB,6BAA6Bb;IAC/C,MAAMiB,qBACJJ,gBAAgB,sBAAsBhB;IACxC,MAAMqB,wBAAwBL,gBAAgB,yBAAyB,EAAE;IACzE,MAAMM,6BACJP,kCAAkCC;IACpC,MAAMO,gCACJP,gBAAgB,iCAAiC;IACnD,MAAMQ,4BACJR,gBAAgB,6BAA6B;IAE/C,OAAO;QACLK;QACA,2BAA2B,CAACI;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBnB;YACpB;YACA,OAAOa,0BAA0BO;QACnC;QACA,oBAAoB,CAACD,QACnBL,mBAAmB;gBACjB,GAAGK,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBnB;YACpB;QACFgB;QACAC;QACAC;IACF;AACF"}

package/dist/es/ai-model/models/kimi.mjs

@@ -40,6 +40,24 @@ const buildKimiChatCompletionParams = (input)=>{
         }
     };
 };
+const buildKimi3ChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const commonOverrideConfig = {};
+    commonOverrideConfig.temperature = void 0;
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
+    const reasoningEffort = userConfig.reasoningEffort;
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            ...reasoningEffort ? {
+                reasoning_effort: reasoningEffort
+            } : {}
+        }
+    };
+};
 const kimiAdapters = {
     kimi: {
         chatCompletion: {
@@ -56,6 +74,23 @@ const kimiAdapters = {
                 parseRawLocateValue: parseKimiRawLocateValue
             }
         }
+    },
+    kimi3: {
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEnabled',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: buildKimi3ChatCompletionParams,
+            useReasoningAsContentFallback: true,
+            replayRawAssistantMessage: true
+        },
+        locate: {
+            resultAdapter: {
+                coordinates: kimiNormalizedPointCoordinatesMeta,
+                parseRawLocateValue: parseKimiRawLocateValue
+            }
+        }
     }
 };
 export { kimiAdapters };

package/dist/es/ai-model/models/kimi.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/kimi.mjs","sources":["../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  parseCoordinateList,\n} from '../shared/model-locate-result';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1,\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n  const point = parseCoordinateList(input, 'point');\n  if (point.length < 2) {\n    throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n  }\n  const [x, y] = point;\n  // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n  // normalized coordinates, otherwise they are treated as screenshot pixels.\n  const coordinatesMeta =\n    x <= 1 && y <= 1\n      ? kimiNormalizedPointCoordinatesMeta\n      : kimiPixelPointCoordinatesMeta;\n  return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // kimi disallow custom temperature\n  commonOverrideConfig.temperature = undefined;\n\n  // Kimi Chat Completions response_format:\n  // https://platform.kimi.com/docs/api/chat\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const kimiAdapters = {\n  kimi: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimiChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'kimi'>;\n"],"names":["kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","kimiAdapters"],"mappings":";AAYA,MAAMA,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;AACT;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,oBAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,wBAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C;QACnD,UAAU;YACR,MAAMH,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BR;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAaZ;gBACb,qBAAqBE;YACvB;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/kimi.mjs","sources":["../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  parseCoordinateList,\n} from '../shared/model-locate-result';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1,\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n  const point = parseCoordinateList(input, 'point');\n  if (point.length < 2) {\n    throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n  }\n  const [x, y] = point;\n  // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n  // normalized coordinates, otherwise they are treated as screenshot pixels.\n  const coordinatesMeta =\n    x <= 1 && y <= 1\n      ? kimiNormalizedPointCoordinatesMeta\n      : kimiPixelPointCoordinatesMeta;\n  return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // kimi disallow custom temperature\n  commonOverrideConfig.temperature = undefined;\n\n  // Kimi Chat Completions response_format:\n  // https://platform.kimi.com/docs/api/chat\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst buildKimi3ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // Kimi disallows custom temperature.\n  commonOverrideConfig.temperature = undefined;\n\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  // Kimi K3 currently only supports reasoning_effort=\"max\"; its docs say\n  // additional effort levels will be available later.\n  const reasoningEffort = userConfig.reasoningEffort;\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}),\n    },\n  };\n};\n\nexport const kimiAdapters = {\n  kimi: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimiChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n  kimi3: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEnabled', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimi3ChatCompletionParams,\n      useReasoningAsContentFallback: true,\n      // Kimi K3 multi-turn and tool calls must replay the complete assistant\n      // message, including reasoning_content and tool_calls.\n      // https://platform.kimi.com/docs/guide/use-thinking-effort\n      replayRawAssistantMessage: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'kimi' | 'kimi3'\n>;\n"],"names":["kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","buildKimi3ChatCompletionParams","reasoningEffort","kimiAdapters"],"mappings":";AAYA,MAAMA,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;AACT;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,oBAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,wBAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C;QACnD,UAAU;YACR,MAAMH,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,iCAAiC,CACrCjB;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAMc,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAEnC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAK/D,MAAMI,kBAAkBP,WAAW,eAAe;IAElD,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAII,kBAAkB;gBAAE,kBAAkBA;YAAgB,IAAI,CAAC,CAAC;QAClE;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BV;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAaZ;gBACb,qBAAqBE;YACvB;QACF;IACF;IACA,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBAAC;gBAAoB;aAAkB;YAC9D,2BAA2BkB;YAC3B,+BAA+B;YAI/B,2BAA2B;QAC7B;QACA,QAAQ;YACN,eAAe;gBACb,aAAapB;gBACb,qBAAqBE;YACvB;QACF;IACF;AACF"}

package/dist/es/ai-model/prompt/extraction.mjs

@@ -135,14 +135,16 @@ By viewing the screenshot and page contents, you can extract the following data:
 </data-json>
 `;
 }
-const extractDataQueryPrompt = (pageDescription, dataQuery)=>{
+const extractDataQueryPrompt = (pageDescription, dataQuery, context)=>{
     let dataQueryText = '';
     dataQueryText = 'string' == typeof dataQuery ? dataQuery : JSON.stringify(dataQuery, null, 2);
+    const trimmedContext = context?.trim();
+    const contextSection = trimmedContext ? `\n<CONTEXT>\n${trimmedContext}\n</CONTEXT>\n` : '';
     return `
 <PageDescription>
 ${pageDescription}
 </PageDescription>
-
+${contextSection}
 <DATA_DEMAND>
 ${dataQueryText}
 </DATA_DEMAND>

package/dist/es/ai-model/prompt/extraction.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/extraction.mjs","sources":["../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { parseModelResponseJson } from '../service-caller/json';\nimport { extractXMLTag } from './util';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  // Keep the internal field named `thought`, but ask models to emit\n  // <observation>. Gemini may only return <thought>-named content when\n  // thinking summaries are enabled.\n  const thought = extractXMLTag(xmlString, 'observation');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = parseModelResponseJson(dataJsonStr, {\n      source: 'generic-object',\n      requireObject: false,\n    }) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = parseModelResponseJson(errorsStr, {\n        source: 'generic-object',\n        requireObject: false,\n      });\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract(options?: {\n  screenshotIncluded?: boolean;\n  referenceImagesIncluded?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n  const screenshotIncluded = options?.screenshotIncluded ?? true;\n  const referenceImagesIncluded = options?.referenceImagesIncluded ?? false;\n\n  const contextPrompts = [\n    \"The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\",\n  ];\n\n  if (screenshotIncluded) {\n    contextPrompts.push(\n      'The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.',\n    );\n  } else {\n    contextPrompts.push(\n      'The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.',\n    );\n  }\n\n  if (referenceImagesIncluded) {\n    const referenceImagesPrompt =\n      'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';\n    contextPrompts.push(\n      screenshotIncluded\n        ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.`\n        : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`,\n    );\n  }\n  const contextPrompt = contextPrompts.join('\\n\\n');\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\n${contextPrompt}\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\n\nReturn in the following XML format:\n<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","parseModelResponseJson","e","errors","parsedErrors","Array","systemPromptToExtract","options","preferredLanguage","getPreferredLanguage","screenshotIncluded","referenceImagesIncluded","contextPrompts","referenceImagesPrompt","contextPrompt","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;AAKO,SAASA,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAKO,SAASE,2BACdC,SAAiB;IAKjB,MAAMC,UAAUC,cAAcF,WAAW;IACzC,MAAMG,cAAcD,cAAcF,WAAW;IAC7C,MAAMI,YAAYF,cAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,uBAAuBJ,aAAa;YACzC,QAAQ;YACR,eAAe;QACjB;IACF,EAAE,OAAOK,GAAG;QACV,MAAM,IAAIH,MAAM,CAAC,2BAA2B,EAAEG,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIL,WACF,IAAI;QACF,MAAMM,eAAeH,uBAAuBH,WAAW;YACrD,QAAQ;YACR,eAAe;QACjB;QACA,IAAIO,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIP,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAIG,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG,sBAAsBC,OAGrC;IACC,MAAMC,oBAAoBC;IAC1B,MAAMC,qBAAqBH,SAAS,sBAAsB;IAC1D,MAAMI,0BAA0BJ,SAAS,2BAA2B;IAEpE,MAAMK,iBAAiB;QACrB;KACD;IAED,IAAIF,oBACFE,eAAe,IAAI,CACjB;SAGFA,eAAe,IAAI,CACjB;IAIJ,IAAID,yBAAyB;QAC3B,MAAME,wBACJ;QACFD,eAAe,IAAI,CACjBF,qBACI,GAAGG,sBAAsB,mLAAmL,CAAC,GAC7M,GAAGA,sBAAsB,kIAAkI,CAAC;IAEpK;IACA,MAAMC,gBAAgBF,eAAe,IAAI,CAAC;IAE1C,OAAO,CAAC;;;AAGV,EAAEE,cAAc;;;;;;;;kFAQkE,EAAEN,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEtG,CAAC;AACD;AAEO,MAAMO,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}
+{"version":3,"file":"ai-model/prompt/extraction.mjs","sources":["../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { parseModelResponseJson } from '../service-caller/json';\nimport { extractXMLTag } from './util';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  // Keep the internal field named `thought`, but ask models to emit\n  // <observation>. Gemini may only return <thought>-named content when\n  // thinking summaries are enabled.\n  const thought = extractXMLTag(xmlString, 'observation');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = parseModelResponseJson(dataJsonStr, {\n      source: 'generic-object',\n      requireObject: false,\n    }) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = parseModelResponseJson(errorsStr, {\n        source: 'generic-object',\n        requireObject: false,\n      });\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract(options?: {\n  screenshotIncluded?: boolean;\n  referenceImagesIncluded?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n  const screenshotIncluded = options?.screenshotIncluded ?? true;\n  const referenceImagesIncluded = options?.referenceImagesIncluded ?? false;\n\n  const contextPrompts = [\n    \"The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\",\n  ];\n\n  if (screenshotIncluded) {\n    contextPrompts.push(\n      'The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.',\n    );\n  } else {\n    contextPrompts.push(\n      'The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.',\n    );\n  }\n\n  if (referenceImagesIncluded) {\n    const referenceImagesPrompt =\n      'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';\n    contextPrompts.push(\n      screenshotIncluded\n        ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.`\n        : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`,\n    );\n  }\n  const contextPrompt = contextPrompts.join('\\n\\n');\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\n${contextPrompt}\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\n\nReturn in the following XML format:\n<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n  context?: string,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  const trimmedContext = context?.trim();\n  const contextSection = trimmedContext\n    ? `\\n<CONTEXT>\\n${trimmedContext}\\n</CONTEXT>\\n`\n    : '';\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n${contextSection}\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","parseModelResponseJson","e","errors","parsedErrors","Array","systemPromptToExtract","options","preferredLanguage","getPreferredLanguage","screenshotIncluded","referenceImagesIncluded","contextPrompts","referenceImagesPrompt","contextPrompt","extractDataQueryPrompt","pageDescription","dataQuery","context","dataQueryText","JSON","trimmedContext","contextSection"],"mappings":";;;AAKO,SAASA,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAKO,SAASE,2BACdC,SAAiB;IAKjB,MAAMC,UAAUC,cAAcF,WAAW;IACzC,MAAMG,cAAcD,cAAcF,WAAW;IAC7C,MAAMI,YAAYF,cAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,uBAAuBJ,aAAa;YACzC,QAAQ;YACR,eAAe;QACjB;IACF,EAAE,OAAOK,GAAG;QACV,MAAM,IAAIH,MAAM,CAAC,2BAA2B,EAAEG,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIL,WACF,IAAI;QACF,MAAMM,eAAeH,uBAAuBH,WAAW;YACrD,QAAQ;YACR,eAAe;QACjB;QACA,IAAIO,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIP,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAIG,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG,sBAAsBC,OAGrC;IACC,MAAMC,oBAAoBC;IAC1B,MAAMC,qBAAqBH,SAAS,sBAAsB;IAC1D,MAAMI,0BAA0BJ,SAAS,2BAA2B;IAEpE,MAAMK,iBAAiB;QACrB;KACD;IAED,IAAIF,oBACFE,eAAe,IAAI,CACjB;SAGFA,eAAe,IAAI,CACjB;IAIJ,IAAID,yBAAyB;QAC3B,MAAME,wBACJ;QACFD,eAAe,IAAI,CACjBF,qBACI,GAAGG,sBAAsB,mLAAmL,CAAC,GAC7M,GAAGA,sBAAsB,kIAAkI,CAAC;IAEpK;IACA,MAAMC,gBAAgBF,eAAe,IAAI,CAAC;IAE1C,OAAO,CAAC;;;AAGV,EAAEE,cAAc;;;;;;;;kFAQkE,EAAEN,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEtG,CAAC;AACD;AAEO,MAAMO,yBAAyB,CACpCC,iBACAC,WACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOF,YACOA,YAEAG,KAAK,SAAS,CAACH,WAAW,MAAM;IAGlD,MAAMI,iBAAiBH,SAAS;IAChC,MAAMI,iBAAiBD,iBACnB,CAAC,aAAa,EAAEA,eAAe,cAAc,CAAC,GAC9C;IAEJ,OAAO,CAAC;;AAEV,EAAEL,gBAAgB;;AAElB,EAAEM,eAAe;;AAEjB,EAAEH,cAAc;;EAEd,CAAC;AACH"}

package/dist/es/ai-model/service-caller/codex-app-server.mjs

@@ -52,6 +52,7 @@ const toNonEmptyString = (value)=>{
     const trimmed = value.trim();
     return trimmed || void 0;
 };
+const toCodexImageDetail = (value)=>'auto' === value || 'low' === value || 'high' === value || 'original' === value ? value : void 0;
 const normalizeCodexLocalImagePath = (imageUrl, platform = process.platform)=>{
     if (!imageUrl.startsWith('file://')) return imageUrl;
     try {
@@ -80,7 +81,7 @@ const extractTextFromMessage = (message)=>{
     }).filter(Boolean).join('\n');
     return '';
 };
-const extractImageInputs = (message)=>{
+const extractImageInputs = (message, imageDetailOverride)=>{
     const content = message.content;
     if (!Array.isArray(content)) return [];
     const inputs = [];
@@ -88,32 +89,37 @@ const extractImageInputs = (message)=>{
         if (!part || 'object' != typeof part) continue;
         const partType = String(part.type || '');
         const imageUrl = 'image_url' === partType ? toNonEmptyString(part.image_url?.url) : 'input_image' === partType ? toNonEmptyString(part.image_url || part.url) : void 0;
+        const imageDetail = imageDetailOverride ?? toCodexImageDetail('image_url' === partType ? part.image_url?.detail : part.detail);
         if (imageUrl) {
             if (imageUrl.startsWith('/') || imageUrl.startsWith('./') || imageUrl.startsWith('../') || imageUrl.startsWith('file://')) {
                 const path = imageUrl.startsWith('file://') ? normalizeCodexLocalImagePath(imageUrl) : imageUrl;
                 inputs.push({
                     type: 'localImage',
-                    path
+                    path,
+                    ...imageDetail ? {
+                        detail: imageDetail
+                    } : {}
                 });
                 continue;
             }
             inputs.push({
                 type: 'image',
-                url: imageUrl
+                url: imageUrl,
+                ...imageDetail ? {
+                    detail: imageDetail
+                } : {}
             });
         }
     }
     return inputs;
 };
 const resolveCodexReasoningEffort = ({ reasoningEnabled, modelConfig })=>{
-    if (true === reasoningEnabled) return 'high';
-    if (false === reasoningEnabled) return 'none';
-    if ('default' === reasoningEnabled) return;
+    if (true !== reasoningEnabled) return 'none';
     const normalized = modelConfig.reasoningEffort?.trim().toLowerCase();
     if ('none' === normalized || 'minimal' === normalized || 'low' === normalized || 'medium' === normalized || 'high' === normalized || 'xhigh' === normalized) return normalized;
-    return 'none';
+    return 'medium';
 };
-const buildCodexTurnPayloadFromMessages = (messages)=>{
+const buildCodexTurnPayloadFromMessages = (messages, imageDetailOverride)=>{
     const developerInstructionParts = [];
     const transcriptParts = [];
     const imageInputs = [];
@@ -127,7 +133,7 @@ const buildCodexTurnPayloadFromMessages = (messages)=>{
         const roleTag = role.toUpperCase();
         if (text.trim()) transcriptParts.push(`[${roleTag}]\n${text.trim()}`);
         else transcriptParts.push(`[${roleTag}]\n(no text content)`);
-        if ('user' === role) imageInputs.push(...extractImageInputs(message));
+        if ('user' === role) imageInputs.push(...extractImageInputs(message, imageDetailOverride));
     }
     const fullTranscript = transcriptParts.join('\n\n');
     const transcriptText = (fullTranscript.length > CODEX_TEXT_INPUT_MAX_LENGTH ? fullTranscript.slice(-CODEX_TEXT_INPUT_MAX_LENGTH) : fullTranscript) || 'Please answer the latest user request.';
@@ -175,12 +181,12 @@ class CodexAppServerConnection {
     isClosed() {
         return this.closed;
     }
-    async runTurn({ messages, modelConfig, stream, onChunk, reasoningEnabled, abortSignal }) {
+    async runTurn({ messages, modelConfig, stream, onChunk, reasoningEnabled, abortSignal, imageDetail }) {
         const startTime = Date.now();
         const timeoutMs = modelConfig.timeout || CODEX_DEFAULT_TIMEOUT_MS;
         const deadlineAt = Date.now() + timeoutMs;
         const isStreaming = !!(stream && onChunk);
-        const { developerInstructions, input } = buildCodexTurnPayloadFromMessages(messages);
+        const { developerInstructions, input } = buildCodexTurnPayloadFromMessages(messages, imageDetail);
         const effort = resolveCodexReasoningEffort({
             reasoningEnabled,
             modelConfig
@@ -518,7 +524,7 @@ class CodexAppServerConnection {
     }
 }
 class CodexAppServerConnectionManager {
-    async runTurn({ messages, modelConfig, stream, onChunk, reasoningEnabled, abortSignal }) {
+    async runTurn({ messages, modelConfig, stream, onChunk, reasoningEnabled, abortSignal, imageDetail }) {
         return this.runner.run(async ()=>{
             const connection = await this.getConnection();
             try {
@@ -528,7 +534,8 @@ class CodexAppServerConnectionManager {
                     stream,
                     onChunk,
                     reasoningEnabled,
-                    abortSignal
+                    abortSignal,
+                    imageDetail
                 });
             } catch (error) {
                 if (connection.isClosed() || !isAbortError(error)) await this.resetConnection();
@@ -567,7 +574,8 @@ async function callAIWithCodexAppServer(messages, modelConfig, options) {
         stream: options?.stream,
         onChunk: options?.onChunk,
         reasoningEnabled: options?.reasoningEnabled,
-        abortSignal: options?.abortSignal
+        abortSignal: options?.abortSignal,
+        imageDetail: options?.imageDetail
     });
 }
 async function __shutdownCodexAppServerForTests() {