@midscene/core 1.10.1 → 1.10.2

package/dist/lib/agent/utils.js

@@ -224,7 +224,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.10.1";
+const getMidsceneVersion = ()=>"1.10.2";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/ai-model/llm-planning.js

@@ -42,7 +42,7 @@ const warnLog = (0, logger_namespaceObject.getDebug)('planning', {
 });
 const noPreviousActionsText = 'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';
 function parseXMLPlanningResponse(xmlString, jsonParser) {
-    const thought = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'thought');
+    const thought = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'planning');
     const memory = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'memory');
     const log = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'log') || '';
     const error = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'error');

package/dist/lib/ai-model/llm-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport { normalizePlanningActionLocateFields } from './workflows/planning/locate-normalization';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  let {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n    rawChoiceMessage,\n  } = await callAI(msgs, modelRuntime, {\n    abortSignal: opts.abortSignal,\n    // Planning with locate results is localization-sensitive. Adapters decide\n    // whether this should request original image detail.\n    requiresOriginalImageDetail: opts.includeLocateInPlanning,\n  });\n\n  // Parse XML response to JSON object, retry once on parse failure\n  let planFromAI: RawResponsePlanningAIResponse;\n  try {\n    try {\n      planFromAI = parseXMLPlanningResponse(rawResponse, adapter.jsonParser);\n    } catch {\n      const retry = await callAI(msgs, modelRuntime, {\n        abortSignal: opts.abortSignal,\n        // Keep retry requests consistent with the initial planning call.\n        requiresOriginalImageDetail: opts.includeLocateInPlanning,\n      });\n      rawResponse = retry.content;\n      usage = retry.usage;\n      reasoning_content = retry.reasoning_content;\n      rawChoiceMessage = retry.rawChoiceMessage;\n      planFromAI = parseXMLPlanningResponse(rawResponse, adapter.jsonParser);\n    }\n\n    if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n      warnLog(\n        'Planning response included both an action and <complete>; ignoring <complete> output.',\n      );\n      planFromAI.finalizeMessage = undefined;\n      planFromAI.finalizeSuccess = undefined;\n    }\n\n    const actions = planFromAI.action ? [planFromAI.action] : [];\n    let shouldContinuePlanning = true;\n\n    // Check if task is completed via <complete> tag\n    if (planFromAI.finalizeSuccess !== undefined) {\n      debug('task completed via <complete> tag, stop planning');\n      shouldContinuePlanning = false;\n      // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n      if (includeSubGoals) {\n        conversationHistory.markAllSubGoalsFinished();\n      }\n    }\n\n    const returnValue: PlanningAIResponse = {\n      ...planFromAI,\n      actions,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n      yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),\n      shouldContinuePlanning,\n    };\n\n    assert(planFromAI, \"can't get plans from AI\");\n\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: opts.actionSpace,\n      includeLocateInPlanning: opts.includeLocateInPlanning,\n      locateResultAdapter,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    });\n\n    // Update sub-goals in conversation history only in planning deep-think mode.\n    if (includeSubGoals) {\n      if (planFromAI.updateSubGoals?.length) {\n        conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n      }\n      if (planFromAI.markFinishedIndexes?.length) {\n        for (const index of planFromAI.markFinishedIndexes) {\n          conversationHistory.markSubGoalFinished(index);\n        }\n      }\n      // Append the planning log to the currently running sub-goal\n      if (planFromAI.log) {\n        conversationHistory.appendSubGoalLog(planFromAI.log);\n      }\n    } else {\n      // Without planning deep-think mode, accumulate logs as historical execution steps.\n      if (planFromAI.log) {\n        conversationHistory.appendHistoricalLog(planFromAI.log);\n      }\n    }\n\n    // Append memory to conversation history if present\n    if (planFromAI.memory) {\n      conversationHistory.appendMemory(planFromAI.memory);\n    }\n\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n\n    return returnValue;\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n      rawChoiceMessage,\n    );\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","plan","userInstruction","opts","context","conversationHistory","modelRuntime","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","locateResultAdapter","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","callAI","planFromAI","retry","actions","shouldContinuePlanning","returnValue","buildYamlFlowFromPlans","assert","normalizePlanningActionLocateFields","index","parseError","errorMessage","String","AIResponseParseError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACgBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAEtB,MAAMC,UAAUC,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACzC,MAAMI,SAASD,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACxC,MAAMK,MAAMF,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACvC,MAAMO,aAAaJ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,AAAAA,IAAAA,wBAAAA,oBAAAA,AAAAA,EAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,AAAAA,IAAAA,wBAAAA,wBAAAA,AAAAA,EAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAEO,eAAeO,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMG,eAAeH,KAAK,YAAY;IACtC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpB,MAAM,EAAEE,QAAQ,EAAE,GAAGJ;IACrB,MAAMK,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACG,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIN,MACRU,AAAAA,IAAAA,mCAAAA,2CAAAA,AAAAA,EAA4CJ,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMK,sBACJL,aAAa,MAAM,CAAC,WAAW,IAAIC,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BlB;IAGN,MAAMuB,kBAAkBT,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMU,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAaX,KAAK,WAAW;QAC7B,kBAAkBQ,qBAAqB;QACvC,yBAAyBR,KAAK,uBAAuB;QACrD,gBAAgB;QAChBS;IACF;IAEA,MAAMG,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBjB;IAC/C,MAAMkB,gBAAgBjB,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMkB,yBAAyBlB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMmB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BP,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMoB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BnB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAE/B,uBAAuB;IAGpC,MAAMoD,eAAerB,oBAAoB,cAAc;IACvD,MAAMsB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIrB,oBAAoB,sBAAsB,EAAE;QAC9CkB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGlB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEsB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAZ,oBAAoB,mCAAmC;IACzD,OACEkB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFZ,oBAAoB,MAAM,CAACkB;IAG3BlB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMuB,aAAavB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAM0B,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,IAAI,EACF,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOL,MAAMvB,cAAc;QACnC,aAAaH,KAAK,WAAW;QAG7B,6BAA6BA,KAAK,uBAAuB;IAC3D;IAGA,IAAIgC;IACJ,IAAI;QACF,IAAI;YACFA,aAAa5D,yBAAyBuD,aAAavB,QAAQ,UAAU;QACvE,EAAE,OAAM;YACN,MAAM6B,QAAQ,MAAMF,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOL,MAAMvB,cAAc;gBAC7C,aAAaH,KAAK,WAAW;gBAE7B,6BAA6BA,KAAK,uBAAuB;YAC3D;YACA2B,cAAcM,MAAM,OAAO;YAC3BL,QAAQK,MAAM,KAAK;YACnBJ,oBAAoBI,MAAM,iBAAiB;YAC3CH,mBAAmBG,MAAM,gBAAgB;YACzCD,aAAa5D,yBAAyBuD,aAAavB,QAAQ,UAAU;QACvE;QAEA,IAAI4B,WAAW,MAAM,IAAIA,AAA+B9C,WAA/B8C,WAAW,eAAe,EAAgB;YACjE9D,QACE;YAEF8D,WAAW,eAAe,GAAG9C;YAC7B8C,WAAW,eAAe,GAAG9C;QAC/B;QAEA,MAAMgD,UAAUF,WAAW,MAAM,GAAG;YAACA,WAAW,MAAM;SAAC,GAAG,EAAE;QAC5D,IAAIG,yBAAyB;QAG7B,IAAIH,AAA+B9C,WAA/B8C,WAAW,eAAe,EAAgB;YAC5ChE,MAAM;YACNmE,yBAAyB;YAEzB,IAAI1B,iBACFP,oBAAoB,uBAAuB;QAE/C;QAEA,MAAMkC,cAAkC;YACtC,GAAGJ,UAAU;YACbE;YACAP;YACAG;YACAF;YACAC;YACA,UAAUQ,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBH,SAASlC,KAAK,WAAW;YAC1DmC;QACF;QAEAG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAON,YAAY;QAEnBO,IAAAA,wCAAAA,mCAAAA,AAAAA,EAAoCL,SAAS;YAC3C,aAAalC,KAAK,WAAW;YAC7B,yBAAyBA,KAAK,uBAAuB;YACrDQ;YACA,qBAAqB;gBACnB,cAAcI,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;QACF;QAGA,IAAIH,iBAAiB;YACnB,IAAIuB,WAAW,cAAc,EAAE,QAC7B9B,oBAAoB,aAAa,CAAC8B,WAAW,cAAc;YAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAMQ,SAASR,WAAW,mBAAmB,CAChD9B,oBAAoB,mBAAmB,CAACsC;YAI5C,IAAIR,WAAW,GAAG,EAChB9B,oBAAoB,gBAAgB,CAAC8B,WAAW,GAAG;QAEvD,OAEE,IAAIA,WAAW,GAAG,EAChB9B,oBAAoB,mBAAmB,CAAC8B,WAAW,GAAG;QAK1D,IAAIA,WAAW,MAAM,EACnB9B,oBAAoB,YAAY,CAAC8B,WAAW,MAAM;QAGpD9B,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMyB;gBACR;aACD;QACH;QAEA,OAAOS;IACT,EAAE,OAAOK,YAAY;QAEnB,MAAMC,eACJD,sBAAsB5C,QAAQ4C,WAAW,OAAO,GAAGE,OAAOF;QAC5D,MAAM,IAAIG,yBAAAA,oBAAoBA,CAC5B,CAAC,iBAAiB,EAAEF,cAAc,EAClCf,aACAC,OACAE;IAEJ;AACF"}
+{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport { normalizePlanningActionLocateFields } from './workflows/planning/locate-normalization';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  // Use <planning> instead of <thought> to avoid colliding with Gemini thought\n  // summaries, which may also be emitted as <thought> in OpenAI-compatible\n  // response content.\n  const thought = extractXMLTag(xmlString, 'planning');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  let {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n    rawChoiceMessage,\n  } = await callAI(msgs, modelRuntime, {\n    abortSignal: opts.abortSignal,\n    // Planning with locate results is localization-sensitive. Adapters decide\n    // whether this should request original image detail.\n    requiresOriginalImageDetail: opts.includeLocateInPlanning,\n  });\n\n  // Parse XML response to JSON object, retry once on parse failure\n  let planFromAI: RawResponsePlanningAIResponse;\n  try {\n    try {\n      planFromAI = parseXMLPlanningResponse(rawResponse, adapter.jsonParser);\n    } catch {\n      const retry = await callAI(msgs, modelRuntime, {\n        abortSignal: opts.abortSignal,\n        // Keep retry requests consistent with the initial planning call.\n        requiresOriginalImageDetail: opts.includeLocateInPlanning,\n      });\n      rawResponse = retry.content;\n      usage = retry.usage;\n      reasoning_content = retry.reasoning_content;\n      rawChoiceMessage = retry.rawChoiceMessage;\n      planFromAI = parseXMLPlanningResponse(rawResponse, adapter.jsonParser);\n    }\n\n    if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n      warnLog(\n        'Planning response included both an action and <complete>; ignoring <complete> output.',\n      );\n      planFromAI.finalizeMessage = undefined;\n      planFromAI.finalizeSuccess = undefined;\n    }\n\n    const actions = planFromAI.action ? [planFromAI.action] : [];\n    let shouldContinuePlanning = true;\n\n    // Check if task is completed via <complete> tag\n    if (planFromAI.finalizeSuccess !== undefined) {\n      debug('task completed via <complete> tag, stop planning');\n      shouldContinuePlanning = false;\n      // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n      if (includeSubGoals) {\n        conversationHistory.markAllSubGoalsFinished();\n      }\n    }\n\n    const returnValue: PlanningAIResponse = {\n      ...planFromAI,\n      actions,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n      yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),\n      shouldContinuePlanning,\n    };\n\n    assert(planFromAI, \"can't get plans from AI\");\n\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: opts.actionSpace,\n      includeLocateInPlanning: opts.includeLocateInPlanning,\n      locateResultAdapter,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    });\n\n    // Update sub-goals in conversation history only in planning deep-think mode.\n    if (includeSubGoals) {\n      if (planFromAI.updateSubGoals?.length) {\n        conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n      }\n      if (planFromAI.markFinishedIndexes?.length) {\n        for (const index of planFromAI.markFinishedIndexes) {\n          conversationHistory.markSubGoalFinished(index);\n        }\n      }\n      // Append the planning log to the currently running sub-goal\n      if (planFromAI.log) {\n        conversationHistory.appendSubGoalLog(planFromAI.log);\n      }\n    } else {\n      // Without planning deep-think mode, accumulate logs as historical execution steps.\n      if (planFromAI.log) {\n        conversationHistory.appendHistoricalLog(planFromAI.log);\n      }\n    }\n\n    // Append memory to conversation history if present\n    if (planFromAI.memory) {\n      conversationHistory.appendMemory(planFromAI.memory);\n    }\n\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n\n    return returnValue;\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n      rawChoiceMessage,\n    );\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","plan","userInstruction","opts","context","conversationHistory","modelRuntime","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","locateResultAdapter","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","callAI","planFromAI","retry","actions","shouldContinuePlanning","returnValue","buildYamlFlowFromPlans","assert","normalizePlanningActionLocateFields","index","parseError","errorMessage","String","AIResponseParseError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACgBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAKtB,MAAMC,UAAUC,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACzC,MAAMI,SAASD,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACxC,MAAMK,MAAMF,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACvC,MAAMO,aAAaJ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,AAAAA,IAAAA,wBAAAA,oBAAAA,AAAAA,EAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,AAAAA,IAAAA,wBAAAA,wBAAAA,AAAAA,EAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAEO,eAAeO,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMG,eAAeH,KAAK,YAAY;IACtC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpB,MAAM,EAAEE,QAAQ,EAAE,GAAGJ;IACrB,MAAMK,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACG,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIN,MACRU,AAAAA,IAAAA,mCAAAA,2CAAAA,AAAAA,EAA4CJ,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMK,sBACJL,aAAa,MAAM,CAAC,WAAW,IAAIC,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BlB;IAGN,MAAMuB,kBAAkBT,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMU,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAaX,KAAK,WAAW;QAC7B,kBAAkBQ,qBAAqB;QACvC,yBAAyBR,KAAK,uBAAuB;QACrD,gBAAgB;QAChBS;IACF;IAEA,MAAMG,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBjB;IAC/C,MAAMkB,gBAAgBjB,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMkB,yBAAyBlB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMmB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BP,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMoB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BnB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAE/B,uBAAuB;IAGpC,MAAMoD,eAAerB,oBAAoB,cAAc;IACvD,MAAMsB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIrB,oBAAoB,sBAAsB,EAAE;QAC9CkB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGlB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEsB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAZ,oBAAoB,mCAAmC;IACzD,OACEkB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFZ,oBAAoB,MAAM,CAACkB;IAG3BlB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMuB,aAAavB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAM0B,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,IAAI,EACF,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOL,MAAMvB,cAAc;QACnC,aAAaH,KAAK,WAAW;QAG7B,6BAA6BA,KAAK,uBAAuB;IAC3D;IAGA,IAAIgC;IACJ,IAAI;QACF,IAAI;YACFA,aAAa5D,yBAAyBuD,aAAavB,QAAQ,UAAU;QACvE,EAAE,OAAM;YACN,MAAM6B,QAAQ,MAAMF,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOL,MAAMvB,cAAc;gBAC7C,aAAaH,KAAK,WAAW;gBAE7B,6BAA6BA,KAAK,uBAAuB;YAC3D;YACA2B,cAAcM,MAAM,OAAO;YAC3BL,QAAQK,MAAM,KAAK;YACnBJ,oBAAoBI,MAAM,iBAAiB;YAC3CH,mBAAmBG,MAAM,gBAAgB;YACzCD,aAAa5D,yBAAyBuD,aAAavB,QAAQ,UAAU;QACvE;QAEA,IAAI4B,WAAW,MAAM,IAAIA,AAA+B9C,WAA/B8C,WAAW,eAAe,EAAgB;YACjE9D,QACE;YAEF8D,WAAW,eAAe,GAAG9C;YAC7B8C,WAAW,eAAe,GAAG9C;QAC/B;QAEA,MAAMgD,UAAUF,WAAW,MAAM,GAAG;YAACA,WAAW,MAAM;SAAC,GAAG,EAAE;QAC5D,IAAIG,yBAAyB;QAG7B,IAAIH,AAA+B9C,WAA/B8C,WAAW,eAAe,EAAgB;YAC5ChE,MAAM;YACNmE,yBAAyB;YAEzB,IAAI1B,iBACFP,oBAAoB,uBAAuB;QAE/C;QAEA,MAAMkC,cAAkC;YACtC,GAAGJ,UAAU;YACbE;YACAP;YACAG;YACAF;YACAC;YACA,UAAUQ,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBH,SAASlC,KAAK,WAAW;YAC1DmC;QACF;QAEAG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAON,YAAY;QAEnBO,IAAAA,wCAAAA,mCAAAA,AAAAA,EAAoCL,SAAS;YAC3C,aAAalC,KAAK,WAAW;YAC7B,yBAAyBA,KAAK,uBAAuB;YACrDQ;YACA,qBAAqB;gBACnB,cAAcI,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;QACF;QAGA,IAAIH,iBAAiB;YACnB,IAAIuB,WAAW,cAAc,EAAE,QAC7B9B,oBAAoB,aAAa,CAAC8B,WAAW,cAAc;YAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAMQ,SAASR,WAAW,mBAAmB,CAChD9B,oBAAoB,mBAAmB,CAACsC;YAI5C,IAAIR,WAAW,GAAG,EAChB9B,oBAAoB,gBAAgB,CAAC8B,WAAW,GAAG;QAEvD,OAEE,IAAIA,WAAW,GAAG,EAChB9B,oBAAoB,mBAAmB,CAAC8B,WAAW,GAAG;QAK1D,IAAIA,WAAW,MAAM,EACnB9B,oBAAoB,YAAY,CAAC8B,WAAW,MAAM;QAGpD9B,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMyB;gBACR;aACD;QACH;QAEA,OAAOS;IACT,EAAE,OAAOK,YAAY;QAEnB,MAAMC,eACJD,sBAAsB5C,QAAQ4C,WAAW,OAAO,GAAGE,OAAOF;QAC5D,MAAM,IAAIG,yBAAAA,oBAAoBA,CAC5B,CAAC,iBAAiB,EAAEF,cAAc,EAClCf,aACAC,OACAE;IAEJ;AACF"}

package/dist/lib/ai-model/models/default.js

@@ -32,7 +32,8 @@ const defaultOpenAICompatibleAdapterConfig = {
             'reasoningEnabled',
             'reasoningEffort',
             'reasoningBudget'
-        ]
+        ],
+        useReasoningAsContentFallback: true
     }
 };
 exports.defaultOpenAICompatibleAdapterConfig = __webpack_exports__.defaultOpenAICompatibleAdapterConfig;

package/dist/lib/ai-model/models/default.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/default.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/models/default.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ModelAdapterDefinition } from '../model-adapter/types';\n\nexport const defaultOpenAICompatibleAdapterConfig: ModelAdapterDefinition = {\n  chatCompletion: {\n    unsupportedUserConfig: [\n      'reasoningEnabled',\n      'reasoningEffort',\n      'reasoningBudget',\n    ],\n  },\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultOpenAICompatibleAdapterConfig"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;ACJO,MAAMI,uCAA+D;IAC1E,gBAAgB;QACd,uBAAuB;YACrB;YACA;YACA;SACD;IACH;AACF"}
+{"version":3,"file":"ai-model/models/default.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/models/default.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ModelAdapterDefinition } from '../model-adapter/types';\n\nexport const defaultOpenAICompatibleAdapterConfig: ModelAdapterDefinition = {\n  chatCompletion: {\n    unsupportedUserConfig: [\n      'reasoningEnabled',\n      'reasoningEffort',\n      'reasoningBudget',\n    ],\n    // Users may omit modelFamily when using an OpenAI-compatible endpoint.\n    // Many models expose useful assistant output through reasoning_content, so\n    // the default adapter should still recover when content is empty.\n    useReasoningAsContentFallback: true,\n  },\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultOpenAICompatibleAdapterConfig"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;ACJO,MAAMI,uCAA+D;IAC1E,gBAAgB;QACd,uBAAuB;YACrB;YACA;YACA;SACD;QAID,+BAA+B;IACjC;AACF"}

package/dist/lib/ai-model/models/gemini.js

@@ -28,29 +28,19 @@ __webpack_require__.d(__webpack_exports__, {
     geminiAdapters: ()=>geminiAdapters
 });
 const buildGeminiChatCompletionParams = (input)=>{
-    const { midsceneDefaults, userConfig, intent } = input;
+    const { midsceneDefaults, userConfig } = input;
     const { reasoningEnabled, reasoningEffort } = userConfig;
     const commonOverrideConfig = {};
     if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
     const modelSpecificConfig = {};
-    if ('default' !== reasoningEnabled) if (reasoningEffort) modelSpecificConfig.extra_body = {
+    if ('default' !== reasoningEnabled) modelSpecificConfig.extra_body = {
         google: {
             thinking_config: {
-                thinking_level: reasoningEffort,
-                include_thoughts: true
+                include_thoughts: true,
+                thinking_level: reasoningEnabled ? reasoningEffort ?? 'medium' : 'minimal'
             }
         }
     };
-    else {
-        if ('insight' === intent) modelSpecificConfig.extra_body = {
-            google: {
-                thinking_config: {
-                    include_thoughts: true
-                }
-            }
-        };
-        modelSpecificConfig.reasoning_effort = 'minimal';
-    }
     return {
         config: {
             ...midsceneDefaults,
@@ -113,7 +103,6 @@ const geminiAdapters = {
     gemini: {
         chatCompletion: {
             unsupportedUserConfig: [
-                'reasoningEnabled',
                 'reasoningBudget'
             ],
             buildChatCompletionParams: buildGeminiChatCompletionParams,

package/dist/lib/ai-model/models/gemini.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/gemini.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/models/gemini.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport type OpenAI from 'openai';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionContentSource,\n  ChatCompletionParamsResult,\n  ContentAndReasoning,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\n\ntype GeminiContentPart = Record<string, unknown> & {\n  text?: string;\n  thought?: boolean;\n};\n\ntype GeminiContentExtension = {\n  content: string | null | GeminiContentPart[];\n  reasoning_content?: string;\n  extra_content?: unknown;\n};\n\ntype GeminiContentSource =\n  | ChatCompletionContentSource\n  | (Omit<OpenAI.Chat.Completions.ChatCompletionMessage, 'content'> &\n      GeminiContentExtension)\n  | (Omit<OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta, 'content'> &\n      GeminiContentExtension);\n\nconst buildGeminiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig, intent } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: {\n    extra_body?: {\n      google?: {\n        thinking_config: Record<string, unknown>;\n      };\n    };\n    reasoning_effort?: unknown;\n  } = {};\n\n  if (reasoningEnabled !== 'default') {\n    if (reasoningEffort) {\n      modelSpecificConfig.extra_body = {\n        google: {\n          thinking_config: {\n            thinking_level: reasoningEffort,\n            include_thoughts: true,\n          },\n        },\n      };\n    } else {\n      if (intent === 'insight') {\n        modelSpecificConfig.extra_body = {\n          google: {\n            thinking_config: {\n              // In real Gemini tests, insight calls need `include_thoughts` to get\n              // the model's thinking, and Gemini puts that thinking into `content`.\n              include_thoughts: true,\n            },\n          },\n        };\n      }\n\n      // Gemini 3.x cannot fully disable native thinking, so use the lowest\n      // supported effort unless the user explicitly requests another level.\n      modelSpecificConfig.reasoning_effort = 'minimal';\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst extractInlineThought = (content: string): string | undefined => {\n  const match = content.match(/<thought>([\\s\\S]*?)<\\/thought>/i);\n  return match?.[1]?.trim() || undefined;\n};\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return !!value && typeof value === 'object' && !Array.isArray(value);\n}\n\ntype GeminiExtractedContent = {\n  content: string;\n  geminiReasoningContent: string;\n};\n\nconst formatReasoningContent = ({\n  geminiReasoningContent,\n  providerReasoningContent,\n}: {\n  geminiReasoningContent: string;\n  providerReasoningContent: string;\n}): string => {\n  if (geminiReasoningContent && providerReasoningContent) {\n    return `thoughtParts：${geminiReasoningContent}; reasoning_content：${providerReasoningContent}`;\n  }\n\n  return geminiReasoningContent || providerReasoningContent || '';\n};\n\nconst extractGeminiThoughtParts = (\n  content: GeminiContentPart[],\n): GeminiExtractedContent => {\n  const contentParts: string[] = [];\n  const reasoningParts: string[] = [];\n\n  for (const part of content) {\n    if (!isRecord(part)) {\n      continue;\n    }\n\n    const text = typeof part.text === 'string' ? part.text : undefined;\n    if (!text) {\n      continue;\n    }\n\n    if (part.thought === true) {\n      reasoningParts.push(text);\n    } else {\n      contentParts.push(text);\n    }\n  }\n\n  return {\n    content: contentParts.join(''),\n    geminiReasoningContent: reasoningParts.join(''),\n  };\n};\n\nexport const extractGeminiContentAndReasoning = (\n  message: GeminiContentSource | undefined,\n): ContentAndReasoning => {\n  // Gemini native API exposes thought summaries in\n  // `response.candidates[0].content.parts`, where each text part can carry\n  // `thought: true`. Some OpenAI-compatible adapters may pass through a\n  // native-like content-parts shape, even though the Gemini OpenAI\n  // compatibility docs do not guarantee it.\n  // Docs: https://ai.google.dev/gemini-api/docs/thinking#thought-summaries\n\n  if (!message) {\n    return {\n      content: '',\n      reasoning_content: '',\n    };\n  }\n\n  if (Array.isArray(message.content)) {\n    const extracted = extractGeminiThoughtParts(message.content);\n\n    return {\n      content: extracted.content,\n      reasoning_content: formatReasoningContent({\n        geminiReasoningContent: extracted.geminiReasoningContent,\n        providerReasoningContent:\n          typeof message.reasoning_content === 'string'\n            ? message.reasoning_content\n            : '',\n      }),\n    };\n  }\n\n  const content = typeof message.content === 'string' ? message.content : '';\n  // In real Gemini OpenAI-compatible responses we observed that\n  // `include_thoughts` can still return a plain string `message.content`,\n  // with the thought summary prepended as `<thought>...</thought>` before the\n  // visible answer. Keep content unchanged, but extract that leading thought\n  // text for report display.\n  const geminiReasoningContent = extractInlineThought(content) || '';\n\n  return {\n    content,\n    reasoning_content: formatReasoningContent({\n      geminiReasoningContent,\n      providerReasoningContent:\n        typeof message.reasoning_content === 'string'\n          ? message.reasoning_content\n          : '',\n    }),\n  };\n};\n\nexport const geminiAdapters = {\n  gemini: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEnabled', 'reasoningBudget'],\n      buildChatCompletionParams: buildGeminiChatCompletionParams,\n      messageExtraction: {\n        kind: 'custom',\n        extractContentAndReasoning: extractGeminiContentAndReasoning,\n      },\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'yx', normalizedBy: 1000 },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'gemini'>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","buildGeminiChatCompletionParams","input","midsceneDefaults","userConfig","intent","reasoningEnabled","reasoningEffort","commonOverrideConfig","undefined","modelSpecificConfig","extractInlineThought","content","match","isRecord","value","Array","formatReasoningContent","geminiReasoningContent","providerReasoningContent","extractGeminiThoughtParts","contentParts","reasoningParts","part","text","extractGeminiContentAndReasoning","message","extracted","geminiAdapters"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACsBA,MAAMI,kCAAkC,CACtCC;IAEA,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAEC,MAAM,EAAE,GAAGH;IACjD,MAAM,EAAEI,gBAAgB,EAAEC,eAAe,EAAE,GAAGH;IAC9C,MAAMI,uBAAgD,CAAC;IAEvD,IAAIJ,AAA2BK,WAA3BL,WAAW,WAAW,EACxBI,qBAAqB,WAAW,GAAGJ,WAAW,WAAW;IAG3D,MAAMM,sBAOF,CAAC;IAEL,IAAIJ,AAAqB,cAArBA,kBACF,IAAIC,iBACFG,oBAAoB,UAAU,GAAG;QAC/B,QAAQ;YACN,iBAAiB;gBACf,gBAAgBH;gBAChB,kBAAkB;YACpB;QACF;IACF;SACK;QACL,IAAIF,AAAW,cAAXA,QACFK,oBAAoB,UAAU,GAAG;YAC/B,QAAQ;gBACN,iBAAiB;oBAGf,kBAAkB;gBACpB;YACF;QACF;QAKFA,oBAAoB,gBAAgB,GAAG;IACzC;IAGF,OAAO;QACL,QAAQ;YACN,GAAGP,gBAAgB;YACnB,GAAGK,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,uBAAuB,CAACC;IAC5B,MAAMC,QAAQD,QAAQ,KAAK,CAAC;IAC5B,OAAOC,OAAO,CAAC,EAAE,EAAE,UAAUJ;AAC/B;AAEA,SAASK,SAASC,KAAc;IAC9B,OAAO,CAAC,CAACA,SAAS,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,MAAM,OAAO,CAACD;AAChE;AAOA,MAAME,yBAAyB,CAAC,EAC9BC,sBAAsB,EACtBC,wBAAwB,EAIzB;IACC,IAAID,0BAA0BC,0BAC5B,OAAO,CAAC,aAAa,EAAED,uBAAuB,oBAAoB,EAAEC,0BAA0B;IAGhG,OAAOD,0BAA0BC,4BAA4B;AAC/D;AAEA,MAAMC,4BAA4B,CAChCR;IAEA,MAAMS,eAAyB,EAAE;IACjC,MAAMC,iBAA2B,EAAE;IAEnC,KAAK,MAAMC,QAAQX,QAAS;QAC1B,IAAI,CAACE,SAASS,OACZ;QAGF,MAAMC,OAAO,AAAqB,YAArB,OAAOD,KAAK,IAAI,GAAgBA,KAAK,IAAI,GAAGd;QACzD,IAAKe,MAIL,IAAID,AAAiB,SAAjBA,KAAK,OAAO,EACdD,eAAe,IAAI,CAACE;aAEpBH,aAAa,IAAI,CAACG;IAEtB;IAEA,OAAO;QACL,SAASH,aAAa,IAAI,CAAC;QAC3B,wBAAwBC,eAAe,IAAI,CAAC;IAC9C;AACF;AAEO,MAAMG,mCAAmC,CAC9CC;IASA,IAAI,CAACA,SACH,OAAO;QACL,SAAS;QACT,mBAAmB;IACrB;IAGF,IAAIV,MAAM,OAAO,CAACU,QAAQ,OAAO,GAAG;QAClC,MAAMC,YAAYP,0BAA0BM,QAAQ,OAAO;QAE3D,OAAO;YACL,SAASC,UAAU,OAAO;YAC1B,mBAAmBV,uBAAuB;gBACxC,wBAAwBU,UAAU,sBAAsB;gBACxD,0BACE,AAAqC,YAArC,OAAOD,QAAQ,iBAAiB,GAC5BA,QAAQ,iBAAiB,GACzB;YACR;QACF;IACF;IAEA,MAAMd,UAAU,AAA2B,YAA3B,OAAOc,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;IAMxE,MAAMR,yBAAyBP,qBAAqBC,YAAY;IAEhE,OAAO;QACLA;QACA,mBAAmBK,uBAAuB;YACxCC;YACA,0BACE,AAAqC,YAArC,OAAOQ,QAAQ,iBAAiB,GAC5BA,QAAQ,iBAAiB,GACzB;QACR;IACF;AACF;AAEO,MAAME,iBAAiB;IAC5B,QAAQ;QACN,gBAAgB;YACd,uBAAuB;gBAAC;gBAAoB;aAAkB;YAC9D,2BAA2B3B;YAC3B,mBAAmB;gBACjB,MAAM;gBACN,4BAA4BwB;YAC9B;QACF;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;oBAAM,cAAc;gBAAK;YAChE;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/gemini.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/models/gemini.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport type OpenAI from 'openai';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionContentSource,\n  ChatCompletionParamsResult,\n  ContentAndReasoning,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\n\ntype GeminiContentPart = Record<string, unknown> & {\n  text?: string;\n  thought?: boolean;\n};\n\ntype GeminiContentExtension = {\n  content: string | null | GeminiContentPart[];\n  reasoning_content?: string;\n  extra_content?: unknown;\n};\n\ntype GeminiContentSource =\n  | ChatCompletionContentSource\n  | (Omit<OpenAI.Chat.Completions.ChatCompletionMessage, 'content'> &\n      GeminiContentExtension)\n  | (Omit<OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta, 'content'> &\n      GeminiContentExtension);\n\nconst buildGeminiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: {\n    extra_body?: {\n      google?: {\n        thinking_config: {\n          include_thoughts: true;\n          thinking_level: string;\n        };\n      };\n    };\n  } = {};\n\n  if (reasoningEnabled !== 'default') {\n    // Gemini cannot fully disable native thinking, so \"disabled\" maps to the\n    // lowest thinking level. `include_thoughts` only controls whether Gemini\n    // returns thought summaries; it does not enable thinking, and thought token\n    // usage is based on the full thoughts Gemini generates regardless of\n    // whether summaries are included.\n    modelSpecificConfig.extra_body = {\n      google: {\n        thinking_config: {\n          include_thoughts: true,\n          thinking_level: reasoningEnabled\n            ? (reasoningEffort ?? 'medium')\n            : 'minimal',\n        },\n      },\n    };\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst extractInlineThought = (content: string): string | undefined => {\n  const match = content.match(/<thought>([\\s\\S]*?)<\\/thought>/i);\n  return match?.[1]?.trim() || undefined;\n};\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return !!value && typeof value === 'object' && !Array.isArray(value);\n}\n\ntype GeminiExtractedContent = {\n  content: string;\n  geminiReasoningContent: string;\n};\n\nconst formatReasoningContent = ({\n  geminiReasoningContent,\n  providerReasoningContent,\n}: {\n  geminiReasoningContent: string;\n  providerReasoningContent: string;\n}): string => {\n  if (geminiReasoningContent && providerReasoningContent) {\n    return `thoughtParts：${geminiReasoningContent}; reasoning_content：${providerReasoningContent}`;\n  }\n\n  return geminiReasoningContent || providerReasoningContent || '';\n};\n\nconst extractGeminiThoughtParts = (\n  content: GeminiContentPart[],\n): GeminiExtractedContent => {\n  const contentParts: string[] = [];\n  const reasoningParts: string[] = [];\n\n  for (const part of content) {\n    if (!isRecord(part)) {\n      continue;\n    }\n\n    const text = typeof part.text === 'string' ? part.text : undefined;\n    if (!text) {\n      continue;\n    }\n\n    if (part.thought === true) {\n      reasoningParts.push(text);\n    } else {\n      contentParts.push(text);\n    }\n  }\n\n  return {\n    content: contentParts.join(''),\n    geminiReasoningContent: reasoningParts.join(''),\n  };\n};\n\nexport const extractGeminiContentAndReasoning = (\n  message: GeminiContentSource | undefined,\n): ContentAndReasoning => {\n  // Gemini native API exposes thought summaries in\n  // `response.candidates[0].content.parts`, where each text part can carry\n  // `thought: true`. Some OpenAI-compatible adapters may pass through a\n  // native-like content-parts shape, even though the Gemini OpenAI\n  // compatibility docs do not guarantee it.\n  // Docs: https://ai.google.dev/gemini-api/docs/thinking#thought-summaries\n\n  if (!message) {\n    return {\n      content: '',\n      reasoning_content: '',\n    };\n  }\n\n  if (Array.isArray(message.content)) {\n    const extracted = extractGeminiThoughtParts(message.content);\n\n    return {\n      content: extracted.content,\n      reasoning_content: formatReasoningContent({\n        geminiReasoningContent: extracted.geminiReasoningContent,\n        providerReasoningContent:\n          typeof message.reasoning_content === 'string'\n            ? message.reasoning_content\n            : '',\n      }),\n    };\n  }\n\n  const content = typeof message.content === 'string' ? message.content : '';\n  // In real Gemini OpenAI-compatible responses we observed that\n  // `include_thoughts` can still return a plain string `message.content`,\n  // with the thought summary prepended as `<thought>...</thought>` before the\n  // visible answer. Keep content unchanged, but extract that leading thought\n  // text for report display.\n  const geminiReasoningContent = extractInlineThought(content) || '';\n\n  return {\n    content,\n    reasoning_content: formatReasoningContent({\n      geminiReasoningContent,\n      providerReasoningContent:\n        typeof message.reasoning_content === 'string'\n          ? message.reasoning_content\n          : '',\n    }),\n  };\n};\n\nexport const geminiAdapters = {\n  gemini: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningBudget'],\n      buildChatCompletionParams: buildGeminiChatCompletionParams,\n      messageExtraction: {\n        kind: 'custom',\n        extractContentAndReasoning: extractGeminiContentAndReasoning,\n      },\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'yx', normalizedBy: 1000 },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'gemini'>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","buildGeminiChatCompletionParams","input","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","undefined","modelSpecificConfig","extractInlineThought","content","match","isRecord","value","Array","formatReasoningContent","geminiReasoningContent","providerReasoningContent","extractGeminiThoughtParts","contentParts","reasoningParts","part","text","extractGeminiContentAndReasoning","message","extracted","geminiAdapters"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACsBA,MAAMI,kCAAkC,CACtCC;IAEA,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGF;IACzC,MAAM,EAAEG,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,MAAMK,sBASF,CAAC;IAEL,IAAIJ,AAAqB,cAArBA,kBAMFI,oBAAoB,UAAU,GAAG;QAC/B,QAAQ;YACN,iBAAiB;gBACf,kBAAkB;gBAClB,gBAAgBJ,mBACXC,mBAAmB,WACpB;YACN;QACF;IACF;IAGF,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,uBAAuB,CAACC;IAC5B,MAAMC,QAAQD,QAAQ,KAAK,CAAC;IAC5B,OAAOC,OAAO,CAAC,EAAE,EAAE,UAAUJ;AAC/B;AAEA,SAASK,SAASC,KAAc;IAC9B,OAAO,CAAC,CAACA,SAAS,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,MAAM,OAAO,CAACD;AAChE;AAOA,MAAME,yBAAyB,CAAC,EAC9BC,sBAAsB,EACtBC,wBAAwB,EAIzB;IACC,IAAID,0BAA0BC,0BAC5B,OAAO,CAAC,aAAa,EAAED,uBAAuB,oBAAoB,EAAEC,0BAA0B;IAGhG,OAAOD,0BAA0BC,4BAA4B;AAC/D;AAEA,MAAMC,4BAA4B,CAChCR;IAEA,MAAMS,eAAyB,EAAE;IACjC,MAAMC,iBAA2B,EAAE;IAEnC,KAAK,MAAMC,QAAQX,QAAS;QAC1B,IAAI,CAACE,SAASS,OACZ;QAGF,MAAMC,OAAO,AAAqB,YAArB,OAAOD,KAAK,IAAI,GAAgBA,KAAK,IAAI,GAAGd;QACzD,IAAKe,MAIL,IAAID,AAAiB,SAAjBA,KAAK,OAAO,EACdD,eAAe,IAAI,CAACE;aAEpBH,aAAa,IAAI,CAACG;IAEtB;IAEA,OAAO;QACL,SAASH,aAAa,IAAI,CAAC;QAC3B,wBAAwBC,eAAe,IAAI,CAAC;IAC9C;AACF;AAEO,MAAMG,mCAAmC,CAC9CC;IASA,IAAI,CAACA,SACH,OAAO;QACL,SAAS;QACT,mBAAmB;IACrB;IAGF,IAAIV,MAAM,OAAO,CAACU,QAAQ,OAAO,GAAG;QAClC,MAAMC,YAAYP,0BAA0BM,QAAQ,OAAO;QAE3D,OAAO;YACL,SAASC,UAAU,OAAO;YAC1B,mBAAmBV,uBAAuB;gBACxC,wBAAwBU,UAAU,sBAAsB;gBACxD,0BACE,AAAqC,YAArC,OAAOD,QAAQ,iBAAiB,GAC5BA,QAAQ,iBAAiB,GACzB;YACR;QACF;IACF;IAEA,MAAMd,UAAU,AAA2B,YAA3B,OAAOc,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;IAMxE,MAAMR,yBAAyBP,qBAAqBC,YAAY;IAEhE,OAAO;QACLA;QACA,mBAAmBK,uBAAuB;YACxCC;YACA,0BACE,AAAqC,YAArC,OAAOQ,QAAQ,iBAAiB,GAC5BA,QAAQ,iBAAiB,GACzB;QACR;IACF;AACF;AAEO,MAAME,iBAAiB;IAC5B,QAAQ;QACN,gBAAgB;YACd,uBAAuB;gBAAC;aAAkB;YAC1C,2BAA2B1B;YAC3B,mBAAmB;gBACjB,MAAM;gBACN,4BAA4BuB;YAC9B;QACF;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;oBAAM,cAAc;gBAAK;YAChE;QACF;IACF;AACF"}

package/dist/lib/ai-model/prompt/describe.js

@@ -49,53 +49,50 @@ const getExamples = (language)=>{
     const examples = examplesMap[language] || examplesMap.English;
     return examples.map((e)=>`- ${e}`).join('\n');
 };
+const describeElementReturnJsonSchema = ()=>`{
+  "description": "unique element identifier",
+  "error"?: "error message if any"
+}`;
 const elementDescriberInstruction = ()=>{
     const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
     return `
-Describe the element in the red rectangle for precise identification.
+Describe the real page element indicated by the temporary callout.
+The callout is an annotation overlay. It is not part of the page or target.
+The description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption.
 
 IMPORTANT: You MUST write the description in ${preferredLanguage}.
 
-CRITICAL REQUIREMENTS:
-1. UNIQUENESS: The description must uniquely identify this element on the current page
-2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts
-3. PRECISION: Be specific enough to distinguish from similar elements
-
-DESCRIPTION STRUCTURE:
-1. Element type (button, input, link, div, etc.)
-2. Primary identifier (in order of preference):
-   - Unique text content: "with text 'Login'"
-   - Unique attribute: "with aria-label 'Search'"
-   - Unique class/ID: "with class 'primary-button'"
-   - Unique position: "in header navigation"
-3. Secondary identifiers (if needed for uniqueness):
-   - Visual features: "blue background", "with icon"
-   - Relative position: "below search bar", "in sidebar"
-   - Parent context: "in login form", "in main menu"
-   - Neighboring stable text: "to the right of the 'Settings' section title"
+OBSERVE IN THIS ORDER:
+1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.
+2. Primitive: name what that smallest part is before adding surrounding context.
+3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text.
+4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state.
 
-GUIDELINES:
-- Keep description under 25 words
-- Prioritize semantic identifiers over visual ones
-- Use consistent terminology across similar elements
-- Avoid page-specific or temporary content
-- Don't mention the red rectangle or selection box
-- Focus on stable, reusable characteristics
-- If the selected point/box is inside a text input, textarea, search box, or form field, describe the whole field/control, not the individual placeholder character, typed character, caret, or inner text fragment.
-- For icon-only buttons or unlabeled controls, include the nearest stable label, section title, menu item text, row text, or parent region that owns the control.
-- When multiple similar icons or controls appear in a list/sidebar/menu, the description MUST distinguish the selected one by its owning stable text or section, not by generic position such as "bottom", "nearby", or "sidebar button".
-- For expand/collapse, disclosure, chevron, close, menu, and settings icons, describe both the icon purpose and the stable text/section it controls.
-- Use the actual visible neighboring text from the current screenshot when available; do not copy labels from the examples.
+RULES:
+- Keep description under 35 words.
+- Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it.
+- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target.
+- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center.
+- For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group.
+- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself.
+- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.
+- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.
+- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries.
+- Only use primitive "icon" or "arrow" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.
+- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible.
+- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line.
+- For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label.
+- For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible.
+- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state.
+- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.
+- Use actual visible text from the current screenshot when available; do not copy labels from the examples.
 - **Write the description in ${preferredLanguage}**
 
 EXAMPLES:
 ${getExamples(preferredLanguage)}
 
 Return JSON:
-{
-  "description": "unique element identifier",
-  "error"?: "error message if any"
-}`;
+${describeElementReturnJsonSchema()}`;
 };
 exports.elementDescriberInstruction = __webpack_exports__.elementDescriberInstruction;
 for(var __rspack_i in __webpack_exports__)if (-1 === [

package/dist/lib/ai-model/prompt/describe.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/describe.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/describe.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@midscene/shared/env';\n\nconst examplesMap: Record<string, string[]> = {\n  Chinese: [\n    '\"登录表单中的\"登录\"按钮\"',\n    '\"搜索输入框，placeholder 为\"请输入关键词\"\"',\n    '\"顶部导航栏中文字为\"首页\"的链接\"',\n    '\"联系表单中的提交按钮\"',\n    '\"aria-label 为\"打开菜单\"的菜单图标\"',\n    '\"左侧导航栏中当前分组标题右侧的折叠图标\"',\n  ],\n  English: [\n    '\"Login button with text \\'Sign In\\'\"',\n    '\"Search input with placeholder \\'Enter keywords\\'\"',\n    '\"Navigation link with text \\'Home\\' in header\"',\n    '\"Submit button in contact form\"',\n    '\"Menu icon with aria-label \\'Open menu\\'\"',\n    '\"Collapse icon to the right of the current section title in the left sidebar\"',\n  ],\n};\n\nconst getExamples = (language: string) => {\n  const examples = examplesMap[language] || examplesMap.English;\n  return examples.map((e) => `- ${e}`).join('\\n');\n};\n\nexport const elementDescriberInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nDescribe the element in the red rectangle for precise identification.\n\nIMPORTANT: You MUST write the description in ${preferredLanguage}.\n\nCRITICAL REQUIREMENTS:\n1. UNIQUENESS: The description must uniquely identify this element on the current page\n2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts\n3. PRECISION: Be specific enough to distinguish from similar elements\n\nDESCRIPTION STRUCTURE:\n1. Element type (button, input, link, div, etc.)\n2. Primary identifier (in order of preference):\n   - Unique text content: \"with text 'Login'\"\n   - Unique attribute: \"with aria-label 'Search'\"\n   - Unique class/ID: \"with class 'primary-button'\"\n   - Unique position: \"in header navigation\"\n3. Secondary identifiers (if needed for uniqueness):\n   - Visual features: \"blue background\", \"with icon\"\n   - Relative position: \"below search bar\", \"in sidebar\"\n   - Parent context: \"in login form\", \"in main menu\"\n   - Neighboring stable text: \"to the right of the 'Settings' section title\"\n\nGUIDELINES:\n- Keep description under 25 words\n- Prioritize semantic identifiers over visual ones\n- Use consistent terminology across similar elements\n- Avoid page-specific or temporary content\n- Don't mention the red rectangle or selection box\n- Focus on stable, reusable characteristics\n- If the selected point/box is inside a text input, textarea, search box, or form field, describe the whole field/control, not the individual placeholder character, typed character, caret, or inner text fragment.\n- For icon-only buttons or unlabeled controls, include the nearest stable label, section title, menu item text, row text, or parent region that owns the control.\n- When multiple similar icons or controls appear in a list/sidebar/menu, the description MUST distinguish the selected one by its owning stable text or section, not by generic position such as \"bottom\", \"nearby\", or \"sidebar button\".\n- For expand/collapse, disclosure, chevron, close, menu, and settings icons, describe both the icon purpose and the stable text/section it controls.\n- Use the actual visible neighboring text from the current screenshot when available; do not copy labels from the examples.\n- **Write the description in ${preferredLanguage}**\n\nEXAMPLES:\n${getExamples(preferredLanguage)}\n\nReturn JSON:\n{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","examplesMap","getExamples","language","examples","e","elementDescriberInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACJA,MAAMI,cAAwC;IAC5C,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;IACD,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,MAAMC,cAAc,CAACC;IACnB,MAAMC,WAAWH,WAAW,CAACE,SAAS,IAAIF,YAAY,OAAO;IAC7D,OAAOG,SAAS,GAAG,CAAC,CAACC,IAAM,CAAC,EAAE,EAAEA,GAAG,EAAE,IAAI,CAAC;AAC5C;AAEO,MAAMC,8BAA8B;IACzC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;6CAGmC,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6BAgCpC,EAAEA,kBAAkB;;;AAGjD,EAAEL,YAAYK,mBAAmB;;;;;;CAMhC,CAAC;AACF"}
+{"version":3,"file":"ai-model/prompt/describe.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/describe.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@midscene/shared/env';\n\nconst examplesMap: Record<string, string[]> = {\n  Chinese: [\n    '\"登录表单中的\"登录\"按钮\"',\n    '\"搜索输入框，placeholder 为\"请输入关键词\"\"',\n    '\"顶部导航栏中文字为\"首页\"的链接\"',\n    '\"联系表单中的提交按钮\"',\n    '\"aria-label 为\"打开菜单\"的菜单图标\"',\n    '\"左侧导航栏中当前分组标题右侧的折叠图标\"',\n  ],\n  English: [\n    '\"Login button with text \\'Sign In\\'\"',\n    '\"Search input with placeholder \\'Enter keywords\\'\"',\n    '\"Navigation link with text \\'Home\\' in header\"',\n    '\"Submit button in contact form\"',\n    '\"Menu icon with aria-label \\'Open menu\\'\"',\n    '\"Collapse icon to the right of the current section title in the left sidebar\"',\n  ],\n};\n\nconst getExamples = (language: string) => {\n  const examples = examplesMap[language] || examplesMap.English;\n  return examples.map((e) => `- ${e}`).join('\\n');\n};\n\nconst describeElementReturnJsonSchema = () => `{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n\nexport const elementDescriberInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nDescribe the real page element indicated by the temporary callout.\nThe callout is an annotation overlay. It is not part of the page or target.\nThe description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption.\n\nIMPORTANT: You MUST write the description in ${preferredLanguage}.\n\nOBSERVE IN THIS ORDER:\n1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.\n2. Primitive: name what that smallest part is before adding surrounding context.\n3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text.\n4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state.\n\nRULES:\n- Keep description under 35 words.\n- Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it.\n- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target.\n- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center.\n- For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group.\n- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself.\n- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive \"dropdown\" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.\n- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.\n- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive \"option\" for selectable list options or \"menuitem\" for command menu entries.\n- Only use primitive \"icon\" or \"arrow\" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.\n- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible.\n- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line.\n- For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label.\n- For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible.\n- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state.\n- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.\n- Use actual visible text from the current screenshot when available; do not copy labels from the examples.\n- **Write the description in ${preferredLanguage}**\n\nEXAMPLES:\n${getExamples(preferredLanguage)}\n\nReturn JSON:\n${describeElementReturnJsonSchema()}`;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","examplesMap","getExamples","language","examples","e","describeElementReturnJsonSchema","elementDescriberInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACJA,MAAMI,cAAwC;IAC5C,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;IACD,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,MAAMC,cAAc,CAACC;IACnB,MAAMC,WAAWH,WAAW,CAACE,SAAS,IAAIF,YAAY,OAAO;IAC7D,OAAOG,SAAS,GAAG,CAAC,CAACC,IAAM,CAAC,EAAE,EAAEA,GAAG,EAAE,IAAI,CAAC;AAC5C;AAEA,MAAMC,kCAAkC,IAAM,CAAC;;;CAG9C,CAAC;AAEK,MAAMC,8BAA8B;IACzC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;;;6CAKmC,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;6BA0BpC,EAAEA,kBAAkB;;;AAGjD,EAAEN,YAAYM,mBAAmB;;;AAGjC,EAAEF,mCAAmC;AACrC"}

package/dist/lib/ai-model/prompt/extraction.js

@@ -39,7 +39,7 @@ function buildTypeQueryDemandValue(type, demand) {
     return `${type}, ${currentScreenshotConstraint}, ${demand}`;
 }
 function parseXMLExtractionResponse(xmlString) {
-    const thought = (0, external_util_js_namespaceObject.extractXMLTag)(xmlString, 'thought');
+    const thought = (0, external_util_js_namespaceObject.extractXMLTag)(xmlString, 'observation');
     const dataJsonStr = (0, external_util_js_namespaceObject.extractXMLTag)(xmlString, 'data-json');
     const errorsStr = (0, external_util_js_namespaceObject.extractXMLTag)(xmlString, 'errors');
     if (!dataJsonStr) throw new Error('Missing required field: data-json');
@@ -89,7 +89,7 @@ When DATA_DEMAND is a JSON object, the keys in your response must exactly match
 
 
 Return in the following XML format:
-<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>
+<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>
 <data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>
 <errors>optional error messages as JSON array, e.g., ["error1", "error2"]</errors>
 
@@ -106,7 +106,7 @@ For example, if the DATA_DEMAND is:
 
 By viewing the screenshot and page contents, you can extract the following data:
 
-<thought>According to the screenshot, i can see ...</thought>
+<observation>According to the screenshot, i can see ...</observation>
 <data-json>
 {
   "name": "John",
@@ -124,7 +124,7 @@ the todo items list, string[]
 
 By viewing the screenshot and page contents, you can extract the following data:
 
-<thought>According to the screenshot, i can see ...</thought>
+<observation>According to the screenshot, i can see ...</observation>
 <data-json>
 ["todo 1", "todo 2", "todo 3"]
 </data-json>
@@ -138,7 +138,7 @@ the page title, string
 
 By viewing the screenshot and page contents, you can extract the following data:
 
-<thought>According to the screenshot, i can see ...</thought>
+<observation>According to the screenshot, i can see ...</observation>
 <data-json>
 "todo list"
 </data-json>
@@ -154,7 +154,7 @@ If the DATA_DEMAND is:
 
 By viewing the screenshot and page contents, you can extract the following data:
 
-<thought>According to the screenshot, i can see ...</thought>
+<observation>According to the screenshot, i can see ...</observation>
 <data-json>
 { "StatementIsTruthy": true }
 </data-json>

package/dist/lib/ai-model/prompt/extraction.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { safeParseJson } from '../service-caller/json';\nimport { extractXMLTag } from './util';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = safeParseJson(dataJsonStr) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = safeParseJson(errorsStr);\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract(options?: {\n  screenshotIncluded?: boolean;\n  referenceImagesIncluded?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n  const screenshotIncluded = options?.screenshotIncluded ?? true;\n  const referenceImagesIncluded = options?.referenceImagesIncluded ?? false;\n\n  const contextPrompts = [\n    \"The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\",\n  ];\n\n  if (screenshotIncluded) {\n    contextPrompts.push(\n      'The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.',\n    );\n  } else {\n    contextPrompts.push(\n      'The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.',\n    );\n  }\n\n  if (referenceImagesIncluded) {\n    const referenceImagesPrompt =\n      'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';\n    contextPrompts.push(\n      screenshotIncluded\n        ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.`\n        : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`,\n    );\n  }\n  const contextPrompt = contextPrompts.join('\\n\\n');\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\n${contextPrompt}\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\n\nReturn in the following XML format:\n<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","safeParseJson","e","errors","parsedErrors","Array","systemPromptToExtract","options","preferredLanguage","getPreferredLanguage","screenshotIncluded","referenceImagesIncluded","contextPrompts","referenceImagesPrompt","contextPrompt","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACDO,SAASI,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAKO,SAASE,2BACdC,SAAiB;IAEjB,MAAMC,UAAUC,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IACzC,MAAMG,cAAcD,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAC7C,MAAMI,YAAYF,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcJ;IACvB,EAAE,OAAOK,GAAG;QACV,MAAM,IAAIH,MAAM,CAAC,2BAA2B,EAAEG,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIL,WACF,IAAI;QACF,MAAMM,eAAeH,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH;QACnC,IAAIO,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIP,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAIG,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG,sBAAsBC,OAGrC;IACC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAC1B,MAAMC,qBAAqBH,SAAS,sBAAsB;IAC1D,MAAMI,0BAA0BJ,SAAS,2BAA2B;IAEpE,MAAMK,iBAAiB;QACrB;KACD;IAED,IAAIF,oBACFE,eAAe,IAAI,CACjB;SAGFA,eAAe,IAAI,CACjB;IAIJ,IAAID,yBAAyB;QAC3B,MAAME,wBACJ;QACFD,eAAe,IAAI,CACjBF,qBACI,GAAGG,sBAAsB,mLAAmL,CAAC,GAC7M,GAAGA,sBAAsB,kIAAkI,CAAC;IAEpK;IACA,MAAMC,gBAAgBF,eAAe,IAAI,CAAC;IAE1C,OAAO,CAAC;;;AAGV,EAAEE,cAAc;;;;;;;;0EAQ0D,EAAEN,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqE9F,CAAC;AACD;AAEO,MAAMO,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}
+{"version":3,"file":"ai-model/prompt/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { safeParseJson } from '../service-caller/json';\nimport { extractXMLTag } from './util';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  // Keep the internal field named `thought`, but ask models to emit\n  // <observation>. Gemini may only return <thought>-named content when\n  // thinking summaries are enabled.\n  const thought = extractXMLTag(xmlString, 'observation');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = safeParseJson(dataJsonStr) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = safeParseJson(errorsStr);\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract(options?: {\n  screenshotIncluded?: boolean;\n  referenceImagesIncluded?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n  const screenshotIncluded = options?.screenshotIncluded ?? true;\n  const referenceImagesIncluded = options?.referenceImagesIncluded ?? false;\n\n  const contextPrompts = [\n    \"The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\",\n  ];\n\n  if (screenshotIncluded) {\n    contextPrompts.push(\n      'The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.',\n    );\n  } else {\n    contextPrompts.push(\n      'The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.',\n    );\n  }\n\n  if (referenceImagesIncluded) {\n    const referenceImagesPrompt =\n      'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';\n    contextPrompts.push(\n      screenshotIncluded\n        ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.`\n        : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`,\n    );\n  }\n  const contextPrompt = contextPrompts.join('\\n\\n');\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\n${contextPrompt}\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\n\nReturn in the following XML format:\n<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","safeParseJson","e","errors","parsedErrors","Array","systemPromptToExtract","options","preferredLanguage","getPreferredLanguage","screenshotIncluded","referenceImagesIncluded","contextPrompts","referenceImagesPrompt","contextPrompt","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACDO,SAASI,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAKO,SAASE,2BACdC,SAAiB;IAKjB,MAAMC,UAAUC,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IACzC,MAAMG,cAAcD,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAC7C,MAAMI,YAAYF,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcJ;IACvB,EAAE,OAAOK,GAAG;QACV,MAAM,IAAIH,MAAM,CAAC,2BAA2B,EAAEG,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIL,WACF,IAAI;QACF,MAAMM,eAAeH,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH;QACnC,IAAIO,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIP,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAIG,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG,sBAAsBC,OAGrC;IACC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAC1B,MAAMC,qBAAqBH,SAAS,sBAAsB;IAC1D,MAAMI,0BAA0BJ,SAAS,2BAA2B;IAEpE,MAAMK,iBAAiB;QACrB;KACD;IAED,IAAIF,oBACFE,eAAe,IAAI,CACjB;SAGFA,eAAe,IAAI,CACjB;IAIJ,IAAID,yBAAyB;QAC3B,MAAME,wBACJ;QACFD,eAAe,IAAI,CACjBF,qBACI,GAAGG,sBAAsB,mLAAmL,CAAC,GAC7M,GAAGA,sBAAsB,kIAAkI,CAAC;IAEpK;IACA,MAAMC,gBAAgBF,eAAe,IAAI,CAAC;IAE1C,OAAO,CAAC;;;AAGV,EAAEE,cAAc;;;;;;;;kFAQkE,EAAEN,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEtG,CAAC;AACD;AAEO,MAAMO,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}