@donggui/core 1.5.4-donggui.3

package/dist/lib/ai-model/llm-planning.js

@@ -0,0 +1,270 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    parseXMLPlanningResponse: ()=>parseXMLPlanningResponse,
+    plan: ()=>plan
+});
+const img_namespaceObject = require("@midscene/shared/img");
+const logger_namespaceObject = require("@midscene/shared/logger");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const external_common_js_namespaceObject = require("../common.js");
+const llm_planning_js_namespaceObject = require("./prompt/llm-planning.js");
+const util_js_namespaceObject = require("./prompt/util.js");
+const index_js_namespaceObject = require("./service-caller/index.js");
+const debug = (0, logger_namespaceObject.getDebug)('planning');
+const warnLog = (0, logger_namespaceObject.getDebug)('planning', {
+    console: true
+});
+function parseXMLPlanningResponse(xmlString, modelFamily) {
+    const thought = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'thought');
+    const memory = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'memory');
+    const log = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'log') || '';
+    const error = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'error');
+    const actionType = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'action-type');
+    const actionParamStr = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'action-param-json');
+    const completeGoalRegex = /<complete\s+success="(true|false)">([\s\S]*?)<\/complete>/i;
+    const completeGoalMatch = xmlString.match(completeGoalRegex);
+    let finalizeMessage;
+    let finalizeSuccess;
+    if (completeGoalMatch) {
+        finalizeSuccess = 'true' === completeGoalMatch[1];
+        finalizeMessage = completeGoalMatch[2]?.trim() || void 0;
+    }
+    const updatePlanContent = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'update-plan-content');
+    const markSubGoalDone = (0, util_js_namespaceObject.extractXMLTag)(xmlString, 'mark-sub-goal-done');
+    const updateSubGoals = updatePlanContent ? (0, util_js_namespaceObject.parseSubGoalsFromXML)(updatePlanContent) : void 0;
+    const markFinishedIndexes = markSubGoalDone ? (0, util_js_namespaceObject.parseMarkFinishedIndexes)(markSubGoalDone) : void 0;
+    let action = null;
+    if (actionType && 'null' !== actionType.toLowerCase()) {
+        const type = actionType.trim();
+        let param;
+        if (actionParamStr) try {
+            param = (0, index_js_namespaceObject.safeParseJson)(actionParamStr, modelFamily);
+        } catch (e) {
+            throw new Error(`Failed to parse action-param-json: ${e}`);
+        }
+        action = {
+            type,
+            ...void 0 !== param ? {
+                param
+            } : {}
+        };
+    }
+    return {
+        ...thought ? {
+            thought
+        } : {},
+        ...memory ? {
+            memory
+        } : {},
+        log,
+        ...error ? {
+            error
+        } : {},
+        action,
+        ...void 0 !== finalizeMessage ? {
+            finalizeMessage
+        } : {},
+        ...void 0 !== finalizeSuccess ? {
+            finalizeSuccess
+        } : {},
+        ...updateSubGoals?.length ? {
+            updateSubGoals
+        } : {},
+        ...markFinishedIndexes?.length ? {
+            markFinishedIndexes
+        } : {}
+    };
+}
+async function plan(userInstruction, opts) {
+    const { context, modelConfig, conversationHistory } = opts;
+    const { shotSize } = context;
+    const screenshotBase64 = context.screenshot.base64;
+    const { modelFamily } = modelConfig;
+    const includeSubGoals = true === opts.deepThink;
+    const systemPrompt = await (0, llm_planning_js_namespaceObject.systemPromptToTaskPlanning)({
+        actionSpace: opts.actionSpace,
+        modelFamily,
+        includeBbox: opts.includeBbox,
+        includeThought: true,
+        includeSubGoals
+    });
+    let imagePayload = screenshotBase64;
+    let imageWidth = shotSize.width;
+    let imageHeight = shotSize.height;
+    if ('qwen2.5-vl' === modelFamily) {
+        const paddedResult = await (0, img_namespaceObject.paddingToMatchBlockByBase64)(imagePayload);
+        imageWidth = paddedResult.width;
+        imageHeight = paddedResult.height;
+        imagePayload = paddedResult.imageBase64;
+    }
+    const actionContext = opts.actionContext ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\n` : '';
+    const instruction = [
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`
+                }
+            ]
+        }
+    ];
+    let latestFeedbackMessage;
+    const subGoalsText = includeSubGoals ? conversationHistory.subGoalsToText() : conversationHistory.historicalLogsToText();
+    const subGoalsSection = subGoalsText ? `\n\n${subGoalsText}` : '';
+    const memoriesText = conversationHistory.memoriesToText();
+    const memoriesSection = memoriesText ? `\n\n${memoriesText}` : '';
+    if (conversationHistory.pendingFeedbackMessage) {
+        latestFeedbackMessage = {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${subGoalsSection}`
+                },
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                }
+            ]
+        };
+        conversationHistory.resetPendingFeedbackMessageIfExists();
+    } else latestFeedbackMessage = {
+        role: 'user',
+        content: [
+            {
+                type: 'text',
+                text: `this is the latest screenshot${memoriesSection}${subGoalsSection}`
+            },
+            {
+                type: 'image_url',
+                image_url: {
+                    url: imagePayload,
+                    detail: 'high'
+                }
+            }
+        ]
+    };
+    conversationHistory.append(latestFeedbackMessage);
+    conversationHistory.compressHistory(50, 20);
+    const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        ...instruction,
+        ...historyLog
+    ];
+    let { content: rawResponse, usage, reasoning_content } = await (0, index_js_namespaceObject.callAI)(msgs, modelConfig, {
+        deepThink: 'unset' === opts.deepThink ? void 0 : opts.deepThink,
+        abortSignal: opts.abortSignal
+    });
+    let planFromAI;
+    try {
+        try {
+            planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);
+        } catch  {
+            const retry = await (0, index_js_namespaceObject.callAI)(msgs, modelConfig, {
+                deepThink: 'unset' === opts.deepThink ? void 0 : opts.deepThink,
+                abortSignal: opts.abortSignal
+            });
+            rawResponse = retry.content;
+            usage = retry.usage;
+            reasoning_content = retry.reasoning_content;
+            planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);
+        }
+        if (planFromAI.action && void 0 !== planFromAI.finalizeSuccess) {
+            warnLog('Planning response included both an action and <complete>; ignoring <complete> output.');
+            planFromAI.finalizeMessage = void 0;
+            planFromAI.finalizeSuccess = void 0;
+        }
+        const actions = planFromAI.action ? [
+            planFromAI.action
+        ] : [];
+        let shouldContinuePlanning = true;
+        if (void 0 !== planFromAI.finalizeSuccess) {
+            debug('task completed via <complete> tag, stop planning');
+            shouldContinuePlanning = false;
+            if (includeSubGoals) conversationHistory.markAllSubGoalsFinished();
+        }
+        const returnValue = {
+            ...planFromAI,
+            actions,
+            rawResponse,
+            usage,
+            reasoning_content,
+            yamlFlow: (0, external_common_js_namespaceObject.buildYamlFlowFromPlans)(actions, opts.actionSpace),
+            shouldContinuePlanning
+        };
+        (0, utils_namespaceObject.assert)(planFromAI, "can't get plans from AI");
+        actions.forEach((action)=>{
+            const type = action.type;
+            const actionInActionSpace = opts.actionSpace.find((action)=>action.name === type);
+            debug('actionInActionSpace matched', actionInActionSpace);
+            const locateFields = actionInActionSpace ? (0, external_common_js_namespaceObject.findAllMidsceneLocatorField)(actionInActionSpace.paramSchema) : [];
+            debug('locateFields', locateFields);
+            locateFields.forEach((field)=>{
+                const locateResult = action.param[field];
+                if (locateResult && void 0 !== modelFamily) action.param[field] = (0, external_common_js_namespaceObject.fillBboxParam)(locateResult, imageWidth, imageHeight, modelFamily);
+            });
+        });
+        if (includeSubGoals) {
+            if (planFromAI.updateSubGoals?.length) conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);
+            if (planFromAI.markFinishedIndexes?.length) for (const index of planFromAI.markFinishedIndexes)conversationHistory.markSubGoalFinished(index);
+            if (planFromAI.log) conversationHistory.appendSubGoalLog(planFromAI.log);
+        } else if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
+        if (planFromAI.memory) conversationHistory.appendMemory(planFromAI.memory);
+        conversationHistory.append({
+            role: 'assistant',
+            content: [
+                {
+                    type: 'text',
+                    text: rawResponse
+                }
+            ]
+        });
+        return returnValue;
+    } catch (parseError) {
+        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+        throw new index_js_namespaceObject.AIResponseParseError(`XML parse error: ${errorMessage}`, rawResponse, usage);
+    }
+}
+exports.parseXMLPlanningResponse = __webpack_exports__.parseXMLPlanningResponse;
+exports.plan = __webpack_exports__.plan;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "parseXMLPlanningResponse",
+    "plan"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=llm-planning.js.map

package/dist/lib/ai-model/llm-planning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DeepThinkOption,\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n} from '../common';\nimport type { ConversationHistory } from './conversation-history';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport {\n  AIResponseParseError,\n  callAI,\n  safeParseJson,\n} from './service-caller/index';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  modelFamily: TModelFamily | undefined,\n): RawResponsePlanningAIResponse {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    const type = actionType.trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = safeParseJson(actionParamStr, modelFamily);\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    actionContext?: string;\n    modelConfig: IModelConfig;\n    conversationHistory: ConversationHistory;\n    includeBbox: boolean;\n    imagesIncludeCount?: number;\n    deepThink?: DeepThinkOption;\n    abortSignal?: AbortSignal;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig, conversationHistory } = opts;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const { modelFamily } = modelConfig;\n\n  // Only enable sub-goals when deepThink is true\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    modelFamily,\n    includeBbox: opts.includeBbox,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = shotSize.width;\n  let imageHeight = shotSize.height;\n  const rightLimit = imageWidth;\n  const bottomLimit = imageHeight;\n\n  // Process image based on VL mode requirements\n  if (modelFamily === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`,\n        },\n      ],\n    },\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In deepThink mode: show full sub-goals with logs\n  // In non-deepThink mode: show historical execution logs\n  const subGoalsText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const subGoalsSection = subGoalsText ? `\\n\\n${subGoalsText}` : '';\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${subGoalsSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `this is the latest screenshot${memoriesSection}${subGoalsSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  let {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n  } = await callAI(msgs, modelConfig, {\n    deepThink: opts.deepThink === 'unset' ? undefined : opts.deepThink,\n    abortSignal: opts.abortSignal,\n  });\n\n  // Parse XML response to JSON object, retry once on parse failure\n  let planFromAI: RawResponsePlanningAIResponse;\n  try {\n    try {\n      planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);\n    } catch {\n      const retry = await callAI(msgs, modelConfig, {\n        deepThink: opts.deepThink === 'unset' ? undefined : opts.deepThink,\n        abortSignal: opts.abortSignal,\n      });\n      rawResponse = retry.content;\n      usage = retry.usage;\n      reasoning_content = retry.reasoning_content;\n      planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);\n    }\n\n    if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n      warnLog(\n        'Planning response included both an action and <complete>; ignoring <complete> output.',\n      );\n      planFromAI.finalizeMessage = undefined;\n      planFromAI.finalizeSuccess = undefined;\n    }\n\n    const actions = planFromAI.action ? [planFromAI.action] : [];\n    let shouldContinuePlanning = true;\n\n    // Check if task is completed via <complete> tag\n    if (planFromAI.finalizeSuccess !== undefined) {\n      debug('task completed via <complete> tag, stop planning');\n      shouldContinuePlanning = false;\n      // Mark all sub-goals as finished when goal is completed (only when deepThink is enabled)\n      if (includeSubGoals) {\n        conversationHistory.markAllSubGoalsFinished();\n      }\n    }\n\n    const returnValue: PlanningAIResponse = {\n      ...planFromAI,\n      actions,\n      rawResponse,\n      usage,\n      reasoning_content,\n      yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),\n      shouldContinuePlanning,\n    };\n\n    assert(planFromAI, \"can't get plans from AI\");\n\n    actions.forEach((action) => {\n      const type = action.type;\n      const actionInActionSpace = opts.actionSpace.find(\n        (action) => action.name === type,\n      );\n\n      debug('actionInActionSpace matched', actionInActionSpace);\n      const locateFields = actionInActionSpace\n        ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n        : [];\n\n      debug('locateFields', locateFields);\n\n      locateFields.forEach((field) => {\n        const locateResult = action.param[field];\n        if (locateResult && modelFamily !== undefined) {\n          // Always use model family to fill bbox parameters\n          action.param[field] = fillBboxParam(\n            locateResult,\n            imageWidth,\n            imageHeight,\n            modelFamily,\n          );\n        }\n      });\n    });\n\n    // Update sub-goals in conversation history based on response (only when deepThink is enabled)\n    if (includeSubGoals) {\n      if (planFromAI.updateSubGoals?.length) {\n        conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n      }\n      if (planFromAI.markFinishedIndexes?.length) {\n        for (const index of planFromAI.markFinishedIndexes) {\n          conversationHistory.markSubGoalFinished(index);\n        }\n      }\n      // Append the planning log to the currently running sub-goal\n      if (planFromAI.log) {\n        conversationHistory.appendSubGoalLog(planFromAI.log);\n      }\n    } else {\n      // In non-deepThink mode, accumulate logs as historical execution steps\n      if (planFromAI.log) {\n        conversationHistory.appendHistoricalLog(planFromAI.log);\n      }\n    }\n\n    // Append memory to conversation history if present\n    if (planFromAI.memory) {\n      conversationHistory.appendMemory(planFromAI.memory);\n    }\n\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n\n    return returnValue;\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n    );\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","parseXMLPlanningResponse","xmlString","modelFamily","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","safeParseJson","e","Error","plan","userInstruction","opts","context","modelConfig","conversationHistory","shotSize","screenshotBase64","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","imagePayload","imageWidth","imageHeight","paddedResult","paddingToMatchBlockByBase64","actionContext","instruction","latestFeedbackMessage","subGoalsText","subGoalsSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","callAI","planFromAI","retry","actions","shouldContinuePlanning","returnValue","buildYamlFlowFromPlans","assert","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","index","parseError","errorMessage","String","AIResponseParseError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACyBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,YAAY;IAAE,SAAS;AAAK;AAK9C,SAASE,yBACdC,SAAiB,EACjBC,WAAqC;IAErC,MAAMC,UAAUC,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACzC,MAAMI,SAASD,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACxC,MAAMK,MAAMF,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACvC,MAAMO,aAAaJ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,AAAAA,IAAAA,wBAAAA,oBAAAA,AAAAA,EAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,AAAAA,IAAAA,wBAAAA,wBAAAA,AAAAA,EAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QACrD,MAAMc,OAAOd,WAAW,IAAI;QAC5B,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQC,AAAAA,IAAAA,yBAAAA,aAAAA,AAAAA,EAAcf,gBAAgBP;QACxC,EAAE,OAAOuB,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFJ,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAEO,eAAeQ,KACpBC,eAAuB,EACvBC,IAWC;IAED,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGH;IACtD,MAAM,EAAEI,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM,EAAE5B,WAAW,EAAE,GAAG6B;IAGxB,MAAMI,kBAAkBN,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMO,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAaR,KAAK,WAAW;QAC7B3B;QACA,aAAa2B,KAAK,WAAW;QAC7B,gBAAgB;QAChBM;IACF;IAEA,IAAIG,eAAeJ;IACnB,IAAIK,aAAaN,SAAS,KAAK;IAC/B,IAAIO,cAAcP,SAAS,MAAM;IAKjC,IAAI/B,AAAgB,iBAAhBA,aAA8B;QAChC,MAAMuC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BJ;QACvDC,aAAaE,aAAa,KAAK;QAC/BD,cAAcC,aAAa,MAAM;QACjCH,eAAeG,aAAa,WAAW;IACzC;IAEA,MAAME,gBAAgBd,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMe,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGD,cAAc,kBAAkB,EAAEf,gBAAgB,mBAAmB,CAAC;gBACjF;aACD;QACH;KACD;IAED,IAAIiB;IAKJ,MAAMC,eAAeX,kBACjBH,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMe,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAG/D,MAAME,eAAehB,oBAAoB,cAAc;IACvD,MAAMiB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIhB,oBAAoB,sBAAsB,EAAE;QAC9Ca,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGb,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEiB,kBAAkBF,iBAAiB;gBAChN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKT;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAN,oBAAoB,mCAAmC;IACzD,OACEa,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,6BAA6B,EAAEI,kBAAkBF,iBAAiB;YAC3E;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKT;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFN,oBAAoB,MAAM,CAACa;IAG3Bb,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMkB,aAAalB,oBAAoB,QAAQ,CAACH,KAAK,kBAAkB;IAEvE,MAAMsB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,IAAI,EACF,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOJ,MAAMpB,aAAa;QAClC,WAAWF,AAAmB,YAAnBA,KAAK,SAAS,GAAef,SAAYe,KAAK,SAAS;QAClE,aAAaA,KAAK,WAAW;IAC/B;IAGA,IAAI2B;IACJ,IAAI;QACF,IAAI;YACFA,aAAaxD,yBAAyBoD,aAAalD;QACrD,EAAE,OAAM;YACN,MAAMuD,QAAQ,MAAMF,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOJ,MAAMpB,aAAa;gBAC5C,WAAWF,AAAmB,YAAnBA,KAAK,SAAS,GAAef,SAAYe,KAAK,SAAS;gBAClE,aAAaA,KAAK,WAAW;YAC/B;YACAuB,cAAcK,MAAM,OAAO;YAC3BJ,QAAQI,MAAM,KAAK;YACnBH,oBAAoBG,MAAM,iBAAiB;YAC3CD,aAAaxD,yBAAyBoD,aAAalD;QACrD;QAEA,IAAIsD,WAAW,MAAM,IAAIA,AAA+B1C,WAA/B0C,WAAW,eAAe,EAAgB;YACjEzD,QACE;YAEFyD,WAAW,eAAe,GAAG1C;YAC7B0C,WAAW,eAAe,GAAG1C;QAC/B;QAEA,MAAM4C,UAAUF,WAAW,MAAM,GAAG;YAACA,WAAW,MAAM;SAAC,GAAG,EAAE;QAC5D,IAAIG,yBAAyB;QAG7B,IAAIH,AAA+B1C,WAA/B0C,WAAW,eAAe,EAAgB;YAC5C3D,MAAM;YACN8D,yBAAyB;YAEzB,IAAIxB,iBACFH,oBAAoB,uBAAuB;QAE/C;QAEA,MAAM4B,cAAkC;YACtC,GAAGJ,UAAU;YACbE;YACAN;YACAC;YACAC;YACA,UAAUO,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBH,SAAS7B,KAAK,WAAW;YAC1D8B;QACF;QAEAG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAON,YAAY;QAEnBE,QAAQ,OAAO,CAAC,CAACrC;YACf,MAAMC,OAAOD,OAAO,IAAI;YACxB,MAAM0C,sBAAsBlC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACR,SAAWA,OAAO,IAAI,KAAKC;YAG9BzB,MAAM,+BAA+BkE;YACrC,MAAMC,eAAeD,sBACjBE,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BF,oBAAoB,WAAW,IAC3D,EAAE;YAENlE,MAAM,gBAAgBmE;YAEtBA,aAAa,OAAO,CAAC,CAACE;gBACpB,MAAMC,eAAe9C,OAAO,KAAK,CAAC6C,MAAM;gBACxC,IAAIC,gBAAgBjE,AAAgBY,WAAhBZ,aAElBmB,OAAO,KAAK,CAAC6C,MAAM,GAAGE,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EACpBD,cACA5B,YACAC,aACAtC;YAGN;QACF;QAGA,IAAIiC,iBAAiB;YACnB,IAAIqB,WAAW,cAAc,EAAE,QAC7BxB,oBAAoB,aAAa,CAACwB,WAAW,cAAc;YAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAMa,SAASb,WAAW,mBAAmB,CAChDxB,oBAAoB,mBAAmB,CAACqC;YAI5C,IAAIb,WAAW,GAAG,EAChBxB,oBAAoB,gBAAgB,CAACwB,WAAW,GAAG;QAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBxB,oBAAoB,mBAAmB,CAACwB,WAAW,GAAG;QAK1D,IAAIA,WAAW,MAAM,EACnBxB,oBAAoB,YAAY,CAACwB,WAAW,MAAM;QAGpDxB,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMoB;gBACR;aACD;QACH;QAEA,OAAOQ;IACT,EAAE,OAAOU,YAAY;QAEnB,MAAMC,eACJD,sBAAsB5C,QAAQ4C,WAAW,OAAO,GAAGE,OAAOF;QAC5D,MAAM,IAAIG,yBAAAA,oBAAoBA,CAC5B,CAAC,iBAAiB,EAAEF,cAAc,EAClCnB,aACAC;IAEJ;AACF"}

package/dist/lib/ai-model/prompt/common.js

@@ -0,0 +1,41 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    bboxDescription: ()=>bboxDescription
+});
+function bboxDescription(modelFamily) {
+    if ('gemini' === modelFamily) return 'box_2d bounding box for the target element, should be [ymin, xmin, ymax, xmax] normalized to 0-1000.';
+    return '2d bounding box as [xmin, ymin, xmax, ymax]';
+}
+exports.bboxDescription = __webpack_exports__.bboxDescription;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "bboxDescription"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=common.js.map

package/dist/lib/ai-model/prompt/common.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/common.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/common.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nexport function bboxDescription(modelFamily: TModelFamily | undefined) {\n  if (modelFamily === 'gemini') {\n    return 'box_2d bounding box for the target element, should be [ymin, xmin, ymax, xmax] normalized to 0-1000.';\n  }\n  return '2d bounding box as [xmin, ymin, xmax, ymax]';\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","bboxDescription","modelFamily"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;ACLO,SAASI,gBAAgBC,WAAqC;IACnE,IAAIA,AAAgB,aAAhBA,aACF,OAAO;IAET,OAAO;AACT"}

package/dist/lib/ai-model/prompt/describe.js

@@ -0,0 +1,100 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    elementDescriberInstruction: ()=>elementDescriberInstruction
+});
+const env_namespaceObject = require("@midscene/shared/env");
+const examplesMap = {
+    Chinese: [
+        '"登录表单中的"登录"按钮"',
+        '"搜索输入框，placeholder 为"请输入关键词""',
+        '"顶部导航栏中文字为"首页"的链接"',
+        '"联系表单中的提交按钮"',
+        '"aria-label 为"打开菜单"的菜单图标"'
+    ],
+    English: [
+        '"Login button with text \'Sign In\'"',
+        '"Search input with placeholder \'Enter keywords\'"',
+        '"Navigation link with text \'Home\' in header"',
+        '"Submit button in contact form"',
+        '"Menu icon with aria-label \'Open menu\'"'
+    ]
+};
+const getExamples = (language)=>{
+    const examples = examplesMap[language] || examplesMap.English;
+    return examples.map((e)=>`- ${e}`).join('\n');
+};
+const elementDescriberInstruction = ()=>{
+    const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
+    return `
+Describe the element in the red rectangle for precise identification.
+
+IMPORTANT: You MUST write the description in ${preferredLanguage}.
+
+CRITICAL REQUIREMENTS:
+1. UNIQUENESS: The description must uniquely identify this element on the current page
+2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts
+3. PRECISION: Be specific enough to distinguish from similar elements
+
+DESCRIPTION STRUCTURE:
+1. Element type (button, input, link, div, etc.)
+2. Primary identifier (in order of preference):
+   - Unique text content: "with text 'Login'"
+   - Unique attribute: "with aria-label 'Search'"
+   - Unique class/ID: "with class 'primary-button'"
+   - Unique position: "in header navigation"
+3. Secondary identifiers (if needed for uniqueness):
+   - Visual features: "blue background", "with icon"
+   - Relative position: "below search bar", "in sidebar"
+   - Parent context: "in login form", "in main menu"
+
+GUIDELINES:
+- Keep description under 25 words
+- Prioritize semantic identifiers over visual ones
+- Use consistent terminology across similar elements
+- Avoid page-specific or temporary content
+- Don't mention the red rectangle or selection box
+- Focus on stable, reusable characteristics
+- **Write the description in ${preferredLanguage}**
+
+EXAMPLES:
+${getExamples(preferredLanguage)}
+
+Return JSON:
+{
+  "description": "unique element identifier",
+  "error"?: "error message if any"
+}`;
+};
+exports.elementDescriberInstruction = __webpack_exports__.elementDescriberInstruction;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "elementDescriberInstruction"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=describe.js.map

package/dist/lib/ai-model/prompt/describe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/describe.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/describe.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@midscene/shared/env';\n\nconst examplesMap: Record<string, string[]> = {\n  Chinese: [\n    '\"登录表单中的\"登录\"按钮\"',\n    '\"搜索输入框，placeholder 为\"请输入关键词\"\"',\n    '\"顶部导航栏中文字为\"首页\"的链接\"',\n    '\"联系表单中的提交按钮\"',\n    '\"aria-label 为\"打开菜单\"的菜单图标\"',\n  ],\n  English: [\n    '\"Login button with text \\'Sign In\\'\"',\n    '\"Search input with placeholder \\'Enter keywords\\'\"',\n    '\"Navigation link with text \\'Home\\' in header\"',\n    '\"Submit button in contact form\"',\n    '\"Menu icon with aria-label \\'Open menu\\'\"',\n  ],\n};\n\nconst getExamples = (language: string) => {\n  const examples = examplesMap[language] || examplesMap.English;\n  return examples.map((e) => `- ${e}`).join('\\n');\n};\n\nexport const elementDescriberInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nDescribe the element in the red rectangle for precise identification.\n\nIMPORTANT: You MUST write the description in ${preferredLanguage}.\n\nCRITICAL REQUIREMENTS:\n1. UNIQUENESS: The description must uniquely identify this element on the current page\n2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts\n3. PRECISION: Be specific enough to distinguish from similar elements\n\nDESCRIPTION STRUCTURE:\n1. Element type (button, input, link, div, etc.)\n2. Primary identifier (in order of preference):\n   - Unique text content: \"with text 'Login'\"\n   - Unique attribute: \"with aria-label 'Search'\"\n   - Unique class/ID: \"with class 'primary-button'\"\n   - Unique position: \"in header navigation\"\n3. Secondary identifiers (if needed for uniqueness):\n   - Visual features: \"blue background\", \"with icon\"\n   - Relative position: \"below search bar\", \"in sidebar\"\n   - Parent context: \"in login form\", \"in main menu\"\n\nGUIDELINES:\n- Keep description under 25 words\n- Prioritize semantic identifiers over visual ones\n- Use consistent terminology across similar elements\n- Avoid page-specific or temporary content\n- Don't mention the red rectangle or selection box\n- Focus on stable, reusable characteristics\n- **Write the description in ${preferredLanguage}**\n\nEXAMPLES:\n${getExamples(preferredLanguage)}\n\nReturn JSON:\n{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","examplesMap","getExamples","language","examples","e","elementDescriberInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACJA,MAAMI,cAAwC;IAC5C,SAAS;QACP;QACA;QACA;QACA;QACA;KACD;IACD,SAAS;QACP;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,MAAMC,cAAc,CAACC;IACnB,MAAMC,WAAWH,WAAW,CAACE,SAAS,IAAIF,YAAY,OAAO;IAC7D,OAAOG,SAAS,GAAG,CAAC,CAACC,IAAM,CAAC,EAAE,EAAEA,GAAG,EAAE,IAAI,CAAC;AAC5C;AAEO,MAAMC,8BAA8B;IACzC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;6CAGmC,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;6BA0BpC,EAAEA,kBAAkB;;;AAGjD,EAAEL,YAAYK,mBAAmB;;;;;;CAMhC,CAAC;AACF"}

package/dist/lib/ai-model/prompt/extraction.js

@@ -0,0 +1,169 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    parseXMLExtractionResponse: ()=>parseXMLExtractionResponse,
+    systemPromptToExtract: ()=>systemPromptToExtract,
+    extractDataQueryPrompt: ()=>extractDataQueryPrompt
+});
+const env_namespaceObject = require("@midscene/shared/env");
+const index_js_namespaceObject = require("../service-caller/index.js");
+const external_util_js_namespaceObject = require("./util.js");
+function parseXMLExtractionResponse(xmlString) {
+    const thought = (0, external_util_js_namespaceObject.extractXMLTag)(xmlString, 'thought');
+    const dataJsonStr = (0, external_util_js_namespaceObject.extractXMLTag)(xmlString, 'data-json');
+    const errorsStr = (0, external_util_js_namespaceObject.extractXMLTag)(xmlString, 'errors');
+    if (!dataJsonStr) throw new Error('Missing required field: data-json');
+    let data;
+    try {
+        data = (0, index_js_namespaceObject.safeParseJson)(dataJsonStr, void 0);
+    } catch (e) {
+        throw new Error(`Failed to parse data-json: ${e}`);
+    }
+    let errors;
+    if (errorsStr) try {
+        const parsedErrors = (0, index_js_namespaceObject.safeParseJson)(errorsStr, void 0);
+        if (Array.isArray(parsedErrors)) errors = parsedErrors;
+    } catch (e) {}
+    return {
+        ...thought ? {
+            thought
+        } : {},
+        data,
+        ...errors && errors.length > 0 ? {
+            errors
+        } : {}
+    };
+}
+function systemPromptToExtract() {
+    const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
+    return `
+You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.
+
+The user will give you a screenshot, the contents of it (optional), and some data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.
+
+If a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.
+
+If the user provides multiple reference images, please carefully review the reference images with the screenshot and provide the correct answer for <DATA_DEMAND>.
+
+
+Return in the following XML format:
+<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>
+<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>
+<errors>optional error messages as JSON array, e.g., ["error1", "error2"]</errors>
+
+# Example 1
+For example, if the DATA_DEMAND is:
+
+<DATA_DEMAND>
+{
+  "name": "name shows on the left panel, string",
+  "age": "age shows on the right panel, number",
+  "isAdmin": "if the user is admin, boolean"
+}
+</DATA_DEMAND>
+
+By viewing the screenshot and page contents, you can extract the following data:
+
+<thought>According to the screenshot, i can see ...</thought>
+<data-json>
+{
+  "name": "John",
+  "age": 30,
+  "isAdmin": true
+}
+</data-json>
+
+# Example 2
+If the DATA_DEMAND is:
+
+<DATA_DEMAND>
+the todo items list, string[]
+</DATA_DEMAND>
+
+By viewing the screenshot and page contents, you can extract the following data:
+
+<thought>According to the screenshot, i can see ...</thought>
+<data-json>
+["todo 1", "todo 2", "todo 3"]
+</data-json>
+
+# Example 3
+If the DATA_DEMAND is:
+
+<DATA_DEMAND>
+the page title, string
+</DATA_DEMAND>
+
+By viewing the screenshot and page contents, you can extract the following data:
+
+<thought>According to the screenshot, i can see ...</thought>
+<data-json>
+"todo list"
+</data-json>
+
+# Example 4
+If the DATA_DEMAND is:
+
+<DATA_DEMAND>
+{
+  "result": "Boolean, is it currently the SMS page?"
+}
+</DATA_DEMAND>
+
+By viewing the screenshot and page contents, you can extract the following data:
+
+<thought>According to the screenshot, i can see ...</thought>
+<data-json>
+{ "result": true }
+</data-json>
+`;
+}
+const extractDataQueryPrompt = (pageDescription, dataQuery)=>{
+    let dataQueryText = '';
+    dataQueryText = 'string' == typeof dataQuery ? dataQuery : JSON.stringify(dataQuery, null, 2);
+    return `
+<PageDescription>
+${pageDescription}
+</PageDescription>
+
+<DATA_DEMAND>
+${dataQueryText}
+</DATA_DEMAND>
+  `;
+};
+exports.extractDataQueryPrompt = __webpack_exports__.extractDataQueryPrompt;
+exports.parseXMLExtractionResponse = __webpack_exports__.parseXMLExtractionResponse;
+exports.systemPromptToExtract = __webpack_exports__.systemPromptToExtract;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "extractDataQueryPrompt",
+    "parseXMLExtractionResponse",
+    "systemPromptToExtract"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=extraction.js.map