@donggui/core 1.5.4-donggui.3

package/dist/lib/ai-model/prompt/extraction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { AIDataExtractionResponse } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\nimport { safeParseJson } from '../service-caller/index';\nimport { extractXMLTag } from './util';\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = safeParseJson(dataJsonStr, undefined) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = safeParseJson(errorsStr, undefined);\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract() {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\nThe user will give you a screenshot, the contents of it (optional), and some data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nIf the user provides multiple reference images, please carefully review the reference images with the screenshot and provide the correct answer for <DATA_DEMAND>.\n\n\nReturn in the following XML format:\n<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"result\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{ \"result\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","safeParseJson","undefined","e","errors","parsedErrors","Array","systemPromptToExtract","preferredLanguage","getPreferredLanguage","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACGO,SAASI,2BACdC,SAAiB;IAEjB,MAAMC,UAAUC,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IACzC,MAAMG,cAAcD,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAC7C,MAAMI,YAAYF,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,AAAAA,IAAAA,yBAAAA,aAAAA,AAAAA,EAAcJ,aAAaK;IACpC,EAAE,OAAOC,GAAG;QACV,MAAM,IAAIJ,MAAM,CAAC,2BAA2B,EAAEI,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIN,WACF,IAAI;QACF,MAAMO,eAAeJ,AAAAA,IAAAA,yBAAAA,aAAAA,AAAAA,EAAcH,WAAWI;QAC9C,IAAII,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIR,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAII,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG;IACd,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;;;;;;;;;0EAWgE,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqE9F,CAAC;AACD;AAEO,MAAME,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}

package/dist/lib/ai-model/prompt/llm-locator.js

@@ -0,0 +1,88 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    systemPromptToLocateElement: ()=>systemPromptToLocateElement,
+    findElementPrompt: ()=>findElementPrompt
+});
+const env_namespaceObject = require("@midscene/shared/env");
+const external_common_js_namespaceObject = require("./common.js");
+function systemPromptToLocateElement(modelFamily) {
+    const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
+    const bboxComment = (0, external_common_js_namespaceObject.bboxDescription)(modelFamily);
+    return `
+## Role:
+You are an AI assistant that helps identify UI elements.
+
+## Objective:
+- Identify elements in screenshots that match the user's description.
+- Provide the coordinates of the element that matches the user's description.
+
+## Important Notes for Locating Elements:
+- When the user describes an element that contains text (such as buttons, input fields, dropdown options, radio buttons, etc.), you should locate ONLY the text region of that element, not the entire element boundary.
+- For example: If an input field is large (both wide and tall) with a placeholder text "Please enter your comment", you should locate only the area where the placeholder text appears, not the entire input field.
+- This principle applies to all text-containing elements: focus on the visible text region rather than the full element container.
+
+## Output Format:
+\`\`\`json
+{
+  "bbox": [number, number, number, number],  // ${bboxComment}
+  "errors"?: string[]
+}
+\`\`\`
+
+Fields:
+* \`bbox\` is the bounding box of the element that matches the user's description
+* \`errors\` is an optional array of error messages (if any)
+
+For example, when an element is found:
+\`\`\`json
+{
+  "bbox": [100, 100, 200, 200],
+  "errors": []
+}
+\`\`\`
+
+When no element is found:
+\`\`\`json
+{
+  "bbox": [],
+  "errors": ["I can see ..., but {some element} is not found. Use ${preferredLanguage}."]
+}
+\`\`\`
+`;
+}
+const findElementPrompt = (targetElementDescription)=>`Find: ${targetElementDescription}`;
+exports.findElementPrompt = __webpack_exports__.findElementPrompt;
+exports.systemPromptToLocateElement = __webpack_exports__.systemPromptToLocateElement;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "findElementPrompt",
+    "systemPromptToLocateElement"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=llm-locator.js.map

package/dist/lib/ai-model/prompt/llm-locator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/llm-locator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-locator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { bboxDescription } from './common';\nexport function systemPromptToLocateElement(\n  modelFamily: TModelFamily | undefined,\n) {\n  const preferredLanguage = getPreferredLanguage();\n  const bboxComment = bboxDescription(modelFamily);\n  return `\n## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Identify elements in screenshots that match the user's description.\n- Provide the coordinates of the element that matches the user's description.\n\n## Important Notes for Locating Elements:\n- When the user describes an element that contains text (such as buttons, input fields, dropdown options, radio buttons, etc.), you should locate ONLY the text region of that element, not the entire element boundary.\n- For example: If an input field is large (both wide and tall) with a placeholder text \"Please enter your comment\", you should locate only the area where the placeholder text appears, not the entire input field.\n- This principle applies to all text-containing elements: focus on the visible text region rather than the full element container.\n\n## Output Format:\n\\`\\`\\`json\n{\n  \"bbox\": [number, number, number, number],  // ${bboxComment}\n  \"errors\"?: string[]\n}\n\\`\\`\\`\n\nFields:\n* \\`bbox\\` is the bounding box of the element that matches the user's description\n* \\`errors\\` is an optional array of error messages (if any)\n\nFor example, when an element is found:\n\\`\\`\\`json\n{\n  \"bbox\": [100, 100, 200, 200],\n  \"errors\": []\n}\n\\`\\`\\`\n\nWhen no element is found:\n\\`\\`\\`json\n{\n  \"bbox\": [],\n  \"errors\": [\"I can see ..., but {some element} is not found. Use ${preferredLanguage}.\"]\n}\n\\`\\`\\`\n`;\n}\n\nexport const findElementPrompt = (targetElementDescription: string) =>\n  `Find: ${targetElementDescription}`;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","systemPromptToLocateElement","modelFamily","preferredLanguage","getPreferredLanguage","bboxComment","bboxDescription","findElementPrompt","targetElementDescription"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACHO,SAASI,4BACdC,WAAqC;IAErC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAC1B,MAAMC,cAAcC,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EAAgBJ;IACpC,OAAO,CAAC;;;;;;;;;;;;;;;;gDAgBsC,EAAEG,YAAY;;;;;;;;;;;;;;;;;;;;;kEAqBI,EAAEF,kBAAkB;;;AAGtF,CAAC;AACD;AAEO,MAAMI,oBAAoB,CAACC,2BAChC,CAAC,MAAM,EAAEA,0BAA0B"}

package/dist/lib/ai-model/prompt/llm-planning.js

@@ -0,0 +1,401 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    descriptionForAction: ()=>descriptionForAction,
+    systemPromptToTaskPlanning: ()=>systemPromptToTaskPlanning
+});
+const external_common_js_namespaceObject = require("../../common.js");
+const env_namespaceObject = require("@midscene/shared/env");
+const zod_schema_utils_namespaceObject = require("@midscene/shared/zod-schema-utils");
+const external_common_js_namespaceObject_1 = require("./common.js");
+const vlLocateParam = (modelFamily)=>{
+    if (modelFamily) return `{bbox: [number, number, number, number], prompt: string } // ${(0, external_common_js_namespaceObject_1.bboxDescription)(modelFamily)}`;
+    return "{ prompt: string /* description of the target element */ }";
+};
+const findDefaultValue = (field)=>{
+    let current = field;
+    const visited = new Set();
+    while(current && !visited.has(current)){
+        visited.add(current);
+        const currentWithDef = current;
+        if (!currentWithDef._def?.typeName) break;
+        if ('ZodDefault' === currentWithDef._def.typeName) return currentWithDef._def.defaultValue?.();
+        if ('ZodOptional' === currentWithDef._def.typeName || 'ZodNullable' === currentWithDef._def.typeName) current = currentWithDef._def.innerType;
+        else break;
+    }
+};
+const SAMPLE_BBOXES = [
+    [
+        50,
+        100,
+        200,
+        200
+    ],
+    [
+        300,
+        400,
+        500,
+        500
+    ],
+    [
+        600,
+        100,
+        800,
+        250
+    ],
+    [
+        50,
+        600,
+        250,
+        750
+    ]
+];
+const injectBboxIntoSample = (sample, locateFields, includeBbox)=>{
+    if (!includeBbox) return sample;
+    const result = {
+        ...sample
+    };
+    let bboxIndex = 0;
+    for (const field of locateFields)if (result[field] && 'object' == typeof result[field] && result[field].prompt) {
+        result[field] = {
+            ...result[field],
+            bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length]
+        };
+        bboxIndex++;
+    }
+    return result;
+};
+const descriptionForAction = (action, locatorSchemaTypeDescription, includeBbox = false)=>{
+    const tab = '  ';
+    const fields = [];
+    fields.push(`- type: "${action.name}"`);
+    if (action.paramSchema) {
+        const paramLines = [];
+        const schema = action.paramSchema;
+        const isZodObject = schema._def?.typeName === 'ZodObject';
+        if (isZodObject && schema.shape) {
+            const shape = schema.shape;
+            for (const [key, field] of Object.entries(shape))if (field && 'object' == typeof field) {
+                const isOptional = 'function' == typeof field.isOptional && field.isOptional();
+                const keyWithOptional = isOptional ? `${key}?` : key;
+                const typeName = (0, zod_schema_utils_namespaceObject.getZodTypeName)(field, locatorSchemaTypeDescription);
+                const description = (0, zod_schema_utils_namespaceObject.getZodDescription)(field);
+                const defaultValue = findDefaultValue(field);
+                const hasDefault = void 0 !== defaultValue;
+                let paramLine = `${keyWithOptional}: ${typeName}`;
+                const comments = [];
+                if (description) comments.push(description);
+                if (hasDefault) {
+                    const defaultStr = 'string' == typeof defaultValue ? `"${defaultValue}"` : JSON.stringify(defaultValue);
+                    comments.push(`default: ${defaultStr}`);
+                }
+                if (comments.length > 0) paramLine += ` // ${comments.join(', ')}`;
+                paramLines.push(paramLine);
+            }
+            if (paramLines.length > 0) {
+                fields.push('- param:');
+                paramLines.forEach((line)=>{
+                    fields.push(`  - ${line}`);
+                });
+            }
+        } else {
+            const typeName = (0, zod_schema_utils_namespaceObject.getZodTypeName)(schema);
+            const description = (0, zod_schema_utils_namespaceObject.getZodDescription)(schema);
+            let paramDescription = `- param: ${typeName}`;
+            if (description) paramDescription += ` // ${description}`;
+            paramDescription += ' (pass the value directly, not as an object)';
+            fields.push(paramDescription);
+        }
+    }
+    if (action.sample && 'object' == typeof action.sample) {
+        const locateFields = (0, external_common_js_namespaceObject.findAllMidsceneLocatorField)(action.paramSchema);
+        const sampleWithBbox = injectBboxIntoSample(action.sample, locateFields, includeBbox);
+        const sampleStr = `- sample:\n${tab}${tab}<action-type>${action.name}</action-type>\n${tab}${tab}<action-param-json>\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\n/g, `\n${tab}${tab}`)}\n${tab}${tab}</action-param-json>`;
+        fields.push(sampleStr);
+    }
+    return `- ${action.name}, ${action.description || "No description provided"}
+${tab}${fields.join(`\n${tab}`)}
+`.trim();
+};
+const EXPLICIT_INSTRUCTION_RULES = `
+**Explicit instructions vs. High-level goals:**
+- **Explicit steps** (e.g., "click X", "type Y", "fill form"): Execute EXACTLY those steps - nothing more
+- **High-level goals** (e.g., "log in", "complete purchase"): Determine necessary steps
+
+**Examples:**
+- "fill out the form" → Fill fields only, do NOT submit
+- "click the button" → Click only, do NOT wait for page load
+- "type 'hello'" → Type only, do NOT press Enter
+- "select the first item" → Select only, do NOT proceed to checkout`;
+const ASSERTION_RULES = `
+**Assertion instructions:**
+- If assertion condition is NOT satisfied and cannot be satisfied, mark as failed (success="false")
+- If page is loading (spinner, skeleton, progress bar), wait before asserting`;
+const PAGE_NAVIGATION_RULES = `
+**Page navigation restriction:**
+- Complete task on current page unless explicitly instructed to navigate
+- Do NOT click links, use browser navigation, or open new URLs
+- If task cannot be accomplished, report failure instead of navigating`;
+const generateSubGoalSection = ()=>`
+* <update-plan-content> tag
+
+<update-plan-content>
+  <sub-goal index="1" status="finished|pending">description</sub-goal>
+  <sub-goal index="2" status="finished|pending">description</sub-goal>
+</update-plan-content>
+
+* <mark-sub-goal-done> tag
+
+<mark-sub-goal-done>
+  <sub-goal index="1" status="finished" />
+</mark-sub-goal-done>
+
+IMPORTANT: Mark sub-goal as "finished" ONLY AFTER confirming completion in screenshot.`;
+const generateSimplifiedExample = (includeSubGoals, locateNameField, locateEmailField)=>{
+    if (includeSubGoals) return `
+## Example: Fill registration form
+
+**User:** "fill out the form with name 'John' and email 'xxxx(真实的email地址)', then return the email"
+
+**Response:**
+<thought>Form has Name and Email fields. Both empty. Break into sub-goals.</thought>
+<update-plan-content>
+  <sub-goal index="1" status="pending">Fill Name with 'John'</sub-goal>
+  <sub-goal index="2" status="pending">Fill Email with 'xxxx(真实的email地址)'</sub-goal>
+  <sub-goal index="3" status="pending">Return email address</sub-goal>
+</update-plan-content>
+<log>Click Name field</log>
+<action-type>Tap</action-type>
+<action-param-json>{"locate": ${locateNameField}}</action-param-json>
+
+(After typing name)
+<thought>Name='John'. Sub-goal 1 complete. Fill Email next.</thought>
+<mark-sub-goal-done><sub-goal index="1" status="finished" /></mark-sub-goal-done>
+<log>Click Email field</log>
+<action-type>Tap</action-type>
+<action-param-json>{"locate": ${locateEmailField}}</action-param-json>
+
+(After typing email)
+<thought>Both fields filled. All sub-goals complete.</thought>
+<mark-sub-goal-done>
+  <sub-goal index="2" status="finished" />
+  <sub-goal index="3" status="finished" />
+</mark-sub-goal-done>
+<complete success="true">xxxx(真实的email地址)</complete>`;
+    return `
+## Example: Fill registration form
+
+**User:** "fill out the form with name 'John' and email 'xxxx(真实的email地址)', then return the email"
+
+**Response:**
+<thought>Form has Name and Email fields. Both empty. Start with Name.</thought>
+<log>Click Name field</log>
+<action-type>Tap</action-type>
+<action-param-json>{"locate": ${locateNameField}}</action-param-json>
+
+(After typing name)
+<thought>Name='John'. Fill Email next.</thought>
+<log>Click Email field</log>
+<action-type>Tap</action-type>
+<action-param-json>{"locate": ${locateEmailField}}</action-param-json>
+
+(After typing email)
+<thought>Both fields filled. Task complete.</thought>
+<complete success="true">xxxx(真实的email地址)</complete>`;
+};
+async function systemPromptToTaskPlanning({ actionSpace, modelFamily, includeBbox, includeThought, includeSubGoals }) {
+    const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
+    if (includeBbox && !modelFamily) throw new Error('modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.');
+    const actionDescriptionList = actionSpace.map((action)=>descriptionForAction(action, vlLocateParam(includeBbox ? modelFamily : void 0), includeBbox));
+    const actionList = actionDescriptionList.join('\n');
+    const shouldIncludeSubGoals = includeSubGoals ?? false;
+    const locateExample1 = includeBbox ? `{
+    "prompt": "Add to cart button for Sauce Labs Backpack",
+    "bbox": [345, 442, 458, 483]
+  }` : `{
+    "prompt": "Add to cart button for Sauce Labs Backpack"
+  }`;
+    const locateNameField = includeBbox ? `{
+    "prompt": "Name input field in the registration form",
+    "bbox": [120, 180, 380, 210]
+  }` : `{
+    "prompt": "Name input field in the registration form"
+  }`;
+    const locateEmailField = includeBbox ? `{
+    "prompt": "Email input field in the registration form",
+    "bbox": [120, 240, 380, 270]
+  }` : `{
+    "prompt": "Email input field in the registration form"
+  }`;
+    const step1Title = shouldIncludeSubGoals ? '## Step 1: Observe and Plan' : '## Step 1: Observe';
+    const step1Description = shouldIncludeSubGoals ? "Observe screenshot and logs, then break down instruction into sub-goals." : 'Observe screenshot and logs to understand current state.';
+    const thoughtTagDescription = `REQUIRED: Always output the <thought> tag.
+
+Answer these questions:
+- What is the user's requirement?
+- What is the current state?
+${shouldIncludeSubGoals ? '- Are all sub-goals completed?' : ''}
+- What should be the next action?
+
+${EXPLICIT_INSTRUCTION_RULES}`;
+    const subGoalSection = shouldIncludeSubGoals ? generateSubGoalSection() : '';
+    const memoryStepNumber = 2;
+    const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;
+    const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;
+    const goalAccomplished = shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled';
+    return `
+# Role
+You are an expert UI automation agent. Accomplish user instructions by observing screenshots and executing actions.
+
+# Steps
+
+${step1Title}
+
+${step1Description}
+
+* <thought> tag (REQUIRED)
+
+${thoughtTagDescription}
+${subGoalSection}
+${shouldIncludeSubGoals ? `
+## Step ${memoryStepNumber}: Memory Data
+
+Record information needed in follow-up actions. Current screenshot will NOT be available later.
+
+Examples: extracted data, element states, content to reference.
+` : ''}
+## Step ${checkGoalStepNumber}: Check if ${shouldIncludeSubGoals ? 'Goal is Accomplished' : 'Instruction is Fulfilled'}
+
+${shouldIncludeSubGoals ? 'Based on screenshot and sub-goals status, determine' : 'Determine'} if task is completed.
+
+### CRITICAL: User's Instruction is Supreme
+
+The user's instruction defines EXACT scope. Follow it precisely - nothing more, nothing less.
+
+${EXPLICIT_INSTRUCTION_RULES}
+
+**What "${goalAccomplished}" means:**
+- Done EXACTLY what user asked - no extra steps, no assumptions
+- Do NOT perform actions beyond explicit instruction
+
+${ASSERTION_RULES}
+${!shouldIncludeSubGoals ? `
+${PAGE_NAVIGATION_RULES}
+` : ''}
+### Output Rules
+
+- If NOT complete, continue to Step ${actionStepNumber}
+- Use <complete success="true|false">message</complete> to output result
+  - success="true" if ${goalAccomplished}
+  - success="false" if ${goalAccomplished.replace('is', 'is not').replace('fulfilled', 'fulfilled and cannot be fulfilled')}
+  - message: information for user (follow requested format)
+- If outputting <complete>, do NOT output <action-type> or <action-param-json>
+
+## Step ${actionStepNumber}: Determine Next Action
+
+ONLY if task is not complete: Determine next action based on current screenshot${shouldIncludeSubGoals ? ' and plan' : ''}.
+
+- Don't give extra actions beyond instruction${shouldIncludeSubGoals ? ' or plan' : ''}
+- Consider current screenshot - if element not visible, find it first
+- Ensure previous actions completed successfully
+- Give just the next ONE action
+- If errors persist >3 times, use <error> tag
+
+### Supporting actions list
+
+${actionList}
+
+### Log tag (preamble message)
+
+Brief message to user explaining next action:
+- **Use ${preferredLanguage}**
+- **Keep concise**: 1-2 sentences, 8-12 words
+- **Build on context**: connect with previous actions
+- **Friendly tone**: collaborative and engaging
+
+**Examples:**
+- <log>Click the login button</log>
+- <log>Scroll to find the 'Yes' button</log>
+
+### Action output
+
+Use <action-type> and <action-param-json> tags:
+- <action-type> MUST be from supporting actions list
+- Use EXACT field names from action description
+- 'complete' is NOT a valid action-type
+
+Example:
+<action-type>Tap</action-type>
+<action-param-json>
+{
+  "locate": ${locateExample1}
+}
+</action-param-json>
+
+### Error output
+
+Use <error> tag for error messages:
+<error>Unable to find the required element on the page</error>
+
+# Return Format
+
+**Always include (REQUIRED):**
+<thought>Your thought process here. NEVER skip this tag.</thought>
+${shouldIncludeSubGoals ? `
+<update-plan-content>...</update-plan-content>
+<mark-sub-goal-done>
+  <sub-goal index="1" status="finished" />
+</mark-sub-goal-done>
+` : ''}${shouldIncludeSubGoals ? `
+<memory>...</memory>
+` : ''}
+**Then choose ONE path:**
+
+**Path A: If ${goalAccomplished} or failed**
+<complete success="true|false">...</complete>
+
+**Path B: If ${goalAccomplished.replace('is', 'is not yet')}**
+<log>...</log>
+<action-type>...</action-type>
+<action-param-json>...</action-param-json>
+
+<!-- OR if there's an error -->
+<error>...</error>
+
+${generateSimplifiedExample(shouldIncludeSubGoals, locateNameField, locateEmailField)}
+`;
+}
+exports.descriptionForAction = __webpack_exports__.descriptionForAction;
+exports.systemPromptToTaskPlanning = __webpack_exports__.systemPromptToTaskPlanning;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "descriptionForAction",
+    "systemPromptToTaskPlanning"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=llm-planning.js.map

package/dist/lib/ai-model/prompt/llm-planning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/types';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport {\n  getZodDescription,\n  getZodTypeName,\n} from '@midscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\nconst vlLocateParam = (modelFamily: TModelFamily | undefined) => {\n  if (modelFamily) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(modelFamily)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\nconst findDefaultValue = (field: unknown): any | undefined => {\n  let current = field;\n  const visited = new Set<unknown>();\n\n  while (current && !visited.has(current)) {\n    visited.add(current);\n    const currentWithDef = current as {\n      _def?: {\n        typeName?: string;\n        defaultValue?: () => any;\n        innerType?: unknown;\n      };\n    };\n\n    if (!currentWithDef._def?.typeName) break;\n\n    if (currentWithDef._def.typeName === 'ZodDefault') {\n      return currentWithDef._def.defaultValue?.();\n    }\n\n    if (\n      currentWithDef._def.typeName === 'ZodOptional' ||\n      currentWithDef._def.typeName === 'ZodNullable'\n    ) {\n      current = currentWithDef._def.innerType;\n    } else {\n      break;\n    }\n  }\n\n  return undefined;\n};\n\nconst SAMPLE_BBOXES: [number, number, number, number][] = [\n  [50, 100, 200, 200],\n  [300, 400, 500, 500],\n  [600, 100, 800, 250],\n  [50, 600, 250, 750],\n];\n\nconst injectBboxIntoSample = (\n  sample: Record<string, any>,\n  locateFields: string[],\n  includeBbox: boolean,\n): Record<string, any> => {\n  if (!includeBbox) return sample;\n  const result = { ...sample };\n  let bboxIndex = 0;\n  for (const field of locateFields) {\n    if (\n      result[field] &&\n      typeof result[field] === 'object' &&\n      result[field].prompt\n    ) {\n      result[field] = {\n        ...result[field],\n        bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length],\n      };\n      bboxIndex++;\n    }\n  }\n  return result;\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n  includeBbox = false,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  fields.push(`- type: \"${action.name}\"`);\n\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      const shape = schema.shape;\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n          const description = getZodDescription(field as z.ZodTypeAny);\n\n          const defaultValue = findDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          const comments: string[] = [];\n          if (description) {\n            comments.push(description);\n          }\n          if (hasDefault) {\n            const defaultStr =\n              typeof defaultValue === 'string'\n                ? `\"${defaultValue}\"`\n                : JSON.stringify(defaultValue);\n            comments.push(`default: ${defaultStr}`);\n          }\n          if (comments.length > 0) {\n            paramLine += ` // ${comments.join(', ')}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  if (action.sample && typeof action.sample === 'object') {\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n    const sampleWithBbox = injectBboxIntoSample(\n      action.sample,\n      locateFields,\n      includeBbox,\n    );\n    const sampleStr = `- sample:\\n${tab}${tab}<action-type>${action.name}</action-type>\\n${tab}${tab}<action-param-json>\\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\\n/g, `\\n${tab}${tab}`)}\\n${tab}${tab}</action-param-json>`;\n    fields.push(sampleStr);\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nconst EXPLICIT_INSTRUCTION_RULES = `\n**Explicit instructions vs. High-level goals:**\n- **Explicit steps** (e.g., \"click X\", \"type Y\", \"fill form\"): Execute EXACTLY those steps - nothing more\n- **High-level goals** (e.g., \"log in\", \"complete purchase\"): Determine necessary steps\n\n**Examples:**\n- \"fill out the form\" → Fill fields only, do NOT submit\n- \"click the button\" → Click only, do NOT wait for page load\n- \"type 'hello'\" → Type only, do NOT press Enter\n- \"select the first item\" → Select only, do NOT proceed to checkout`;\n\nconst ASSERTION_RULES = `\n**Assertion instructions:**\n- If assertion condition is NOT satisfied and cannot be satisfied, mark as failed (success=\"false\")\n- If page is loading (spinner, skeleton, progress bar), wait before asserting`;\n\nconst PAGE_NAVIGATION_RULES = `\n**Page navigation restriction:**\n- Complete task on current page unless explicitly instructed to navigate\n- Do NOT click links, use browser navigation, or open new URLs\n- If task cannot be accomplished, report failure instead of navigating`;\n\nconst generateSubGoalSection = () => `\n* <update-plan-content> tag\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">description</sub-goal>\n</update-plan-content>\n\n* <mark-sub-goal-done> tag\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: Mark sub-goal as \"finished\" ONLY AFTER confirming completion in screenshot.`;\n\nconst generateSimplifiedExample = (\n  includeSubGoals: boolean,\n  locateNameField: string,\n  locateEmailField: string,\n) => {\n  if (includeSubGoals) {\n    return `\n## Example: Fill registration form\n\n**User:** \"fill out the form with name 'John' and email 'xxxx(真实的email地址)', then return the email\"\n\n**Response:**\n<thought>Form has Name and Email fields. Both empty. Break into sub-goals.</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Fill Name with 'John'</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Fill Email with 'xxxx(真实的email地址)'</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Return email address</sub-goal>\n</update-plan-content>\n<log>Click Name field</log>\n<action-type>Tap</action-type>\n<action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n(After typing name)\n<thought>Name='John'. Sub-goal 1 complete. Fill Email next.</thought>\n<mark-sub-goal-done><sub-goal index=\"1\" status=\"finished\" /></mark-sub-goal-done>\n<log>Click Email field</log>\n<action-type>Tap</action-type>\n<action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n(After typing email)\n<thought>Both fields filled. All sub-goals complete.</thought>\n<mark-sub-goal-done>\n  <sub-goal index=\"2\" status=\"finished\" />\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>\n<complete success=\"true\">xxxx(真实的email地址)</complete>`;\n  }\n\n  return `\n## Example: Fill registration form\n\n**User:** \"fill out the form with name 'John' and email 'xxxx(真实的email地址)', then return the email\"\n\n**Response:**\n<thought>Form has Name and Email fields. Both empty. Start with Name.</thought>\n<log>Click Name field</log>\n<action-type>Tap</action-type>\n<action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n(After typing name)\n<thought>Name='John'. Fill Email next.</thought>\n<log>Click Email field</log>\n<action-type>Tap</action-type>\n<action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n(After typing email)\n<thought>Both fields filled. Task complete.</thought>\n<complete success=\"true\">xxxx(真实的email地址)</complete>`;\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  modelFamily,\n  includeBbox,\n  includeThought,\n  includeSubGoals,\n}: {\n  actionSpace: DeviceAction<any>[];\n  modelFamily: TModelFamily | undefined;\n  includeBbox: boolean;\n  includeThought?: boolean;\n  includeSubGoals?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n\n  if (includeBbox && !modelFamily) {\n    throw new Error(\n      'modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? modelFamily : undefined),\n      includeBbox,\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const shouldIncludeThought = includeThought ?? true;\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n\n  const locateExample1 = includeBbox\n    ? `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\",\n    \"bbox\": [345, 442, 458, 483]\n  }`\n    : `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\"\n  }`;\n\n  const locateNameField = includeBbox\n    ? `{\n    \"prompt\": \"Name input field in the registration form\",\n    \"bbox\": [120, 180, 380, 210]\n  }`\n    : `{\n    \"prompt\": \"Name input field in the registration form\"\n  }`;\n\n  const locateEmailField = includeBbox\n    ? `{\n    \"prompt\": \"Email input field in the registration form\",\n    \"bbox\": [120, 240, 380, 270]\n  }`\n    : `{\n    \"prompt\": \"Email input field in the registration form\"\n  }`;\n\n  const step1Title = shouldIncludeSubGoals\n    ? '## Step 1: Observe and Plan'\n    : '## Step 1: Observe';\n\n  const step1Description = shouldIncludeSubGoals\n    ? \"Observe screenshot and logs, then break down instruction into sub-goals.\"\n    : 'Observe screenshot and logs to understand current state.';\n\n  const thoughtTagDescription = `REQUIRED: Always output the <thought> tag.\n\nAnswer these questions:\n- What is the user's requirement?\n- What is the current state?\n${shouldIncludeSubGoals ? '- Are all sub-goals completed?' : ''}\n- What should be the next action?\n\n${EXPLICIT_INSTRUCTION_RULES}`;\n\n  const subGoalSection = shouldIncludeSubGoals ? generateSubGoalSection() : '';\n\n  const memoryStepNumber = 2;\n  const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;\n\n  const goalTerm = shouldIncludeSubGoals ? 'goal' : 'instruction';\n  const goalAccomplished = shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled';\n\n  return `\n# Role\nYou are an expert UI automation agent. Accomplish user instructions by observing screenshots and executing actions.\n\n# Steps\n\n${step1Title}\n\n${step1Description}\n\n* <thought> tag (REQUIRED)\n\n${thoughtTagDescription}\n${subGoalSection}\n${\n  shouldIncludeSubGoals\n    ? `\n## Step ${memoryStepNumber}: Memory Data\n\nRecord information needed in follow-up actions. Current screenshot will NOT be available later.\n\nExamples: extracted data, element states, content to reference.\n`\n    : ''\n}\n## Step ${checkGoalStepNumber}: Check if ${shouldIncludeSubGoals ? 'Goal is Accomplished' : 'Instruction is Fulfilled'}\n\n${shouldIncludeSubGoals ? 'Based on screenshot and sub-goals status, determine' : 'Determine'} if task is completed.\n\n### CRITICAL: User's Instruction is Supreme\n\nThe user's instruction defines EXACT scope. Follow it precisely - nothing more, nothing less.\n\n${EXPLICIT_INSTRUCTION_RULES}\n\n**What \"${goalAccomplished}\" means:**\n- Done EXACTLY what user asked - no extra steps, no assumptions\n- Do NOT perform actions beyond explicit instruction\n\n${ASSERTION_RULES}\n${\n  !shouldIncludeSubGoals\n    ? `\n${PAGE_NAVIGATION_RULES}\n`\n    : ''\n}\n### Output Rules\n\n- If NOT complete, continue to Step ${actionStepNumber}\n- Use <complete success=\"true|false\">message</complete> to output result\n  - success=\"true\" if ${goalAccomplished}\n  - success=\"false\" if ${goalAccomplished.replace('is', 'is not').replace('fulfilled', 'fulfilled and cannot be fulfilled')}\n  - message: information for user (follow requested format)\n- If outputting <complete>, do NOT output <action-type> or <action-param-json>\n\n## Step ${actionStepNumber}: Determine Next Action\n\nONLY if task is not complete: Determine next action based on current screenshot${shouldIncludeSubGoals ? ' and plan' : ''}.\n\n- Don't give extra actions beyond instruction${shouldIncludeSubGoals ? ' or plan' : ''}\n- Consider current screenshot - if element not visible, find it first\n- Ensure previous actions completed successfully\n- Give just the next ONE action\n- If errors persist >3 times, use <error> tag\n\n### Supporting actions list\n\n${actionList}\n\n### Log tag (preamble message)\n\nBrief message to user explaining next action:\n- **Use ${preferredLanguage}**\n- **Keep concise**: 1-2 sentences, 8-12 words\n- **Build on context**: connect with previous actions\n- **Friendly tone**: collaborative and engaging\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button</log>\n\n### Action output\n\nUse <action-type> and <action-param-json> tags:\n- <action-type> MUST be from supporting actions list\n- Use EXACT field names from action description\n- 'complete' is NOT a valid action-type\n\nExample:\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateExample1}\n}\n</action-param-json>\n\n### Error output\n\nUse <error> tag for error messages:\n<error>Unable to find the required element on the page</error>\n\n# Return Format\n\n**Always include (REQUIRED):**\n<thought>Your thought process here. NEVER skip this tag.</thought>\n${\n  shouldIncludeSubGoals\n    ? `\n<update-plan-content>...</update-plan-content>\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n`\n    : ''\n}${\n  shouldIncludeSubGoals\n    ? `\n<memory>...</memory>\n`\n    : ''\n}\n**Then choose ONE path:**\n\n**Path A: If ${goalAccomplished} or failed**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If ${goalAccomplished.replace('is', 'is not yet')}**\n<log>...</log>\n<action-type>...</action-type>\n<action-param-json>...</action-param-json>\n\n<!-- OR if there's an error -->\n<error>...</error>\n\n${generateSimplifiedExample(shouldIncludeSubGoals, locateNameField, locateEmailField)}\n`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","vlLocateParam","modelFamily","bboxDescription","findDefaultValue","field","current","visited","Set","currentWithDef","SAMPLE_BBOXES","injectBboxIntoSample","sample","locateFields","includeBbox","result","bboxIndex","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","defaultValue","hasDefault","undefined","paramLine","comments","defaultStr","JSON","line","paramDescription","findAllMidsceneLocatorField","sampleWithBbox","sampleStr","EXPLICIT_INSTRUCTION_RULES","ASSERTION_RULES","PAGE_NAVIGATION_RULES","generateSubGoalSection","generateSimplifiedExample","includeSubGoals","locateNameField","locateEmailField","systemPromptToTaskPlanning","actionSpace","includeThought","preferredLanguage","getPreferredLanguage","Error","actionDescriptionList","actionList","shouldIncludeSubGoals","locateExample1","step1Title","step1Description","thoughtTagDescription","subGoalSection","memoryStepNumber","checkGoalStepNumber","actionStepNumber","goalAccomplished"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACKA,MAAMI,gBAAgB,CAACC;IACrB,IAAIA,aACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBD,cAAc;IAEvG,OAAO;AACT;AAEA,MAAME,mBAAmB,CAACC;IACxB,IAAIC,UAAUD;IACd,MAAME,UAAU,IAAIC;IAEpB,MAAOF,WAAW,CAACC,QAAQ,GAAG,CAACD,SAAU;QACvCC,QAAQ,GAAG,CAACD;QACZ,MAAMG,iBAAiBH;QAQvB,IAAI,CAACG,eAAe,IAAI,EAAE,UAAU;QAEpC,IAAIA,AAAiC,iBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAC9B,OAAOA,eAAe,IAAI,CAAC,YAAY;QAGzC,IACEA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,IAC5BA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAE5BH,UAAUG,eAAe,IAAI,CAAC,SAAS;aAEvC;IAEJ;AAGF;AAEA,MAAMC,gBAAoD;IACxD;QAAC;QAAI;QAAK;QAAK;KAAI;IACnB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;CACpB;AAED,MAAMC,uBAAuB,CAC3BC,QACAC,cACAC;IAEA,IAAI,CAACA,aAAa,OAAOF;IACzB,MAAMG,SAAS;QAAE,GAAGH,MAAM;IAAC;IAC3B,IAAII,YAAY;IAChB,KAAK,MAAMX,SAASQ,aAClB,IACEE,MAAM,CAACV,MAAM,IACb,AAAyB,YAAzB,OAAOU,MAAM,CAACV,MAAM,IACpBU,MAAM,CAACV,MAAM,CAAC,MAAM,EACpB;QACAU,MAAM,CAACV,MAAM,GAAG;YACd,GAAGU,MAAM,CAACV,MAAM;YAChB,MAAMK,aAAa,CAACM,YAAYN,cAAc,MAAM,CAAC;QACvD;QACAM;IACF;IAEF,OAAOD;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC,8BACAL,cAAc,KAAK;IAEnB,MAAMM,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAE3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAEtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAE/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAC/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAAC3B,KAAKS,MAAM,IAAIR,OAAO,OAAO,CAAC4B,OACxC,IAAIpB,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBACtC,MAAMqB,aACJ,AACE,cADF,OAAQrB,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAMsB,kBAAkBD,aAAa,GAAG9B,IAAI,CAAC,CAAC,GAAGA;gBAEjD,MAAMgC,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAexB,OAAOc;gBAEvC,MAAMW,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkB1B;gBAEtC,MAAM2B,eAAe5B,iBAAiBC;gBACtC,MAAM4B,aAAaD,AAAiBE,WAAjBF;gBAEnB,IAAIG,YAAY,GAAGR,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,MAAMQ,WAAqB,EAAE;gBAC7B,IAAIN,aACFM,SAAS,IAAI,CAACN;gBAEhB,IAAIG,YAAY;oBACd,MAAMI,aACJ,AAAwB,YAAxB,OAAOL,eACH,CAAC,CAAC,EAAEA,aAAa,CAAC,CAAC,GACnBM,KAAK,SAAS,CAACN;oBACrBI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAEC,YAAY;gBACxC;gBACA,IAAID,SAAS,MAAM,GAAG,GACpBD,aAAa,CAAC,IAAI,EAAEC,SAAS,IAAI,CAAC,OAAO;gBAG3Cd,WAAW,IAAI,CAACa;YAClB;YAGF,IAAIb,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACiB;oBAClBlB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEkB,MAAM;gBAC3B;YACF;QACF,OAAO;YACL,MAAMX,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAeN;YAChC,MAAMO,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkBR;YAEtC,IAAIiB,mBAAmB,CAAC,SAAS,EAAEZ,UAAU;YAC7C,IAAIE,aACFU,oBAAoB,CAAC,IAAI,EAAEV,aAAa;YAE1CU,oBAAoB;YAEpBnB,OAAO,IAAI,CAACmB;QACd;IACF;IAEA,IAAItB,OAAO,MAAM,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;QACtD,MAAML,eAAe4B,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BvB,OAAO,WAAW;QACnE,MAAMwB,iBAAiB/B,qBACrBO,OAAO,MAAM,EACbL,cACAC;QAEF,MAAM6B,YAAY,CAAC,WAAW,EAAEvB,MAAMA,IAAI,aAAa,EAAEF,OAAO,IAAI,CAAC,gBAAgB,EAAEE,MAAMA,IAAI,qBAAqB,EAAEA,MAAMA,MAAMkB,KAAK,SAAS,CAACI,gBAAgB,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,EAAEtB,MAAMA,KAAK,EAAE,EAAE,EAAEA,MAAMA,IAAI,oBAAoB,CAAC;QAChPC,OAAO,IAAI,CAACsB;IACd;IAEA,OAAO,CAAC,EAAE,EAAEzB,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEA,MAAMwB,6BAA6B,CAAC;;;;;;;;;mEAS+B,CAAC;AAEpE,MAAMC,kBAAkB,CAAC;;;6EAGoD,CAAC;AAE9E,MAAMC,wBAAwB,CAAC;;;;sEAIuC,CAAC;AAEvE,MAAMC,yBAAyB,IAAM,CAAC;;;;;;;;;;;;;;sFAcgD,CAAC;AAEvF,MAAMC,4BAA4B,CAChCC,iBACAC,iBACAC;IAEA,IAAIF,iBACF,OAAO,CAAC;;;;;;;;;;;;;;8BAckB,EAAEC,gBAAgB;;;;;;;8BAOlB,EAAEC,iBAAiB;;;;;;;;oDAQG,CAAC;IAGnD,OAAO,CAAC;;;;;;;;;8BASoB,EAAED,gBAAgB;;;;;;8BAMlB,EAAEC,iBAAiB;;;;oDAIG,CAAC;AACrD;AAEO,eAAeC,2BAA2B,EAC/CC,WAAW,EACXnD,WAAW,EACXY,WAAW,EACXwC,cAAc,EACdL,eAAe,EAOhB;IACC,MAAMM,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,IAAI1C,eAAe,CAACZ,aAClB,MAAM,IAAIuD,MACR;IAIJ,MAAMC,wBAAwBL,YAAY,GAAG,CAAC,CAACnC,SACtCD,qBACLC,QACAjB,cAAca,cAAcZ,cAAcgC,SAC1CpB;IAGJ,MAAM6C,aAAaD,sBAAsB,IAAI,CAAC;IAG9C,MAAME,wBAAwBX,mBAAmB;IAEjD,MAAMY,iBAAiB/C,cACnB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAMoC,kBAAkBpC,cACpB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAMqC,mBAAmBrC,cACrB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAMgD,aAAaF,wBACf,gCACA;IAEJ,MAAMG,mBAAmBH,wBACrB,6EACA;IAEJ,MAAMI,wBAAwB,CAAC;;;;;AAKjC,EAAEJ,wBAAwB,mCAAmC,GAAG;;;AAGhE,EAAEhB,4BAA4B;IAE5B,MAAMqB,iBAAiBL,wBAAwBb,2BAA2B;IAE1E,MAAMmB,mBAAmB;IACzB,MAAMC,sBAAsBP,wBAAwB,IAAI;IACxD,MAAMQ,mBAAmBR,wBAAwB,IAAI;IAGrD,MAAMS,mBAAmBT,wBAAwB,yBAAyB;IAE1E,OAAO,CAAC;;;;;;AAMV,EAAEE,WAAW;;AAEb,EAAEC,iBAAiB;;;;AAInB,EAAEC,sBAAsB;AACxB,EAAEC,eAAe;AACjB,EACEL,wBACI,CAAC;QACC,EAAEM,iBAAiB;;;;;AAK3B,CAAC,GACK,GACL;QACO,EAAEC,oBAAoB,WAAW,EAAEP,wBAAwB,yBAAyB,2BAA2B;;AAEvH,EAAEA,wBAAwB,wDAAwD,YAAY;;;;;;AAM9F,EAAEhB,2BAA2B;;QAErB,EAAEyB,iBAAiB;;;;AAI3B,EAAExB,gBAAgB;AAClB,EACE,CAACe,wBACG,CAAC;AACP,EAAEd,sBAAsB;AACxB,CAAC,GACK,GACL;;;oCAGmC,EAAEsB,iBAAiB;;sBAEjC,EAAEC,iBAAiB;uBAClB,EAAEA,iBAAiB,OAAO,CAAC,MAAM,UAAU,OAAO,CAAC,aAAa,qCAAqC;;;;QAIpH,EAAED,iBAAiB;;+EAEoD,EAAER,wBAAwB,cAAc,GAAG;;6CAE7E,EAAEA,wBAAwB,aAAa,GAAG;;;;;;;;AAQvF,EAAED,WAAW;;;;;QAKL,EAAEJ,kBAAkB;;;;;;;;;;;;;;;;;;;;YAoBhB,EAAEM,eAAe;;;;;;;;;;;;;AAa7B,EACED,wBACI,CAAC;;;;;AAKP,CAAC,GACK,KAEJA,wBACI,CAAC;;AAEP,CAAC,GACK,GACL;;;aAGY,EAAES,iBAAiB;;;aAGnB,EAAEA,iBAAiB,OAAO,CAAC,MAAM,cAAc;;;;;;;;AAQ5D,EAAErB,0BAA0BY,uBAAuBV,iBAAiBC,kBAAkB;AACtF,CAAC;AACD"}

package/dist/lib/ai-model/prompt/llm-section-locator.js

@@ -0,0 +1,81 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    sectionLocatorInstruction: ()=>sectionLocatorInstruction,
+    systemPromptToLocateSection: ()=>systemPromptToLocateSection
+});
+const env_namespaceObject = require("@midscene/shared/env");
+const external_common_js_namespaceObject = require("./common.js");
+function systemPromptToLocateSection(modelFamily) {
+    const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
+    const bboxFormat = (0, external_common_js_namespaceObject.bboxDescription)(modelFamily);
+    return `
+## Role:
+You are an AI assistant that helps identify UI elements.
+
+## Objective:
+- Find a section containing the target element
+- If the description mentions reference elements, also locate sections containing those references
+
+## Output Format:
+\`\`\`json
+{
+  "bbox": [number, number, number, number],  // ${bboxFormat}
+  "references_bbox"?: [
+    [number, number, number, number],
+    ...
+  ],
+  "error"?: string
+}
+\`\`\`
+
+Fields:
+* \`bbox\` - Bounding box of the section containing the target element
+* \`references_bbox\` - Optional array of bounding boxes for reference elements
+* \`error\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.
+
+Example:
+If the description is "delete button on the second row with title 'Peter'", return:
+\`\`\`json
+{
+  "bbox": [100, 100, 200, 200],
+  "references_bbox": [[100, 100, 200, 200]]
+}
+\`\`\`
+`;
+}
+const sectionLocatorInstruction = (sectionDescription)=>`Find section containing: ${sectionDescription}`;
+exports.sectionLocatorInstruction = __webpack_exports__.sectionLocatorInstruction;
+exports.systemPromptToLocateSection = __webpack_exports__.systemPromptToLocateSection;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "sectionLocatorInstruction",
+    "systemPromptToLocateSection"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=llm-section-locator.js.map