@aiscene/core 9.0.6 → 9.0.8

package/dist/lib/ai-model/shared/multimodal-prompt.js

@@ -0,0 +1,95 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    preparedReferenceImagesToChatMessages: ()=>preparedReferenceImagesToChatMessages,
+    multimodalPromptToChatMessages: ()=>multimodalPromptToChatMessages,
+    prepareUserPrompt: ()=>prepareUserPrompt
+});
+const external_common_js_namespaceObject = require("../../common.js");
+const img_namespaceObject = require("@aiscene/shared/img");
+const prepareReferenceImages = async (multimodalPrompt)=>{
+    const referenceImages = [];
+    for (const image of multimodalPrompt?.images ?? [])referenceImages.push({
+        name: image.name,
+        url: await (0, img_namespaceObject.preProcessImageUrl)(image.url, !!multimodalPrompt?.convertHttpImage2Base64)
+    });
+    return referenceImages;
+};
+const prepareUserPrompt = async (userPrompt)=>({
+        text: (0, external_common_js_namespaceObject.userPromptToString)(userPrompt),
+        referenceImages: await prepareReferenceImages((0, external_common_js_namespaceObject.userPromptToMultimodalPrompt)(userPrompt))
+    });
+const preparedReferenceImagesToChatMessages = (referenceImages)=>{
+    if (0 === referenceImages.length) return [];
+    return [
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.'
+                }
+            ]
+        },
+        ...referenceImages.flatMap((image)=>[
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: `this is the reference image named '${image.name}'. It is a reference image, not the current screenshot:`
+                        }
+                    ]
+                },
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'image_url',
+                            image_url: {
+                                url: image.url,
+                                detail: 'high'
+                            }
+                        }
+                    ]
+                }
+            ])
+    ];
+};
+const multimodalPromptToChatMessages = async (multimodalPrompt)=>preparedReferenceImagesToChatMessages(await prepareReferenceImages(multimodalPrompt));
+exports.multimodalPromptToChatMessages = __webpack_exports__.multimodalPromptToChatMessages;
+exports.prepareUserPrompt = __webpack_exports__.prepareUserPrompt;
+exports.preparedReferenceImagesToChatMessages = __webpack_exports__.preparedReferenceImagesToChatMessages;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "multimodalPromptToChatMessages",
+    "prepareUserPrompt",
+    "preparedReferenceImagesToChatMessages"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=multimodal-prompt.js.map

package/dist/lib/ai-model/shared/multimodal-prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/multimodal-prompt.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/shared/multimodal-prompt.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport { preProcessImageUrl } from '@aiscene/shared/img';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\n\nexport interface PreparedReferenceImage {\n  name: string;\n  url: string;\n}\n\nexport interface PreparedUserPrompt {\n  text: string;\n  referenceImages: PreparedReferenceImage[];\n}\n\nconst prepareReferenceImages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<PreparedReferenceImage[]> => {\n  const referenceImages: PreparedReferenceImage[] = [];\n  for (const image of multimodalPrompt?.images ?? []) {\n    referenceImages.push({\n      name: image.name,\n      url: await preProcessImageUrl(\n        image.url,\n        !!multimodalPrompt?.convertHttpImage2Base64,\n      ),\n    });\n  }\n  return referenceImages;\n};\n\nexport const prepareUserPrompt = async (\n  userPrompt: TUserPrompt,\n): Promise<PreparedUserPrompt> => ({\n  text: userPromptToString(userPrompt),\n  referenceImages: await prepareReferenceImages(\n    userPromptToMultimodalPrompt(userPrompt),\n  ),\n});\n\nexport const preparedReferenceImagesToChatMessages = (\n  referenceImages: PreparedReferenceImage[],\n): ChatCompletionUserMessageParam[] => {\n  if (referenceImages.length === 0) {\n    return [];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.',\n        },\n      ],\n    },\n    ...referenceImages.flatMap((image): ChatCompletionUserMessageParam[] => [\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${image.name}'. It is a reference image, not the current screenshot:`,\n          },\n        ],\n      },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: image.url,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ]),\n  ];\n};\n\nexport const multimodalPromptToChatMessages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> =>\n  preparedReferenceImagesToChatMessages(\n    await prepareReferenceImages(multimodalPrompt),\n  );\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","prepareReferenceImages","multimodalPrompt","referenceImages","image","preProcessImageUrl","prepareUserPrompt","userPrompt","userPromptToString","userPromptToMultimodalPrompt","preparedReferenceImagesToChatMessages","multimodalPromptToChatMessages"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACaA,MAAMI,yBAAyB,OAC7BC;IAEA,MAAMC,kBAA4C,EAAE;IACpD,KAAK,MAAMC,SAASF,kBAAkB,UAAU,EAAE,CAChDC,gBAAgB,IAAI,CAAC;QACnB,MAAMC,MAAM,IAAI;QAChB,KAAK,MAAMC,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EACTD,MAAM,GAAG,EACT,CAAC,CAACF,kBAAkB;IAExB;IAEF,OAAOC;AACT;AAEO,MAAMG,oBAAoB,OAC/BC,aACiC;QACjC,MAAMC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBD;QACzB,iBAAiB,MAAMN,uBACrBQ,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6BF;IAEjC;AAEO,MAAMG,wCAAwC,CACnDP;IAEA,IAAIA,AAA2B,MAA3BA,gBAAgB,MAAM,EACxB,OAAO,EAAE;IAGX,OAAO;QACL;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;WACGA,gBAAgB,OAAO,CAAC,CAACC,QAA4C;gBACtE;oBACE,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,MAAM,CAAC,mCAAmC,EAAEA,MAAM,IAAI,CAAC,uDAAuD,CAAC;wBACjH;qBACD;gBACH;gBACA;oBACE,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,WAAW;gCACT,KAAKA,MAAM,GAAG;gCACd,QAAQ;4BACV;wBACF;qBACD;gBACH;aACD;KACF;AACH;AAEO,MAAMO,iCAAiC,OAC5CT,mBAEAQ,sCACE,MAAMT,uBAAuBC"}

package/dist/lib/ai-model/shared/xml.js

@@ -0,0 +1,79 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    extractRawXMLFragment: ()=>extractRawXMLFragment,
+    extractXMLTag: ()=>extractXMLTag
+});
+function extractXMLTag(xmlString, tagName) {
+    const lowerXmlString = xmlString.toLowerCase();
+    const lowerTagName = tagName.toLowerCase();
+    const closeTag = `</${lowerTagName}>`;
+    const openTag = `<${lowerTagName}>`;
+    const lastCloseIndex = lowerXmlString.lastIndexOf(closeTag);
+    if (-1 === lastCloseIndex) {
+        const lastOpenIndex = lowerXmlString.lastIndexOf(openTag);
+        if (-1 === lastOpenIndex) return;
+        const contentStart = lastOpenIndex + openTag.length;
+        const remaining = xmlString.substring(contentStart);
+        const nextTagIndex = remaining.indexOf('<');
+        const content = -1 === nextTagIndex ? remaining : remaining.substring(0, nextTagIndex);
+        return content.trim();
+    }
+    const searchArea = lowerXmlString.substring(0, lastCloseIndex);
+    const lastOpenIndex = searchArea.lastIndexOf(openTag);
+    if (-1 === lastOpenIndex) return;
+    const contentStart = lastOpenIndex + openTag.length;
+    const contentEnd = lastCloseIndex;
+    const content = xmlString.substring(contentStart, contentEnd);
+    return content.trim();
+}
+const escapeRegExp = (value)=>value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+function extractRawXMLFragment(content, tagNames) {
+    const startTagName = tagNames[0];
+    const endTagName = tagNames.at(-1);
+    const startTagRegex = new RegExp(`<${escapeRegExp(startTagName)}>`, 'i');
+    const startTag = startTagRegex.exec(content);
+    if (startTag?.index === void 0) return '';
+    const contentFromStartTag = content.slice(startTag.index);
+    const endTagRegex = new RegExp(`</${escapeRegExp(endTagName)}>`, 'gi');
+    const endTags = [
+        ...contentFromStartTag.matchAll(endTagRegex)
+    ];
+    const lastEndTag = endTags.at(-1);
+    const end = lastEndTag?.index !== void 0 ? lastEndTag.index + lastEndTag[0].length : contentFromStartTag.length;
+    return contentFromStartTag.slice(0, end);
+}
+exports.extractRawXMLFragment = __webpack_exports__.extractRawXMLFragment;
+exports.extractXMLTag = __webpack_exports__.extractXMLTag;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "extractRawXMLFragment",
+    "extractXMLTag"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=xml.js.map

package/dist/lib/ai-model/shared/xml.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/xml.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/shared/xml.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/**\n * Extract content from an XML tag in a string, searching from the end.\n * This approach handles cases where models prepend thinking content (like <think>...</think>)\n * before the actual response tags, or when there are incomplete/nested tags.\n *\n * Strategy: Find the LAST closing tag, then search backwards for the nearest opening tag.\n * This ensures we get the last complete tag pair, even if there are incomplete tags before it.\n *\n * @param xmlString - The XML string to parse\n * @param tagName - The name of the tag to extract (case-insensitive)\n * @returns The trimmed content of the tag, or undefined if not found\n */\nexport function extractXMLTag(\n  xmlString: string,\n  tagName: string,\n): string | undefined {\n  const lowerXmlString = xmlString.toLowerCase();\n  const lowerTagName = tagName.toLowerCase();\n  const closeTag = `</${lowerTagName}>`;\n  const openTag = `<${lowerTagName}>`;\n\n  // Find the last closing tag\n  const lastCloseIndex = lowerXmlString.lastIndexOf(closeTag);\n  if (lastCloseIndex === -1) {\n    // Fallback: handle half-open tags like `<action-type>Input` without\n    // matching close tag. Extract until the next XML tag boundary.\n    const lastOpenIndex = lowerXmlString.lastIndexOf(openTag);\n    if (lastOpenIndex === -1) {\n      return undefined;\n    }\n\n    const contentStart = lastOpenIndex + openTag.length;\n    const remaining = xmlString.substring(contentStart);\n    const nextTagIndex = remaining.indexOf('<');\n    const content =\n      nextTagIndex === -1 ? remaining : remaining.substring(0, nextTagIndex);\n\n    return content.trim();\n  }\n\n  // Search backwards from the closing tag to find the nearest opening tag\n  const searchArea = lowerXmlString.substring(0, lastCloseIndex);\n  const lastOpenIndex = searchArea.lastIndexOf(openTag);\n  if (lastOpenIndex === -1) {\n    return undefined;\n  }\n\n  // Extract content between the tags (use original string to preserve case)\n  const contentStart = lastOpenIndex + openTag.length;\n  const contentEnd = lastCloseIndex;\n  const content = xmlString.substring(contentStart, contentEnd);\n\n  return content.trim();\n}\n\nconst escapeRegExp = (value: string) =>\n  value.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n\n/** Extract raw XML from the first protocol tag through the last protocol tag. */\nexport function extractRawXMLFragment(\n  content: string,\n  tagNames: readonly [string, ...string[]],\n): string {\n  const startTagName = tagNames[0];\n  const endTagName = tagNames.at(-1)!;\n  const startTagRegex = new RegExp(`<${escapeRegExp(startTagName)}>`, 'i');\n  const startTag = startTagRegex.exec(content);\n  if (startTag?.index === undefined) {\n    return '';\n  }\n\n  const contentFromStartTag = content.slice(startTag.index);\n  const endTagRegex = new RegExp(`</${escapeRegExp(endTagName)}>`, 'gi');\n  const endTags = [...contentFromStartTag.matchAll(endTagRegex)];\n  const lastEndTag = endTags.at(-1);\n  const end =\n    lastEndTag?.index !== undefined\n      ? lastEndTag.index + lastEndTag[0].length\n      : contentFromStartTag.length;\n\n  return contentFromStartTag.slice(0, end);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","extractXMLTag","xmlString","tagName","lowerXmlString","lowerTagName","closeTag","openTag","lastCloseIndex","lastOpenIndex","contentStart","remaining","nextTagIndex","content","searchArea","contentEnd","escapeRegExp","value","extractRawXMLFragment","tagNames","startTagName","endTagName","startTagRegex","RegExp","startTag","undefined","contentFromStartTag","endTagRegex","endTags","lastEndTag","end"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACMO,SAASI,cACdC,SAAiB,EACjBC,OAAe;IAEf,MAAMC,iBAAiBF,UAAU,WAAW;IAC5C,MAAMG,eAAeF,QAAQ,WAAW;IACxC,MAAMG,WAAW,CAAC,EAAE,EAAED,aAAa,CAAC,CAAC;IACrC,MAAME,UAAU,CAAC,CAAC,EAAEF,aAAa,CAAC,CAAC;IAGnC,MAAMG,iBAAiBJ,eAAe,WAAW,CAACE;IAClD,IAAIE,AAAmB,OAAnBA,gBAAuB;QAGzB,MAAMC,gBAAgBL,eAAe,WAAW,CAACG;QACjD,IAAIE,AAAkB,OAAlBA,eACF;QAGF,MAAMC,eAAeD,gBAAgBF,QAAQ,MAAM;QACnD,MAAMI,YAAYT,UAAU,SAAS,CAACQ;QACtC,MAAME,eAAeD,UAAU,OAAO,CAAC;QACvC,MAAME,UACJD,AAAiB,OAAjBA,eAAsBD,YAAYA,UAAU,SAAS,CAAC,GAAGC;QAE3D,OAAOC,QAAQ,IAAI;IACrB;IAGA,MAAMC,aAAaV,eAAe,SAAS,CAAC,GAAGI;IAC/C,MAAMC,gBAAgBK,WAAW,WAAW,CAACP;IAC7C,IAAIE,AAAkB,OAAlBA,eACF;IAIF,MAAMC,eAAeD,gBAAgBF,QAAQ,MAAM;IACnD,MAAMQ,aAAaP;IACnB,MAAMK,UAAUX,UAAU,SAAS,CAACQ,cAAcK;IAElD,OAAOF,QAAQ,IAAI;AACrB;AAEA,MAAMG,eAAe,CAACC,QACpBA,MAAM,OAAO,CAAC,uBAAuB;AAGhC,SAASC,sBACdL,OAAe,EACfM,QAAwC;IAExC,MAAMC,eAAeD,QAAQ,CAAC,EAAE;IAChC,MAAME,aAAaF,SAAS,EAAE,CAAC;IAC/B,MAAMG,gBAAgB,IAAIC,OAAO,CAAC,CAAC,EAAEP,aAAaI,cAAc,CAAC,CAAC,EAAE;IACpE,MAAMI,WAAWF,cAAc,IAAI,CAACT;IACpC,IAAIW,UAAU,UAAUC,QACtB,OAAO;IAGT,MAAMC,sBAAsBb,QAAQ,KAAK,CAACW,SAAS,KAAK;IACxD,MAAMG,cAAc,IAAIJ,OAAO,CAAC,EAAE,EAAEP,aAAaK,YAAY,CAAC,CAAC,EAAE;IACjE,MAAMO,UAAU;WAAIF,oBAAoB,QAAQ,CAACC;KAAa;IAC9D,MAAME,aAAaD,QAAQ,EAAE,CAAC;IAC9B,MAAME,MACJD,YAAY,UAAUJ,SAClBI,WAAW,KAAK,GAAGA,UAAU,CAAC,EAAE,CAAC,MAAM,GACvCH,oBAAoB,MAAM;IAEhC,OAAOA,oBAAoB,KAAK,CAAC,GAAGI;AACtC"}

package/dist/lib/ai-model/workflows/grounding/index.js

@@ -0,0 +1,46 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    AiLocateSection: ()=>external_section_locate_js_namespaceObject.AiLocateSection,
+    AiLocateElement: ()=>external_locate_js_namespaceObject.AiLocateElement,
+    buildSearchAreaConfig: ()=>external_search_area_js_namespaceObject.buildSearchAreaConfig
+});
+const external_locate_js_namespaceObject = require("./locate.js");
+const external_section_locate_js_namespaceObject = require("./section-locate.js");
+const external_search_area_js_namespaceObject = require("./search-area.js");
+exports.AiLocateElement = __webpack_exports__.AiLocateElement;
+exports.AiLocateSection = __webpack_exports__.AiLocateSection;
+exports.buildSearchAreaConfig = __webpack_exports__.buildSearchAreaConfig;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "AiLocateElement",
+    "AiLocateSection",
+    "buildSearchAreaConfig"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=index.js.map

package/dist/lib/ai-model/workflows/grounding/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/ai-model/workflows/grounding/locate-result-rect.js

@@ -0,0 +1,60 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    mergePixelBboxesToRect: ()=>mergePixelBboxesToRect,
+    pixelBboxToRect: ()=>pixelBboxToRect
+});
+function mergePixelBboxesToRect(pixelBboxes) {
+    const minLeft = Math.min(...pixelBboxes.map(([left])=>left));
+    const minTop = Math.min(...pixelBboxes.map(([, top])=>top));
+    const maxRight = Math.max(...pixelBboxes.map(([, , right])=>right));
+    const maxBottom = Math.max(...pixelBboxes.map(([, , , bottom])=>bottom));
+    return pixelBboxToRect([
+        minLeft,
+        minTop,
+        maxRight,
+        maxBottom
+    ]);
+}
+function pixelBboxToRect([left, top, right, bottom]) {
+    return {
+        left,
+        top,
+        width: right - left + 1,
+        height: bottom - top + 1
+    };
+}
+exports.mergePixelBboxesToRect = __webpack_exports__.mergePixelBboxesToRect;
+exports.pixelBboxToRect = __webpack_exports__.pixelBboxToRect;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "mergePixelBboxesToRect",
+    "pixelBboxToRect"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=locate-result-rect.js.map

package/dist/lib/ai-model/workflows/grounding/locate-result-rect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/locate-result-rect.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/locate-result-rect.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PixelBbox } from '@/ai-model/shared/model-locate-result';\nimport type { Rect } from '@/types';\n\n/** Merge model pixel bboxes into one screenshot rectangle. */\nexport function mergePixelBboxesToRect(pixelBboxes: PixelBbox[]): Rect {\n  const minLeft = Math.min(...pixelBboxes.map(([left]) => left));\n  const minTop = Math.min(...pixelBboxes.map(([, top]) => top));\n  const maxRight = Math.max(...pixelBboxes.map(([, , right]) => right));\n  const maxBottom = Math.max(...pixelBboxes.map(([, , , bottom]) => bottom));\n  return pixelBboxToRect([minLeft, minTop, maxRight, maxBottom]);\n}\n\nexport function pixelBboxToRect([left, top, right, bottom]: PixelBbox): Rect {\n  return {\n    left,\n    top,\n    width: right - left + 1,\n    height: bottom - top + 1,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","mergePixelBboxesToRect","pixelBboxes","minLeft","Math","left","minTop","top","maxRight","right","maxBottom","bottom","pixelBboxToRect"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACFO,SAASI,uBAAuBC,WAAwB;IAC7D,MAAMC,UAAUC,KAAK,GAAG,IAAIF,YAAY,GAAG,CAAC,CAAC,CAACG,KAAK,GAAKA;IACxD,MAAMC,SAASF,KAAK,GAAG,IAAIF,YAAY,GAAG,CAAC,CAAC,GAAGK,IAAI,GAAKA;IACxD,MAAMC,WAAWJ,KAAK,GAAG,IAAIF,YAAY,GAAG,CAAC,CAAC,KAAKO,MAAM,GAAKA;IAC9D,MAAMC,YAAYN,KAAK,GAAG,IAAIF,YAAY,GAAG,CAAC,CAAC,OAAOS,OAAO,GAAKA;IAClE,OAAOC,gBAAgB;QAACT;QAASG;QAAQE;QAAUE;KAAU;AAC/D;AAEO,SAASE,gBAAgB,CAACP,MAAME,KAAKE,OAAOE,OAAkB;IACnE,OAAO;QACLN;QACAE;QACA,OAAOE,QAAQJ,OAAO;QACtB,QAAQM,SAASJ,MAAM;IACzB;AACF"}

package/dist/lib/ai-model/workflows/grounding/locate.js

@@ -0,0 +1,208 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    genericLocate: ()=>genericLocate,
+    AiLocateElement: ()=>AiLocateElement,
+    extraTextFromUserPrompt: ()=>external_common_js_namespaceObject.userPromptToString
+});
+const extractor_namespaceObject = require("@aiscene/shared/extractor");
+const logger_namespaceObject = require("@aiscene/shared/logger");
+const utils_namespaceObject = require("@aiscene/shared/utils");
+const external_common_js_namespaceObject = require("../../../common.js");
+const index_js_namespaceObject = require("../../prompt/locate/index.js");
+const external_service_caller_index_js_namespaceObject = require("../../service-caller/index.js");
+const semantic_retry_js_namespaceObject = require("../../service-caller/semantic-retry.js");
+const external_locate_result_rect_js_namespaceObject = require("./locate-result-rect.js");
+const external_search_area_mapping_js_namespaceObject = require("./search-area-mapping.js");
+const external_utils_js_namespaceObject = require("./utils.js");
+const debugGrounding = (0, logger_namespaceObject.getDebug)('ai:grounding');
+async function AiLocateElement(options) {
+    const { targetElementDescription, ...locateOptions } = options;
+    (0, utils_namespaceObject.assert)(targetElementDescription, "cannot find the target element description");
+    const { context } = locateOptions;
+    const locateImage = locateOptions.searchConfig?.image ?? {
+        imageBase64: context.screenshot.base64,
+        width: context.shotSize.width,
+        height: context.shotSize.height
+    };
+    const locateRequest = {
+        targetElementDescription,
+        locateImage,
+        options: locateOptions
+    };
+    const locateAdapter = options.modelRuntime.adapter.locate;
+    const locateFn = 'custom' === locateAdapter.kind ? locateAdapter.locateFn : genericLocate;
+    const locateResponse = await locateFn(locateRequest);
+    const { locatedPixelBbox, rawResponse, rawChoiceMessage, usage, reasoningContent, errors = [] } = locateResponse;
+    const baseLocateResult = {
+        rawResponse,
+        rawChoiceMessage,
+        usage,
+        reasoning_content: reasoningContent
+    };
+    if (!locatedPixelBbox) return {
+        rect: void 0,
+        parseResult: {
+            element: void 0,
+            errors
+        },
+        ...baseLocateResult
+    };
+    try {
+        const rect = (0, external_locate_result_rect_js_namespaceObject.pixelBboxToRect)((0, external_search_area_mapping_js_namespaceObject.mapSearchAreaPixelBboxToOriginalPixelBbox)(locatedPixelBbox, locateOptions.searchConfig?.mapping));
+        debugGrounding('resRect', rect);
+        return {
+            rect,
+            parseResult: {
+                element: (0, extractor_namespaceObject.generateElementByRect)(rect, (0, external_common_js_namespaceObject.userPromptToString)(targetElementDescription)),
+                errors: []
+            },
+            ...baseLocateResult
+        };
+    } catch (error) {
+        const msg = error instanceof Error ? `Failed to parse locate result: ${error.message}` : 'unknown error in locate';
+        return {
+            rect: void 0,
+            parseResult: {
+                element: void 0,
+                errors: errors.length > 0 ? [
+                    ...errors,
+                    `(${msg})`
+                ] : [
+                    msg
+                ]
+            },
+            ...baseLocateResult
+        };
+    }
+}
+async function genericLocate(locateRequest) {
+    const { options, targetElementDescription } = locateRequest;
+    const modelRuntime = options.modelRuntime;
+    const { adapter } = modelRuntime;
+    (0, utils_namespaceObject.assert)('standard' === adapter.locate.kind, 'generic locate requires a standard locate adapter');
+    const { protocol, resultCodec } = adapter.locate.element;
+    const systemPrompt = (0, index_js_namespaceObject.buildElementLocateSystemPrompt)({
+        systemPromptIntroduction: protocol.systemPromptIntroduction,
+        responseInstructions: protocol.buildResponseInstructions(resultCodec.promptSpec)
+    });
+    const userInstructionPrompt = protocol.buildUserPrompt((0, external_common_js_namespaceObject.userPromptToString)(targetElementDescription));
+    const { messages, preparedImage } = await (0, external_utils_js_namespaceObject.prepareLocateModelInput)({
+        systemPrompt,
+        userPrompt: userInstructionPrompt,
+        locateImage: locateRequest.locateImage,
+        imagePreprocess: adapter.imagePreprocess,
+        targetDescription: targetElementDescription,
+        userMessageContentOrder: adapter.locate.userMessageContentOrder
+    });
+    try {
+        return await (0, semantic_retry_js_namespaceObject.callAiAndParseWithRetry)({
+            callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
+                    abortSignal: options.abortSignal,
+                    expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,
+                    semanticRetryAttempt: retryAttempt
+                }),
+            parseResponse: (response)=>{
+                const parsedLocateResult = protocol.parseRawResponse(response.content, resultCodec.promptSpec);
+                const rawResponse = response.content;
+                const locateError = parsedLocateResult.error;
+                if ('not-found' === parsedLocateResult.kind) return {
+                    rawResponse,
+                    rawChoiceMessage: response.rawChoiceMessage,
+                    usage: response.usage,
+                    reasoningContent: response.reasoning_content,
+                    errors: locateError ? [
+                        locateError
+                    ] : []
+                };
+                try {
+                    const locatedPixelBbox = resultCodec.toPixelBbox(parsedLocateResult.target, {
+                        preparedSize: preparedImage.preparedSize,
+                        contentSize: preparedImage.contentSize
+                    });
+                    return {
+                        locatedPixelBbox,
+                        rawResponse,
+                        rawChoiceMessage: response.rawChoiceMessage,
+                        usage: response.usage,
+                        reasoningContent: response.reasoning_content,
+                        errors: []
+                    };
+                } catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    const message = [
+                        locateError && `error in locate result: ${locateError}`,
+                        `coordinate parsing error: ${errorMessage}`,
+                        (0, external_utils_js_namespaceObject.formatLocateModelContext)(modelRuntime)
+                    ].filter(Boolean).join('\n');
+                    throw new Error(message, {
+                        cause: error
+                    });
+                }
+            },
+            toParseError: (error, response)=>{
+                const parseErrorMessage = error instanceof Error ? `Failed to parse locate result: ${error.message}` : 'unknown error in locate result';
+                return new external_service_caller_index_js_namespaceObject.AIResponseParseError(parseErrorMessage, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
+            },
+            parseRetryTimes: modelRuntime.config.retryCount,
+            parseRetryInterval: modelRuntime.config.retryInterval,
+            abortSignal: options.abortSignal,
+            onParseRetry: (error)=>{
+                debugGrounding('retrying locate after coordinate parsing failed: %s', error instanceof Error ? error.message : String(error));
+            }
+        });
+    } catch (callError) {
+        if (callError instanceof external_service_caller_index_js_namespaceObject.AIResponseParseError) return {
+            rawResponse: callError.rawResponse,
+            rawChoiceMessage: callError.rawChoiceMessage,
+            usage: callError.usage,
+            reasoningContent: callError.reasoningContent,
+            errors: [
+                callError.message
+            ]
+        };
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        return {
+            rawResponse: errorMessage,
+            errors: [
+                `AI call error: ${errorMessage}`
+            ]
+        };
+    }
+}
+exports.AiLocateElement = __webpack_exports__.AiLocateElement;
+exports.extraTextFromUserPrompt = __webpack_exports__.extraTextFromUserPrompt;
+exports.genericLocate = __webpack_exports__.genericLocate;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "AiLocateElement",
+    "extraTextFromUserPrompt",
+    "genericLocate"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=locate.js.map

package/dist/lib/ai-model/workflows/grounding/locate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { generateElementByRect } from '@aiscene/shared/extractor';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { pixelBboxToRect } from './locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugGrounding('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          userPromptToString(targetElementDescription),\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const locatedPixelBbox = resultCodec.toPixelBbox(\n            parsedLocateResult.target,\n            {\n              preparedSize: preparedImage.preparedSize,\n              contentSize: preparedImage.contentSize,\n            },\n          );\n          return {\n            locatedPixelBbox,\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","undefined","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ACeA,MAAMI,iBAAiBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAMO;QACN,aAAa;YACX,SAASA;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,OAAOC,AAAAA,IAAAA,+CAAAA,eAAAA,AAAAA,EACXC,AAAAA,IAAAA,gDAAAA,yCAAAA,AAAAA,EACEV,kBACAT,cAAc,YAAY,EAAE;QAGhCL,eAAe,WAAWsB;QAE1B,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACPH,MACAI,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBtB;gBAErB,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOO,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMN;YACN,aAAa;gBACX,SAASA;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAES,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGR,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMqB,eAAe3B,QAAQ,YAAY;IACzC,MAAM,EAAE4B,OAAO,EAAE,GAAGD;IACpBxB,IAAAA,sBAAAA,MAAAA,AAAAA,EACEyB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,AAAAA,IAAAA,yBAAAA,8BAAAA,AAAAA,EAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBtB;IAGrB,MAAM,EAAEiC,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEL;QACA,YAAYE;QACZ,aAAa3B,cAAc,WAAW;QACtC,iBAAiBsB,QAAQ,eAAe;QACxC,mBAAmB3B;QACnB,yBAAyB2B,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAa3B,QAAQ,WAAW;oBAChC,4BAA4B6B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMlB,cAAc8B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL/B;oBACA,kBAAkB8B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMjC,mBAAmBmB,YAAY,WAAW,CAC9Ca,mBAAmB,MAAM,EACzB;wBACE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBAEF,OAAO;wBACLxB;wBACAC;wBACA,kBAAkB8B,SAAS,gBAAgB;wBAC3C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMqB,eACJrB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;oBAClD,MAAMuB,UAAU;wBACdH,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEC,cAAc;wBAC3CG,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBrB;qBAC1B,CACE,MAAM,CAACsB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIvB,MAAMqB,SAAS;wBAAE,OAAOvB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMQ,oBACJ1B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI2B,iDAAAA,oBAAoBA,CAC7BD,mBACAR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAa3B,QAAQ,WAAW;YAChC,cAAc,CAACwB;gBACb3B,eACE,uDACA2B,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAO4B,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB1B,QAAQ0B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}