@aiscene/core 9.0.6 → 9.0.8

package/dist/lib/ai-model/workflows/grounding/planning-action-locate.js

@@ -0,0 +1,131 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    resolvePlanningTapLocator: ()=>resolvePlanningTapLocator
+});
+const logger_namespaceObject = require("@aiscene/shared/logger");
+const utils_namespaceObject = require("@aiscene/shared/utils");
+const external_zod_namespaceObject = require("zod");
+const external_common_js_namespaceObject = require("../../../common.js");
+const external_screenshot_item_js_namespaceObject = require("../../../screenshot-item.js");
+const index_js_namespaceObject = require("../../service-caller/index.js");
+const multimodal_prompt_js_namespaceObject = require("../../shared/multimodal-prompt.js");
+const conversation_history_js_namespaceObject = require("../planning/conversation-history.js");
+const custom_planning_js_namespaceObject = require("../planning/custom-planning.js");
+const debugGrounding = (0, logger_namespaceObject.getDebug)('ai:grounding');
+const planningActionLocatorActionSpace = [
+    {
+        name: 'Tap',
+        description: 'Tap the element',
+        paramSchema: external_zod_namespaceObject.z.object({
+            locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)()
+        }),
+        call: async ()=>void 0
+    }
+];
+async function buildPlanningTapLocatorPlanOptions(locateRequest) {
+    const { options, locateImage } = locateRequest;
+    const { context } = options;
+    return {
+        ...options,
+        context: {
+            ...context,
+            screenshot: external_screenshot_item_js_namespaceObject.ScreenshotItem.create(locateImage.imageBase64, context.screenshot.capturedAt),
+            shotSize: {
+                width: locateImage.width,
+                height: locateImage.height
+            }
+        },
+        actionSpace: planningActionLocatorActionSpace,
+        conversationHistory: new conversation_history_js_namespaceObject.ConversationHistory(),
+        includeLocateInPlanning: true,
+        effort: 'balance'
+    };
+}
+function resolvePlanningTapLocator(definition, planner) {
+    const locatorPlanner = {
+        ...planner,
+        messages: {
+            ...planner.messages,
+            buildSystemPrompt: definition.buildSystemPrompt,
+            buildUserInstruction: (instruction)=>`Tap: ${instruction}`
+        }
+    };
+    return async (locateRequest)=>{
+        const { targetElementDescription } = locateRequest;
+        (0, utils_namespaceObject.assert)(targetElementDescription, "cannot find the target element description");
+        let errors = [];
+        let reasoningContent = '';
+        let rawResponse = '';
+        let rawChoiceMessage;
+        let usage;
+        try {
+            const locatePlanOptions = await buildPlanningTapLocatorPlanOptions(locateRequest);
+            const planningResponse = await (0, custom_planning_js_namespaceObject.runCustomPlanning)(await (0, multimodal_prompt_js_namespaceObject.prepareUserPrompt)(targetElementDescription), locatePlanOptions, locatorPlanner);
+            rawResponse = planningResponse.rawResponse ?? '';
+            rawChoiceMessage = planningResponse.rawChoiceMessage;
+            usage = planningResponse.usage;
+            reasoningContent = planningResponse.log;
+            debugGrounding('planning-tap-locator rawResponse:', rawResponse);
+            const locatedPixelBbox = definition.getLocatedPixelBbox(planningResponse.actions ?? []);
+            if (!locatedPixelBbox) throw new Error('No locatedPixelBbox found in planner response');
+            return {
+                locatedPixelBbox,
+                rawResponse,
+                rawChoiceMessage,
+                usage,
+                reasoningContent
+            };
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            if (error instanceof index_js_namespaceObject.AIResponseParseError) {
+                rawResponse = error.rawResponse;
+                rawChoiceMessage = error.rawChoiceMessage;
+                usage = error.usage;
+            }
+            errors = [
+                errorMessage || 'Failed to parse planning tap locator response'
+            ];
+            debugGrounding('planning-tap-locator parse error:', errors[0]);
+        }
+        return {
+            rawResponse,
+            rawChoiceMessage,
+            usage,
+            reasoningContent,
+            errors
+        };
+    };
+}
+exports.resolvePlanningTapLocator = __webpack_exports__.resolvePlanningTapLocator;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "resolvePlanningTapLocator"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=planning-action-locate.js.map

package/dist/lib/ai-model/workflows/grounding/planning-action-locate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/planning-action-locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/planning-action-locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/device';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport { z } from 'zod';\nimport { getMidsceneLocationSchema } from '../../../common';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type { ResolvedCustomPlanningDefinition } from '../../model-adapter/custom-planning-types';\nimport type { PlanningTapLocatorDefinition } from '../../model-adapter/types';\nimport { AIResponseParseError } from '../../service-caller/index';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\nimport { ConversationHistory } from '../planning/conversation-history';\nimport { runCustomPlanning } from '../planning/custom-planning';\nimport type { PlanOptions } from '../planning/types';\nimport type { LocateFn, LocateModelResponse, LocateRequest } from './types';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nconst planningActionLocatorActionSpace: DeviceAction[] = [\n  {\n    name: 'Tap',\n    description: 'Tap the element',\n    paramSchema: z.object({\n      locate: getMidsceneLocationSchema(),\n    }),\n    call: async () => undefined,\n  },\n];\n\nasync function buildPlanningTapLocatorPlanOptions(\n  locateRequest: LocateRequest,\n): Promise<PlanOptions> {\n  const { options, locateImage } = locateRequest;\n  const { context } = options;\n\n  return {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        locateImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: {\n        width: locateImage.width,\n        height: locateImage.height,\n      },\n    },\n    actionSpace: planningActionLocatorActionSpace,\n    conversationHistory: new ConversationHistory(),\n    includeLocateInPlanning: true,\n    effort: 'balance',\n  };\n}\n\nexport function resolvePlanningTapLocator<TParsed>(\n  definition: PlanningTapLocatorDefinition,\n  planner: ResolvedCustomPlanningDefinition<TParsed>,\n): LocateFn {\n  const locatorPlanner: ResolvedCustomPlanningDefinition<TParsed> = {\n    ...planner,\n    messages: {\n      ...planner.messages,\n      buildSystemPrompt: definition.buildSystemPrompt,\n      buildUserInstruction: (instruction) => `Tap: ${instruction}`,\n    },\n  };\n\n  return async (locateRequest: LocateRequest): Promise<LocateModelResponse> => {\n    const { targetElementDescription } = locateRequest;\n    assert(\n      targetElementDescription,\n      'cannot find the target element description',\n    );\n\n    let errors: string[] = [];\n    let reasoningContent = '';\n    let rawResponse = '';\n    let rawChoiceMessage: unknown;\n    let usage: LocateModelResponse['usage'];\n\n    try {\n      const locatePlanOptions =\n        await buildPlanningTapLocatorPlanOptions(locateRequest);\n      const planningResponse = await runCustomPlanning(\n        await prepareUserPrompt(targetElementDescription),\n        locatePlanOptions,\n        locatorPlanner,\n      );\n\n      rawResponse = planningResponse.rawResponse ?? '';\n      rawChoiceMessage = planningResponse.rawChoiceMessage;\n      usage = planningResponse.usage;\n      reasoningContent = planningResponse.log;\n\n      debugGrounding('planning-tap-locator rawResponse:', rawResponse);\n\n      const locatedPixelBbox = definition.getLocatedPixelBbox(\n        planningResponse.actions ?? [],\n      );\n\n      if (!locatedPixelBbox) {\n        throw new Error('No locatedPixelBbox found in planner response');\n      }\n\n      return {\n        locatedPixelBbox,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoningContent,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      if (error instanceof AIResponseParseError) {\n        rawResponse = error.rawResponse;\n        rawChoiceMessage = error.rawChoiceMessage;\n        usage = error.usage;\n      }\n      errors = [\n        errorMessage || 'Failed to parse planning tap locator response',\n      ];\n      debugGrounding('planning-tap-locator parse error:', errors[0]);\n    }\n\n    return {\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoningContent,\n      errors,\n    };\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugGrounding","getDebug","planningActionLocatorActionSpace","z","getMidsceneLocationSchema","undefined","buildPlanningTapLocatorPlanOptions","locateRequest","options","locateImage","context","ScreenshotItem","ConversationHistory","resolvePlanningTapLocator","planner","locatorPlanner","instruction","targetElementDescription","assert","errors","reasoningContent","rawResponse","rawChoiceMessage","usage","locatePlanOptions","planningResponse","runCustomPlanning","prepareUserPrompt","locatedPixelBbox","Error","error","errorMessage","String","AIResponseParseError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;ACSA,MAAMI,iBAAiBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhC,MAAMC,mCAAmD;IACvD;QACE,MAAM;QACN,aAAa;QACb,aAAaC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;YACpB,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA;QACV;QACA,MAAM,UAAYC;IACpB;CACD;AAED,eAAeC,mCACbC,aAA4B;IAE5B,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGF;IACjC,MAAM,EAAEG,OAAO,EAAE,GAAGF;IAEpB,OAAO;QACL,GAAGA,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYC,4CAAAA,cAAAA,CAAAA,MAAqB,CAC/BF,YAAY,WAAW,EACvBC,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAU;gBACR,OAAOD,YAAY,KAAK;gBACxB,QAAQA,YAAY,MAAM;YAC5B;QACF;QACA,aAAaP;QACb,qBAAqB,IAAIU,wCAAAA,mBAAmBA;QAC5C,yBAAyB;QACzB,QAAQ;IACV;AACF;AAEO,SAASC,0BACdnB,UAAwC,EACxCoB,OAAkD;IAElD,MAAMC,iBAA4D;QAChE,GAAGD,OAAO;QACV,UAAU;YACR,GAAGA,QAAQ,QAAQ;YACnB,mBAAmBpB,WAAW,iBAAiB;YAC/C,sBAAsB,CAACsB,cAAgB,CAAC,KAAK,EAAEA,aAAa;QAC9D;IACF;IAEA,OAAO,OAAOT;QACZ,MAAM,EAAEU,wBAAwB,EAAE,GAAGV;QACrCW,IAAAA,sBAAAA,MAAAA,AAAAA,EACED,0BACA;QAGF,IAAIE,SAAmB,EAAE;QACzB,IAAIC,mBAAmB;QACvB,IAAIC,cAAc;QAClB,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMC,oBACJ,MAAMlB,mCAAmCC;YAC3C,MAAMkB,mBAAmB,MAAMC,AAAAA,IAAAA,mCAAAA,iBAAAA,AAAAA,EAC7B,MAAMC,AAAAA,IAAAA,qCAAAA,iBAAAA,AAAAA,EAAkBV,2BACxBO,mBACAT;YAGFM,cAAcI,iBAAiB,WAAW,IAAI;YAC9CH,mBAAmBG,iBAAiB,gBAAgB;YACpDF,QAAQE,iBAAiB,KAAK;YAC9BL,mBAAmBK,iBAAiB,GAAG;YAEvCzB,eAAe,qCAAqCqB;YAEpD,MAAMO,mBAAmBlC,WAAW,mBAAmB,CACrD+B,iBAAiB,OAAO,IAAI,EAAE;YAGhC,IAAI,CAACG,kBACH,MAAM,IAAIC,MAAM;YAGlB,OAAO;gBACLD;gBACAP;gBACAC;gBACAC;gBACAH;YACF;QACF,EAAE,OAAOU,OAAO;YACd,MAAMC,eACJD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;YAClD,IAAIA,iBAAiBG,yBAAAA,oBAAoBA,EAAE;gBACzCZ,cAAcS,MAAM,WAAW;gBAC/BR,mBAAmBQ,MAAM,gBAAgB;gBACzCP,QAAQO,MAAM,KAAK;YACrB;YACAX,SAAS;gBACPY,gBAAgB;aACjB;YACD/B,eAAe,qCAAqCmB,MAAM,CAAC,EAAE;QAC/D;QAEA,OAAO;YACLE;YACAC;YACAC;YACAH;YACAD;QACF;IACF;AACF"}

package/dist/lib/ai-model/workflows/grounding/search-area-mapping.js

@@ -0,0 +1,52 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    mapSearchAreaPixelBboxToOriginalPixelBbox: ()=>mapSearchAreaPixelBboxToOriginalPixelBbox
+});
+function mapSearchAreaPixelBboxToOriginalPixelBbox([left, top, right, bottom], mapping) {
+    const offset = mapping?.offset ?? {
+        x: 0,
+        y: 0
+    };
+    const scale = mapping?.scale ?? 1;
+    const mapX = (x)=>(1 !== scale ? Math.round(x / scale) : x) + offset.x;
+    const mapY = (y)=>(1 !== scale ? Math.round(y / scale) : y) + offset.y;
+    return [
+        mapX(left),
+        mapY(top),
+        mapX(right),
+        mapY(bottom)
+    ];
+}
+exports.mapSearchAreaPixelBboxToOriginalPixelBbox = __webpack_exports__.mapSearchAreaPixelBboxToOriginalPixelBbox;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "mapSearchAreaPixelBboxToOriginalPixelBbox"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=search-area-mapping.js.map

package/dist/lib/ai-model/workflows/grounding/search-area-mapping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/search-area-mapping.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/search-area-mapping.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PixelBbox } from '@/ai-model/shared/model-locate-result';\nimport type { SearchAreaImageMapping } from './types';\n\n/** Map a bbox in the prepared search-area image back to screenshot pixels. */\nexport function mapSearchAreaPixelBboxToOriginalPixelBbox(\n  [left, top, right, bottom]: PixelBbox,\n  mapping?: SearchAreaImageMapping,\n): PixelBbox {\n  const offset = mapping?.offset ?? { x: 0, y: 0 };\n  const scale = mapping?.scale ?? 1;\n  const mapX = (x: number) =>\n    (scale !== 1 ? Math.round(x / scale) : x) + offset.x;\n  const mapY = (y: number) =>\n    (scale !== 1 ? Math.round(y / scale) : y) + offset.y;\n\n  return [mapX(left), mapY(top), mapX(right), mapY(bottom)];\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","mapSearchAreaPixelBboxToOriginalPixelBbox","left","top","right","bottom","mapping","offset","scale","mapX","x","Math","mapY","y"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;ACFO,SAASI,0CACd,CAACC,MAAMC,KAAKC,OAAOC,OAAkB,EACrCC,OAAgC;IAEhC,MAAMC,SAASD,SAAS,UAAU;QAAE,GAAG;QAAG,GAAG;IAAE;IAC/C,MAAME,QAAQF,SAAS,SAAS;IAChC,MAAMG,OAAO,CAACC,IACXF,AAAAA,CAAAA,AAAU,MAAVA,QAAcG,KAAK,KAAK,CAACD,IAAIF,SAASE,CAAAA,IAAKH,OAAO,CAAC;IACtD,MAAMK,OAAO,CAACC,IACXL,AAAAA,CAAAA,AAAU,MAAVA,QAAcG,KAAK,KAAK,CAACE,IAAIL,SAASK,CAAAA,IAAKN,OAAO,CAAC;IAEtD,OAAO;QAACE,KAAKP;QAAOU,KAAKT;QAAMM,KAAKL;QAAQQ,KAAKP;KAAQ;AAC3D"}

package/dist/lib/ai-model/workflows/grounding/search-area.js

@@ -0,0 +1,92 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    expandSearchArea: ()=>expandSearchArea,
+    buildSearchAreaConfig: ()=>buildSearchAreaConfig
+});
+const img_namespaceObject = require("@aiscene/shared/img");
+function expandSearchArea(rect, screenSize) {
+    const minArea = 160000;
+    const expandSize = 100;
+    const expandedLeft = Math.max(rect.left - expandSize, 0);
+    const expandedTop = Math.max(rect.top - expandSize, 0);
+    const expandRect = {
+        left: expandedLeft,
+        top: expandedTop,
+        width: Math.min(rect.left - expandedLeft + rect.width + expandSize, screenSize.width - expandedLeft),
+        height: Math.min(rect.top - expandedTop + rect.height + expandSize, screenSize.height - expandedTop)
+    };
+    const currentArea = expandRect.width * expandRect.height;
+    if (currentArea >= minArea) return expandRect;
+    const centerX = expandRect.left + expandRect.width / 2;
+    const centerY = expandRect.top + expandRect.height / 2;
+    const scaleFactor = Math.sqrt(minArea / currentArea);
+    const newWidth = Math.round(expandRect.width * scaleFactor);
+    const newHeight = Math.round(expandRect.height * scaleFactor);
+    const newLeft = Math.round(centerX - newWidth / 2);
+    const newTop = Math.round(centerY - newHeight / 2);
+    const left = Math.max(newLeft, 0);
+    const top = Math.max(newTop, 0);
+    return {
+        left,
+        top,
+        width: Math.min(newWidth, screenSize.width - left),
+        height: Math.min(newHeight, screenSize.height - top)
+    };
+}
+async function buildSearchAreaConfig(options) {
+    const { context, baseRect } = options;
+    const scaleRatio = 2;
+    const sectionRect = expandSearchArea(baseRect, context.shotSize);
+    const croppedResult = await (0, img_namespaceObject.cropByRect)(context.screenshot.base64, sectionRect);
+    const scaledResult = await (0, img_namespaceObject.scaleImage)(croppedResult.imageBase64, scaleRatio);
+    return {
+        sourceRect: sectionRect,
+        image: {
+            imageBase64: scaledResult.imageBase64,
+            width: scaledResult.width,
+            height: scaledResult.height
+        },
+        mapping: {
+            offset: {
+                x: sectionRect.left,
+                y: sectionRect.top
+            },
+            scale: scaleRatio
+        }
+    };
+}
+exports.buildSearchAreaConfig = __webpack_exports__.buildSearchAreaConfig;
+exports.expandSearchArea = __webpack_exports__.expandSearchArea;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "buildSearchAreaConfig",
+    "expandSearchArea"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=search-area.js.map

package/dist/lib/ai-model/workflows/grounding/search-area.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/search-area.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/search-area.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { Rect, Size, UIContext } from '@/types';\nimport { cropByRect, scaleImage } from '@aiscene/shared/img';\nimport type { SearchAreaConfig } from './types';\n\n/**\n * Expand the search area to at least 400 x 400 pixels\n *\n * Step 1: Extend 100px on each side (top, right, bottom, left)\n * - If the element is near a boundary, expansion on that side will be limited\n * - No compensation is made for boundary limitations (this is intentional)\n *\n * Step 2: Ensure the area is at least 400x400 pixels\n * - Scale up proportionally from the center if needed\n * - Final result is clamped to screen boundaries\n */\nexport function expandSearchArea(rect: Rect, screenSize: Size): Rect {\n  const minArea = 400 * 400;\n  const expandSize = 100;\n\n  // Step 1: Extend each side by expandSize (100px), clamped to screen boundaries\n  // Note: If element is near boundary, actual expansion may be less than 100px on that side\n  const expandedLeft = Math.max(rect.left - expandSize, 0);\n  const expandedTop = Math.max(rect.top - expandSize, 0);\n\n  const expandRect = {\n    left: expandedLeft,\n    top: expandedTop,\n    width: Math.min(\n      rect.left - expandedLeft + rect.width + expandSize,\n      screenSize.width - expandedLeft,\n    ),\n    height: Math.min(\n      rect.top - expandedTop + rect.height + expandSize,\n      screenSize.height - expandedTop,\n    ),\n  };\n\n  // Step 2: Check if area is already >= 400x400\n  const currentArea = expandRect.width * expandRect.height;\n\n  if (currentArea >= minArea) {\n    return expandRect;\n  }\n\n  // Step 2: Scale up from center to reach minimum 400x400 area\n  const centerX = expandRect.left + expandRect.width / 2;\n  const centerY = expandRect.top + expandRect.height / 2;\n\n  // Calculate scale factor needed to reach minimum area\n  const scaleFactor = Math.sqrt(minArea / currentArea);\n  const newWidth = Math.round(expandRect.width * scaleFactor);\n  const newHeight = Math.round(expandRect.height * scaleFactor);\n\n  // Calculate new position based on center point\n  const newLeft = Math.round(centerX - newWidth / 2);\n  const newTop = Math.round(centerY - newHeight / 2);\n\n  // Clamp to screen boundaries\n  const left = Math.max(newLeft, 0);\n  const top = Math.max(newTop, 0);\n\n  return {\n    left,\n    top,\n    width: Math.min(newWidth, screenSize.width - left),\n    height: Math.min(newHeight, screenSize.height - top),\n  };\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","expandSearchArea","rect","screenSize","minArea","expandSize","expandedLeft","Math","expandedTop","expandRect","currentArea","centerX","centerY","scaleFactor","newWidth","newHeight","newLeft","newTop","left","top","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","croppedResult","cropByRect","scaledResult","scaleImage"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACSO,SAASI,iBAAiBC,IAAU,EAAEC,UAAgB;IAC3D,MAAMC,UAAU;IAChB,MAAMC,aAAa;IAInB,MAAMC,eAAeC,KAAK,GAAG,CAACL,KAAK,IAAI,GAAGG,YAAY;IACtD,MAAMG,cAAcD,KAAK,GAAG,CAACL,KAAK,GAAG,GAAGG,YAAY;IAEpD,MAAMI,aAAa;QACjB,MAAMH;QACN,KAAKE;QACL,OAAOD,KAAK,GAAG,CACbL,KAAK,IAAI,GAAGI,eAAeJ,KAAK,KAAK,GAAGG,YACxCF,WAAW,KAAK,GAAGG;QAErB,QAAQC,KAAK,GAAG,CACdL,KAAK,GAAG,GAAGM,cAAcN,KAAK,MAAM,GAAGG,YACvCF,WAAW,MAAM,GAAGK;IAExB;IAGA,MAAME,cAAcD,WAAW,KAAK,GAAGA,WAAW,MAAM;IAExD,IAAIC,eAAeN,SACjB,OAAOK;IAIT,MAAME,UAAUF,WAAW,IAAI,GAAGA,WAAW,KAAK,GAAG;IACrD,MAAMG,UAAUH,WAAW,GAAG,GAAGA,WAAW,MAAM,GAAG;IAGrD,MAAMI,cAAcN,KAAK,IAAI,CAACH,UAAUM;IACxC,MAAMI,WAAWP,KAAK,KAAK,CAACE,WAAW,KAAK,GAAGI;IAC/C,MAAME,YAAYR,KAAK,KAAK,CAACE,WAAW,MAAM,GAAGI;IAGjD,MAAMG,UAAUT,KAAK,KAAK,CAACI,UAAUG,WAAW;IAChD,MAAMG,SAASV,KAAK,KAAK,CAACK,UAAUG,YAAY;IAGhD,MAAMG,OAAOX,KAAK,GAAG,CAACS,SAAS;IAC/B,MAAMG,MAAMZ,KAAK,GAAG,CAACU,QAAQ;IAE7B,OAAO;QACLC;QACAC;QACA,OAAOZ,KAAK,GAAG,CAACO,UAAUX,WAAW,KAAK,GAAGe;QAC7C,QAAQX,KAAK,GAAG,CAACQ,WAAWZ,WAAW,MAAM,GAAGgB;IAClD;AACF;AAEO,eAAeC,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcxB,iBAAiBsB,UAAUD,QAAQ,QAAQ;IAE/D,MAAMI,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BL,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMG,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAAWH,cAAc,WAAW,EAAEF;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaG,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGH,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF"}

package/dist/lib/ai-model/workflows/grounding/section-locate.js

@@ -0,0 +1,180 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    AiLocateSection: ()=>AiLocateSection
+});
+const logger_namespaceObject = require("@aiscene/shared/logger");
+const utils_namespaceObject = require("@aiscene/shared/utils");
+const external_common_js_namespaceObject = require("../../../common.js");
+const index_js_namespaceObject = require("../../prompt/locate/index.js");
+const external_service_caller_index_js_namespaceObject = require("../../service-caller/index.js");
+const semantic_retry_js_namespaceObject = require("../../service-caller/semantic-retry.js");
+const external_locate_result_rect_js_namespaceObject = require("./locate-result-rect.js");
+const external_search_area_js_namespaceObject = require("./search-area.js");
+const external_utils_js_namespaceObject = require("./utils.js");
+const debugSection = (0, logger_namespaceObject.getDebug)('ai:grounding:section');
+async function AiLocateSection(options) {
+    const { context, sectionDescription } = options;
+    const modelRuntime = options.modelRuntime;
+    const { adapter } = modelRuntime;
+    (0, utils_namespaceObject.assert)('standard' === adapter.locate.kind, 'section locate requires a standard locate adapter');
+    const searchArea = adapter.locate.searchArea;
+    (0, utils_namespaceObject.assert)(searchArea, 'section locate requires a search area operation');
+    const { protocol: searchAreaProtocol, resultCodec } = searchArea;
+    const screenshotBase64 = context.screenshot.base64;
+    const systemPrompt = (0, index_js_namespaceObject.buildSearchAreaLocateSystemPrompt)({
+        systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,
+        responseInstructions: searchAreaProtocol.buildResponseInstructions(resultCodec.promptSpec)
+    });
+    const userInstructionPrompt = searchAreaProtocol.buildUserPrompt((0, external_common_js_namespaceObject.userPromptToString)(sectionDescription));
+    const { messages, preparedImage } = await (0, external_utils_js_namespaceObject.prepareLocateModelInput)({
+        systemPrompt,
+        userPrompt: userInstructionPrompt,
+        locateImage: {
+            imageBase64: screenshotBase64,
+            width: context.shotSize.width,
+            height: context.shotSize.height
+        },
+        imagePreprocess: adapter.imagePreprocess,
+        targetDescription: sectionDescription,
+        userMessageContentOrder: adapter.locate.userMessageContentOrder
+    });
+    let parsedResult;
+    try {
+        parsedResult = await (0, semantic_retry_js_namespaceObject.callAiAndParseWithRetry)({
+            callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
+                    abortSignal: options.abortSignal,
+                    expectedJsonObjectResponse: searchAreaProtocol.expectedJsonObjectResponse,
+                    semanticRetryAttempt: retryAttempt
+                }),
+            parseResponse: (response)=>{
+                const parsedLocateResult = searchAreaProtocol.parseRawResponse(response.content, resultCodec.promptSpec);
+                const sectionError = parsedLocateResult.error;
+                if ('not-found' === parsedLocateResult.kind) return {
+                    result: response,
+                    sectionError
+                };
+                try {
+                    const locateResultContext = {
+                        preparedSize: preparedImage.preparedSize,
+                        contentSize: preparedImage.contentSize
+                    };
+                    const target = resultCodec.toPixelBbox(parsedLocateResult.target, locateResultContext);
+                    const references = parsedLocateResult.references?.map((reference)=>resultCodec.toPixelBbox(reference, locateResultContext));
+                    const mergedRect = (0, external_locate_result_rect_js_namespaceObject.mergePixelBboxesToRect)([
+                        target,
+                        ...references ?? []
+                    ]);
+                    debugSection('mergedRect %j', mergedRect);
+                    return {
+                        result: response,
+                        sectionError,
+                        mergedRect
+                    };
+                } catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    const message = [
+                        sectionError && `error in section locate result: ${sectionError}`,
+                        `coordinate parsing error: ${errorMessage}`,
+                        (0, external_utils_js_namespaceObject.formatLocateModelContext)(modelRuntime)
+                    ].filter(Boolean).join('\n');
+                    throw new Error(message, {
+                        cause: error
+                    });
+                }
+            },
+            toParseError: (error, response)=>{
+                const parseErrorMessage = error instanceof Error ? `Failed to parse section locate result: ${error.message}` : 'unknown error in section locate';
+                return new external_service_caller_index_js_namespaceObject.AIResponseParseError(parseErrorMessage, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
+            },
+            parseRetryTimes: modelRuntime.config.retryCount,
+            parseRetryInterval: modelRuntime.config.retryInterval,
+            abortSignal: options.abortSignal,
+            onParseRetry: (error)=>{
+                debugSection('retrying section locate after coordinate parsing failed: %s', error instanceof Error ? error.message : String(error));
+            }
+        });
+    } catch (callError) {
+        if (callError instanceof external_service_caller_index_js_namespaceObject.AIResponseParseError) return {
+            searchAreaConfig: void 0,
+            error: callError.message,
+            rawResponse: callError.rawResponse,
+            rawChoiceMessage: callError.rawChoiceMessage,
+            usage: callError.usage
+        };
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        return {
+            searchAreaConfig: void 0,
+            error: `AI call error: ${errorMessage}`,
+            rawResponse: errorMessage
+        };
+    }
+    const { result, sectionError, mergedRect } = parsedResult;
+    if (!mergedRect) return {
+        searchAreaConfig: void 0,
+        error: sectionError,
+        rawResponse: result.content,
+        rawChoiceMessage: result.rawChoiceMessage,
+        usage: result.usage
+    };
+    try {
+        const expandedRect = (0, external_search_area_js_namespaceObject.expandSearchArea)(mergedRect, context.shotSize);
+        const originalWidth = expandedRect.width;
+        const originalHeight = expandedRect.height;
+        debugSection('expanded sectionRect %j', expandedRect);
+        const searchAreaConfig = await (0, external_search_area_js_namespaceObject.buildSearchAreaConfig)({
+            context,
+            baseRect: mergedRect
+        });
+        debugSection('scaled section image from %dx%d to %dx%d (scale=%d)', originalWidth, originalHeight, searchAreaConfig.image.width, searchAreaConfig.image.height, searchAreaConfig.mapping.scale);
+        return {
+            searchAreaConfig,
+            error: sectionError,
+            rawResponse: result.content,
+            rawChoiceMessage: result.rawChoiceMessage,
+            usage: result.usage
+        };
+    } catch (error) {
+        const parseErrorMessage = error instanceof Error ? `Failed to parse section locate result: ${error.message}` : 'unknown error in section locate';
+        const errorMessage = sectionError ? `${sectionError} (${parseErrorMessage})` : parseErrorMessage;
+        return {
+            searchAreaConfig: void 0,
+            error: errorMessage,
+            rawResponse: result.content,
+            rawChoiceMessage: result.rawChoiceMessage,
+            usage: result.usage
+        };
+    }
+}
+exports.AiLocateSection = __webpack_exports__.AiLocateSection;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "AiLocateSection"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=section-locate.js.map

package/dist/lib/ai-model/workflows/grounding/section-locate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/section-locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mergePixelBboxesToRect } from './locate-result-rect';\nimport { buildSearchAreaConfig, expandSearchArea } from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelBbox(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelBbox(reference, locateResultContext),\n          );\n          const mergedRect = mergePixelBboxesToRect([\n            target,\n            ...(references ?? []),\n          ]);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergePixelBboxesToRect","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;ACWA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,IAAAA,sBAAAA,MAAAA,AAAAA,EACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,AAAAA,IAAAA,yBAAAA,iCAAAA,AAAAA,EAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,WAAW,CACpCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,WAAW,CAACqB,WAAWH;oBAErC,MAAMI,aAAaC,AAAAA,IAAAA,+CAAAA,sBAAAA,AAAAA,EAAuB;wBACxCJ;2BACIC,cAAc,EAAE;qBACrB;oBACD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,iDAAAA,oBAAoBA,CAC7BD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,AAAAA,IAAAA,wCAAAA,gBAAAA,AAAAA,EAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,AAAAA,IAAAA,wCAAAA,qBAAAA,AAAAA,EAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}

package/dist/lib/ai-model/workflows/grounding/types.js

@@ -0,0 +1,20 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+for(var __rspack_i in __webpack_exports__)exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=types.js.map

package/dist/lib/ai-model/workflows/grounding/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/types.js","sources":["webpack/runtime/make_namespace_object"],"sourcesContent":["// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","Symbol","Object"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOC,UAA0BA,OAAO,WAAW,EACrDC,OAAO,cAAc,CAAC,UAASD,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEC,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/ai-model/workflows/grounding/utils.js

@@ -0,0 +1,87 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    prepareLocateModelInput: ()=>prepareLocateModelInput,
+    formatLocateModelContext: ()=>formatLocateModelContext
+});
+const external_common_js_namespaceObject = require("../../../common.js");
+const image_preprocess_js_namespaceObject = require("../../model-adapter/image-preprocess.js");
+const multimodal_prompt_js_namespaceObject = require("../../shared/multimodal-prompt.js");
+async function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, userMessageContentOrder }) {
+    const preparedImage = await (0, image_preprocess_js_namespaceObject.prepareModelImage)({
+        imageBase64: locateImage.imageBase64,
+        width: locateImage.width,
+        height: locateImage.height,
+        policy: imagePreprocess
+    });
+    const referenceImageMessages = await (0, multimodal_prompt_js_namespaceObject.multimodalPromptToChatMessages)((0, external_common_js_namespaceObject.userPromptToMultimodalPrompt)(targetDescription));
+    const imageContent = {
+        type: 'image_url',
+        image_url: {
+            url: preparedImage.imageBase64,
+            detail: 'high'
+        }
+    };
+    const promptContent = {
+        type: 'text',
+        text: userPrompt
+    };
+    return {
+        preparedImage,
+        messages: [
+            {
+                role: 'system',
+                content: systemPrompt
+            },
+            {
+                role: 'user',
+                content: 'prompt-first' === userMessageContentOrder ? [
+                    promptContent,
+                    imageContent
+                ] : [
+                    imageContent,
+                    promptContent
+                ]
+            },
+            ...referenceImageMessages
+        ]
+    };
+}
+function formatLocateModelContext(modelRuntime) {
+    const { modelFamily, modelName } = modelRuntime.config;
+    return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;
+}
+exports.formatLocateModelContext = __webpack_exports__.formatLocateModelContext;
+exports.prepareLocateModelInput = __webpack_exports__.prepareLocateModelInput;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "formatLocateModelContext",
+    "prepareLocateModelInput"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=utils.js.map