@midscene/core 1.12.5-beta-20260908034641.0 → 1.12.5

package/dist/es/ai-model/shared/model-locate-result/factory.mjs

@@ -1,37 +1,37 @@
-import { finalizePixelBbox } from "./bbox.mjs";
 import { parseNumericLocateResult } from "./parse.mjs";
-import { mapLocateResultToPixelBboxByCoordinates } from "./pixel-bbox-mapper.mjs";
+import { clampPixelBboxToSize, clampPixelPointToSize, mapLocateResultToPixelBbox, mapLocateResultToPixelPoint, pixelBboxToRect } from "./pixel-mapper.mjs";
 import { createLocateResultPromptSpec } from "./prompt-spec.mjs";
+import { isBboxLocateResultValue } from "./types.mjs";
+import { assertLocateResultCoordinateRangeAndOrder, assertLocateResultStructure } from "./validation.mjs";
 function resolveLocateResultCoordinates(coordinates) {
-    const order = coordinates.order ?? 'xy';
     if (void 0 !== coordinates.normalizedBy && coordinates.normalizedBy <= 0) throw new Error(`locate result coordinates normalizedBy must be positive: ${coordinates.normalizedBy}`);
     return {
         shape: coordinates.shape,
-        order,
-        normalizedBy: coordinates.normalizedBy
+        order: coordinates.order ?? 'xy',
+        normalizedBy: coordinates.normalizedBy,
+        rounding: coordinates.rounding ?? 'round'
     };
 }
-function assertValidParsedLocateResult(result) {
-    if (!result || 'object' != typeof result) throw new Error(`invalid parsed locate result: expected object, got ${JSON.stringify(result)}`);
-    const coordinatesMeta = result.coordinatesMeta;
-    const expectedLength = coordinatesMeta?.shape === 'bbox' ? 4 : coordinatesMeta?.shape === 'point' ? 2 : 0;
-    if (!expectedLength) throw new Error(`invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(coordinatesMeta?.shape)}`);
-    const coordinates = result.coordinates;
-    if (!Array.isArray(coordinates) || coordinates.length !== expectedLength || !coordinates.every((value)=>'number' == typeof value && Number.isFinite(value))) throw new Error(`invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(coordinates)}`);
-}
 function createLocateResultCodec(config) {
     const resolvedCoordinates = resolveLocateResultCoordinates(config.coordinates);
     const parseRawLocateValue = config.parseRawLocateValue ?? ((input)=>parseNumericLocateResult(resolvedCoordinates, input));
-    const mapLocateResultToPixelBbox = config.mapLocateResultToPixelBbox ?? ((result, ctx)=>mapLocateResultToPixelBboxByCoordinates(result, ctx));
-    const toPixelBbox = (rawResult, context)=>{
-        const parsedResult = parseRawLocateValue(rawResult);
-        assertValidParsedLocateResult(parsedResult);
-        const pixelBbox = mapLocateResultToPixelBbox(parsedResult, context);
-        return finalizePixelBbox(pixelBbox, rawResult, context);
-    };
     return {
         promptSpec: createLocateResultPromptSpec(resolvedCoordinates),
-        toPixelBbox
+        toPixelResult: (rawResult, context)=>{
+            const result = parseRawLocateValue(rawResult);
+            const { preparedSize, contentSize = preparedSize } = context;
+            assertLocateResultStructure(result);
+            assertLocateResultCoordinateRangeAndOrder(result, preparedSize.width - 1, preparedSize.height - 1);
+            const center = clampPixelPointToSize(mapLocateResultToPixelPoint(result, preparedSize), contentSize);
+            const rect = isBboxLocateResultValue(result) ? (()=>{
+                const pixelBbox = clampPixelBboxToSize(mapLocateResultToPixelBbox(result, preparedSize), contentSize);
+                return pixelBboxToRect(pixelBbox);
+            })() : void 0;
+            return {
+                center,
+                rect
+            };
+        }
     };
 }
 export { createLocateResultCodec, resolveLocateResultCoordinates };

package/dist/es/ai-model/shared/model-locate-result/factory.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/factory.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/factory.ts"],"sourcesContent":["import { finalizePixelBbox } from './bbox';\nimport { parseNumericLocateResult } from './parse';\nimport { mapLocateResultToPixelBboxByCoordinates } from './pixel-bbox-mapper';\nimport { createLocateResultPromptSpec } from './prompt-spec';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n  LocateResultCoordinates,\n  LocateResultFormatDefinition,\n  LocateResultValue,\n  ResolvedLocateResultCoordinates,\n} from './types';\n\nexport function resolveLocateResultCoordinates(\n  coordinates: LocateResultCoordinates,\n): ResolvedLocateResultCoordinates {\n  const order = coordinates.order ?? 'xy';\n  if (coordinates.normalizedBy !== undefined && coordinates.normalizedBy <= 0) {\n    throw new Error(\n      `locate result coordinates normalizedBy must be positive: ${coordinates.normalizedBy}`,\n    );\n  }\n  return {\n    shape: coordinates.shape,\n    order,\n    normalizedBy: coordinates.normalizedBy,\n  };\n}\n\nfunction assertValidParsedLocateResult(result: LocateResultValue): void {\n  if (!result || typeof result !== 'object') {\n    throw new Error(\n      `invalid parsed locate result: expected object, got ${JSON.stringify(\n        result,\n      )}`,\n    );\n  }\n\n  const coordinatesMeta = result.coordinatesMeta;\n  const expectedLength =\n    coordinatesMeta?.shape === 'bbox'\n      ? 4\n      : coordinatesMeta?.shape === 'point'\n        ? 2\n        : 0;\n  if (!expectedLength) {\n    throw new Error(\n      `invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(\n        coordinatesMeta?.shape,\n      )}`,\n    );\n  }\n\n  const coordinates = result.coordinates;\n  if (\n    !Array.isArray(coordinates) ||\n    coordinates.length !== expectedLength ||\n    !coordinates.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(\n      `invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(\n        coordinates,\n      )}`,\n    );\n  }\n}\n\nexport function createLocateResultCodec(\n  config: LocateResultFormatDefinition,\n): LocateResultCodec {\n  const resolvedCoordinates = resolveLocateResultCoordinates(\n    config.coordinates,\n  );\n  const parseRawLocateValue =\n    config.parseRawLocateValue ??\n    ((input) => parseNumericLocateResult(resolvedCoordinates, input));\n  const mapLocateResultToPixelBbox =\n    config.mapLocateResultToPixelBbox ??\n    ((result, ctx) => mapLocateResultToPixelBboxByCoordinates(result, ctx));\n\n  const toPixelBbox = (rawResult: unknown, context: LocateResultContext) => {\n    const parsedResult = parseRawLocateValue(rawResult);\n    assertValidParsedLocateResult(parsedResult);\n    const pixelBbox = mapLocateResultToPixelBbox(parsedResult, context);\n    return finalizePixelBbox(pixelBbox, rawResult, context);\n  };\n\n  return {\n    promptSpec: createLocateResultPromptSpec(resolvedCoordinates),\n    toPixelBbox,\n  };\n}\n"],"names":["resolveLocateResultCoordinates","coordinates","order","undefined","Error","assertValidParsedLocateResult","result","JSON","coordinatesMeta","expectedLength","Array","value","Number","createLocateResultCodec","config","resolvedCoordinates","parseRawLocateValue","input","parseNumericLocateResult","mapLocateResultToPixelBbox","ctx","mapLocateResultToPixelBboxByCoordinates","toPixelBbox","rawResult","context","parsedResult","pixelBbox","finalizePixelBbox","createLocateResultPromptSpec"],"mappings":";;;;AAaO,SAASA,+BACdC,WAAoC;IAEpC,MAAMC,QAAQD,YAAY,KAAK,IAAI;IACnC,IAAIA,AAA6BE,WAA7BF,YAAY,YAAY,IAAkBA,YAAY,YAAY,IAAI,GACxE,MAAM,IAAIG,MACR,CAAC,yDAAyD,EAAEH,YAAY,YAAY,EAAE;IAG1F,OAAO;QACL,OAAOA,YAAY,KAAK;QACxBC;QACA,cAAcD,YAAY,YAAY;IACxC;AACF;AAEA,SAASI,8BAA8BC,MAAyB;IAC9D,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,QACpB,MAAM,IAAIF,MACR,CAAC,mDAAmD,EAAEG,KAAK,SAAS,CAClED,SACC;IAIP,MAAME,kBAAkBF,OAAO,eAAe;IAC9C,MAAMG,iBACJD,iBAAiB,UAAU,SACvB,IACAA,iBAAiB,UAAU,UACzB,IACA;IACR,IAAI,CAACC,gBACH,MAAM,IAAIL,MACR,CAAC,gEAAgE,EAAEG,KAAK,SAAS,CAC/EC,iBAAiB,QAChB;IAIP,MAAMP,cAAcK,OAAO,WAAW;IACtC,IACE,CAACI,MAAM,OAAO,CAACT,gBACfA,YAAY,MAAM,KAAKQ,kBACvB,CAACR,YAAY,KAAK,CAChB,CAACU,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SAG1D,MAAM,IAAIP,MACR,CAAC,8BAA8B,EAAEI,gBAAgB,KAAK,CAAC,qBAAqB,EAAEC,eAAe,qBAAqB,EAAEF,KAAK,SAAS,CAChIN,cACC;AAGT;AAEO,SAASY,wBACdC,MAAoC;IAEpC,MAAMC,sBAAsBf,+BAC1Bc,OAAO,WAAW;IAEpB,MAAME,sBACJF,OAAO,mBAAmB,IACxB,EAAAG,QAAUC,yBAAyBH,qBAAqBE,MAAK;IACjE,MAAME,6BACJL,OAAO,0BAA0B,IAC/B,EAAAR,QAAQc,MAAQC,wCAAwCf,QAAQc,IAAG;IAEvE,MAAME,cAAc,CAACC,WAAoBC;QACvC,MAAMC,eAAeT,oBAAoBO;QACzClB,8BAA8BoB;QAC9B,MAAMC,YAAYP,2BAA2BM,cAAcD;QAC3D,OAAOG,kBAAkBD,WAAWH,WAAWC;IACjD;IAEA,OAAO;QACL,YAAYI,6BAA6Bb;QACzCO;IACF;AACF"}
+{"version":3,"file":"ai-model/shared/model-locate-result/factory.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/factory.ts"],"sourcesContent":["import { parseNumericLocateResult } from './parse';\nimport {\n  clampPixelBboxToSize,\n  clampPixelPointToSize,\n  mapLocateResultToPixelBbox,\n  mapLocateResultToPixelPoint,\n  pixelBboxToRect,\n} from './pixel-mapper';\nimport { createLocateResultPromptSpec } from './prompt-spec';\nimport { isBboxLocateResultValue } from './types';\nimport type {\n  LocateResultCodec,\n  LocateResultCoordinates,\n  LocateResultFormatDefinition,\n  ResolvedLocateResultCoordinates,\n} from './types';\nimport {\n  assertLocateResultCoordinateRangeAndOrder,\n  assertLocateResultStructure,\n} from './validation';\n\nexport function resolveLocateResultCoordinates(\n  coordinates: LocateResultCoordinates,\n): ResolvedLocateResultCoordinates {\n  if (coordinates.normalizedBy !== undefined && coordinates.normalizedBy <= 0) {\n    throw new Error(\n      `locate result coordinates normalizedBy must be positive: ${coordinates.normalizedBy}`,\n    );\n  }\n  return {\n    shape: coordinates.shape,\n    order: coordinates.order ?? 'xy',\n    normalizedBy: coordinates.normalizedBy,\n    rounding: coordinates.rounding ?? 'round',\n  };\n}\n\nexport function createLocateResultCodec(\n  config: LocateResultFormatDefinition,\n): LocateResultCodec {\n  const resolvedCoordinates = resolveLocateResultCoordinates(\n    config.coordinates,\n  );\n  const parseRawLocateValue =\n    config.parseRawLocateValue ??\n    ((input) => parseNumericLocateResult(resolvedCoordinates, input));\n  return {\n    promptSpec: createLocateResultPromptSpec(resolvedCoordinates),\n    toPixelResult: (rawResult, context) => {\n      const result = parseRawLocateValue(rawResult);\n      const { preparedSize, contentSize = preparedSize } = context;\n      assertLocateResultStructure(result);\n      assertLocateResultCoordinateRangeAndOrder(\n        result,\n        preparedSize.width - 1,\n        preparedSize.height - 1,\n      );\n      // Clamp the mapped center independently from the optional bbox so padding\n      // removal cannot change the target by recomputing a clipped box's center.\n      const center = clampPixelPointToSize(\n        mapLocateResultToPixelPoint(result, preparedSize),\n        contentSize,\n      );\n      // Preserve an original bbox as region metadata for DeepLocate search areas\n      // and aiLocate's return value. Point-only results leave rect undefined;\n      // click coordinates always use the independently calculated center.\n      const rect = isBboxLocateResultValue(result)\n        ? (() => {\n            const pixelBbox = clampPixelBboxToSize(\n              mapLocateResultToPixelBbox(result, preparedSize),\n              contentSize,\n            );\n            return pixelBboxToRect(pixelBbox);\n          })()\n        : undefined;\n      return { center, rect };\n    },\n  };\n}\n"],"names":["resolveLocateResultCoordinates","coordinates","undefined","Error","createLocateResultCodec","config","resolvedCoordinates","parseRawLocateValue","input","parseNumericLocateResult","createLocateResultPromptSpec","rawResult","context","result","preparedSize","contentSize","assertLocateResultStructure","assertLocateResultCoordinateRangeAndOrder","center","clampPixelPointToSize","mapLocateResultToPixelPoint","rect","isBboxLocateResultValue","pixelBbox","clampPixelBboxToSize","mapLocateResultToPixelBbox","pixelBboxToRect"],"mappings":";;;;;AAqBO,SAASA,+BACdC,WAAoC;IAEpC,IAAIA,AAA6BC,WAA7BD,YAAY,YAAY,IAAkBA,YAAY,YAAY,IAAI,GACxE,MAAM,IAAIE,MACR,CAAC,yDAAyD,EAAEF,YAAY,YAAY,EAAE;IAG1F,OAAO;QACL,OAAOA,YAAY,KAAK;QACxB,OAAOA,YAAY,KAAK,IAAI;QAC5B,cAAcA,YAAY,YAAY;QACtC,UAAUA,YAAY,QAAQ,IAAI;IACpC;AACF;AAEO,SAASG,wBACdC,MAAoC;IAEpC,MAAMC,sBAAsBN,+BAC1BK,OAAO,WAAW;IAEpB,MAAME,sBACJF,OAAO,mBAAmB,IACxB,EAAAG,QAAUC,yBAAyBH,qBAAqBE,MAAK;IACjE,OAAO;QACL,YAAYE,6BAA6BJ;QACzC,eAAe,CAACK,WAAWC;YACzB,MAAMC,SAASN,oBAAoBI;YACnC,MAAM,EAAEG,YAAY,EAAEC,cAAcD,YAAY,EAAE,GAAGF;YACrDI,4BAA4BH;YAC5BI,0CACEJ,QACAC,aAAa,KAAK,GAAG,GACrBA,aAAa,MAAM,GAAG;YAIxB,MAAMI,SAASC,sBACbC,4BAA4BP,QAAQC,eACpCC;YAKF,MAAMM,OAAOC,wBAAwBT,UAChC,AAAC;gBACA,MAAMU,YAAYC,qBAChBC,2BAA2BZ,QAAQC,eACnCC;gBAEF,OAAOW,gBAAgBH;YACzB,OACArB;YACJ,OAAO;gBAAEgB;gBAAQG;YAAK;QACxB;IACF;AACF"}

package/dist/es/ai-model/shared/model-locate-result/pixel-mapper.mjs

@@ -0,0 +1,87 @@
+import { isBboxLocateResultValue } from "./types.mjs";
+function clamp(value, min, max) {
+    return Math.min(Math.max(value, min), max);
+}
+function maxPixelCoordinate(size) {
+    return Math.max(size - 1, 0);
+}
+function normalizedCoordinateToPixel(value, normalizedBy, size, rounding) {
+    const pixel = value * size / normalizedBy;
+    switch(rounding){
+        case 'round':
+            return Math.round(pixel);
+        case 'trunc':
+            return Math.trunc(pixel);
+        case 'none':
+            return pixel;
+        default:
+            throw new Error(`Invalid coordinate rounding: ${rounding}`);
+    }
+}
+function clampPixelPointToSize([x, y], { width, height }) {
+    return [
+        clamp(x, 0, maxPixelCoordinate(width)),
+        clamp(y, 0, maxPixelCoordinate(height))
+    ];
+}
+function clampPixelBboxToSize(pixelBbox, size) {
+    const [left, top, right, bottom] = pixelBbox;
+    return [
+        ...clampPixelPointToSize([
+            left,
+            top
+        ], size),
+        ...clampPixelPointToSize([
+            right,
+            bottom
+        ], size)
+    ];
+}
+function pixelBboxToRect([left, top, right, bottom]) {
+    return {
+        left,
+        top,
+        width: right - left + 1,
+        height: bottom - top + 1
+    };
+}
+function mapLocateResultToPixelBbox(result, { width, height }) {
+    const { normalizedBy, order, rounding } = result.coordinatesMeta;
+    const [first, second, third, fourth] = result.coordinates;
+    const xyBbox = 'yx' === order ? [
+        second,
+        first,
+        fourth,
+        third
+    ] : result.coordinates;
+    return void 0 === normalizedBy ? xyBbox : [
+        normalizedCoordinateToPixel(xyBbox[0], normalizedBy, width, rounding),
+        normalizedCoordinateToPixel(xyBbox[1], normalizedBy, height, rounding),
+        normalizedCoordinateToPixel(xyBbox[2], normalizedBy, width, rounding),
+        normalizedCoordinateToPixel(xyBbox[3], normalizedBy, height, rounding)
+    ];
+}
+function mapLocateResultToPixelPoint(result, { width, height }) {
+    const { order, normalizedBy, rounding } = result.coordinatesMeta;
+    const [first, second] = isBboxLocateResultValue(result) ? [
+        (result.coordinates[0] + result.coordinates[2]) / 2,
+        (result.coordinates[1] + result.coordinates[3]) / 2
+    ] : result.coordinates;
+    const [x, y] = 'yx' === order ? [
+        second,
+        first
+    ] : [
+        first,
+        second
+    ];
+    return void 0 === normalizedBy ? [
+        x,
+        y
+    ] : [
+        normalizedCoordinateToPixel(x, normalizedBy, width, rounding),
+        normalizedCoordinateToPixel(y, normalizedBy, height, rounding)
+    ];
+}
+export { clampPixelBboxToSize, clampPixelPointToSize, mapLocateResultToPixelBbox, mapLocateResultToPixelPoint, maxPixelCoordinate, normalizedCoordinateToPixel, pixelBboxToRect };
+
+//# sourceMappingURL=pixel-mapper.mjs.map

package/dist/es/ai-model/shared/model-locate-result/pixel-mapper.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/pixel-mapper.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/pixel-mapper.ts"],"sourcesContent":["import type { Rect } from '@/types';\nimport { isBboxLocateResultValue } from './types';\nimport type {\n  BboxLocateResultValue,\n  CoordinateRounding,\n  LocateResultPoint,\n  LocateResultValue,\n  PixelBbox,\n} from './types';\n\nfunction clamp(value: number, min: number, max: number) {\n  return Math.min(Math.max(value, min), max);\n}\n\nexport function maxPixelCoordinate(size: number) {\n  return Math.max(size - 1, 0);\n}\n\nexport function normalizedCoordinateToPixel(\n  value: number,\n  normalizedBy: number,\n  size: number,\n  rounding: CoordinateRounding,\n) {\n  const pixel = (value * size) / normalizedBy;\n  switch (rounding) {\n    case 'round':\n      return Math.round(pixel);\n    case 'trunc':\n      return Math.trunc(pixel);\n    case 'none':\n      return pixel;\n    default:\n      throw new Error(`Invalid coordinate rounding: ${rounding}`);\n  }\n}\n\n/** Restrict an already mapped point to the effective content area. */\nexport function clampPixelPointToSize(\n  [x, y]: LocateResultPoint,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  return [\n    clamp(x, 0, maxPixelCoordinate(width)),\n    clamp(y, 0, maxPixelCoordinate(height)),\n  ];\n}\n\n/** Restrict an already mapped bbox to the effective content area. */\nexport function clampPixelBboxToSize(\n  pixelBbox: PixelBbox,\n  size: { width: number; height: number },\n): PixelBbox {\n  const [left, top, right, bottom] = pixelBbox;\n  return [\n    ...clampPixelPointToSize([left, top], size),\n    ...clampPixelPointToSize([right, bottom], size),\n  ];\n}\n\nexport function pixelBboxToRect([left, top, right, bottom]: PixelBbox): Rect {\n  return {\n    left,\n    top,\n    width: right - left + 1,\n    height: bottom - top + 1,\n  };\n}\n\nexport function mapLocateResultToPixelBbox(\n  result: BboxLocateResultValue,\n  { width, height }: { width: number; height: number },\n): PixelBbox {\n  const { normalizedBy, order, rounding } = result.coordinatesMeta;\n  const [first, second, third, fourth] = result.coordinates;\n  const xyBbox: PixelBbox =\n    order === 'yx' ? [second, first, fourth, third] : result.coordinates;\n  return normalizedBy === undefined\n    ? xyBbox\n    : [\n        normalizedCoordinateToPixel(xyBbox[0], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[1], normalizedBy, height, rounding),\n        normalizedCoordinateToPixel(xyBbox[2], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[3], normalizedBy, height, rounding),\n      ];\n}\n\n/** Resolve the target before any rounding or clipping can shift its center. */\nexport function mapLocateResultToPixelPoint(\n  result: LocateResultValue,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  const { order, normalizedBy, rounding } = result.coordinatesMeta;\n  const [first, second] = isBboxLocateResultValue(result)\n    ? [\n        (result.coordinates[0] + result.coordinates[2]) / 2,\n        (result.coordinates[1] + result.coordinates[3]) / 2,\n      ]\n    : result.coordinates;\n  const [x, y] = order === 'yx' ? [second, first] : [first, second];\n  return normalizedBy === undefined\n    ? [x, y]\n    : [\n        normalizedCoordinateToPixel(x, normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(y, normalizedBy, height, rounding),\n      ];\n}\n"],"names":["clamp","value","min","max","Math","maxPixelCoordinate","size","normalizedCoordinateToPixel","normalizedBy","rounding","pixel","Error","clampPixelPointToSize","x","y","width","height","clampPixelBboxToSize","pixelBbox","left","top","right","bottom","pixelBboxToRect","mapLocateResultToPixelBbox","result","order","first","second","third","fourth","xyBbox","undefined","mapLocateResultToPixelPoint","isBboxLocateResultValue"],"mappings":";AAUA,SAASA,MAAMC,KAAa,EAAEC,GAAW,EAAEC,GAAW;IACpD,OAAOC,KAAK,GAAG,CAACA,KAAK,GAAG,CAACH,OAAOC,MAAMC;AACxC;AAEO,SAASE,mBAAmBC,IAAY;IAC7C,OAAOF,KAAK,GAAG,CAACE,OAAO,GAAG;AAC5B;AAEO,SAASC,4BACdN,KAAa,EACbO,YAAoB,EACpBF,IAAY,EACZG,QAA4B;IAE5B,MAAMC,QAAST,QAAQK,OAAQE;IAC/B,OAAQC;QACN,KAAK;YACH,OAAOL,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAON,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAOA;QACT;YACE,MAAM,IAAIC,MAAM,CAAC,6BAA6B,EAAEF,UAAU;IAC9D;AACF;AAGO,SAASG,sBACd,CAACC,GAAGC,EAAqB,EACzB,EAAEC,KAAK,EAAEC,MAAM,EAAqC;IAEpD,OAAO;QACLhB,MAAMa,GAAG,GAAGR,mBAAmBU;QAC/Bf,MAAMc,GAAG,GAAGT,mBAAmBW;KAChC;AACH;AAGO,SAASC,qBACdC,SAAoB,EACpBZ,IAAuC;IAEvC,MAAM,CAACa,MAAMC,KAAKC,OAAOC,OAAO,GAAGJ;IACnC,OAAO;WACFN,sBAAsB;YAACO;YAAMC;SAAI,EAAEd;WACnCM,sBAAsB;YAACS;YAAOC;SAAO,EAAEhB;KAC3C;AACH;AAEO,SAASiB,gBAAgB,CAACJ,MAAMC,KAAKC,OAAOC,OAAkB;IACnE,OAAO;QACLH;QACAC;QACA,OAAOC,QAAQF,OAAO;QACtB,QAAQG,SAASF,MAAM;IACzB;AACF;AAEO,SAASI,2BACdC,MAA6B,EAC7B,EAAEV,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAER,YAAY,EAAEkB,KAAK,EAAEjB,QAAQ,EAAE,GAAGgB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,QAAQC,OAAOC,OAAO,GAAGL,OAAO,WAAW;IACzD,MAAMM,SACJL,AAAU,SAAVA,QAAiB;QAACE;QAAQD;QAAOG;QAAQD;KAAM,GAAGJ,OAAO,WAAW;IACtE,OAAOjB,AAAiBwB,WAAjBxB,eACHuB,SACA;QACExB,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcO,OAAON;QAC5DF,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcQ,QAAQP;QAC7DF,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcO,OAAON;QAC5DF,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcQ,QAAQP;KAC9D;AACP;AAGO,SAASwB,4BACdR,MAAyB,EACzB,EAAEV,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAEU,KAAK,EAAElB,YAAY,EAAEC,QAAQ,EAAE,GAAGgB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,OAAO,GAAGM,wBAAwBT,UAC5C;QACGA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;QACjDA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;KACnD,GACDA,OAAO,WAAW;IACtB,MAAM,CAACZ,GAAGC,EAAE,GAAGY,AAAU,SAAVA,QAAiB;QAACE;QAAQD;KAAM,GAAG;QAACA;QAAOC;KAAO;IACjE,OAAOpB,AAAiBwB,WAAjBxB,eACH;QAACK;QAAGC;KAAE,GACN;QACEP,4BAA4BM,GAAGL,cAAcO,OAAON;QACpDF,4BAA4BO,GAAGN,cAAcQ,QAAQP;KACtD;AACP"}

package/dist/es/ai-model/shared/model-locate-result/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/types.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/types.ts"],"sourcesContent":["import type { Bbox } from '../../../types';\n\nexport type { Bbox };\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\nexport type NonEmptyArray<T> = [T, ...T[]];\nexport type RawLocateValue = unknown;\n\nexport type LocateResultPoint = [number, number];\nexport type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'point';\n};\nexport type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'bbox';\n};\n\nexport type LocateResultValue =\n  | {\n      coordinates: LocateResultPoint;\n      coordinatesMeta: PointLocateResultCoordinates;\n    }\n  | {\n      coordinates: LocateResultBbox;\n      coordinatesMeta: BboxLocateResultCoordinates;\n    };\n\nexport type PointLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'point' } }\n>;\nexport type BboxLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'bbox' } }\n>;\n\nexport function isBboxLocateResultValue(\n  result: LocateResultValue,\n): result is BboxLocateResultValue {\n  return result.coordinatesMeta.shape === 'bbox';\n}\n\nexport function isPointLocateResultValue(\n  result: LocateResultValue,\n): result is PointLocateResultValue {\n  return result.coordinatesMeta.shape === 'point';\n}\n\nexport type LocateResultShape = 'bbox' | 'point';\n\nexport interface LocateResultContext {\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  contentSize?: {\n    width: number;\n    height: number;\n  };\n}\n\nexport interface LocateResultPromptSpec {\n  resultKey: string;\n  resultValueSchema: string;\n  resultValueDescription: string;\n  resultNoun: string;\n  resultNounPlural: string;\n  exampleValues: NonEmptyArray<unknown>;\n}\n\nexport interface LocateResultCodec {\n  promptSpec: LocateResultPromptSpec;\n  toPixelBbox(input: RawLocateValue, ctx: LocateResultContext): PixelBbox;\n}\n\nexport interface LocateResultCoordinates {\n  shape: LocateResultShape;\n  order?: 'xy' | 'yx';\n  normalizedBy?: number;\n}\n\nexport type ResolvedLocateResultCoordinates =\n  | {\n      shape: 'point';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n    }\n  | {\n      shape: 'bbox';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n    };\n\nexport type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;\nexport type LocateResultPixelBboxMapper = (\n  result: LocateResultValue,\n  ctx: LocateResultContext,\n) => PixelBbox;\n\n/**\n * Declarative config for the standard locate workflow.\n *\n * The standard workflow has three steps:\n * 1. `coordinates` is expanded into prompt wording, a default\n *    raw result parser, and a default pixel bbox mapper.\n * 2. `parseRawLocateValue` converts that raw result value into Midscene's\n *    internal `LocateResultValue` shape:\n *    `{ coordinates, coordinatesMeta }`. Omit it when the model returns a\n *    plain numeric bbox/point matching `coordinates`; provide it when the\n *    model needs repair, fallback handling, or per-result coordinate metadata.\n * 3. `mapLocateResultToPixelBbox` converts the parsed result into a pixel bbox\n *    `[left, top, right, bottom]`. Omit it when `coordinates` is enough to describe\n *    the coordinate system and order; provide it only for model-specific\n *    conversion rules.\n *\n * This format only describes one raw coordinate value. The operation protocol\n * owns the response envelope and identifies target/reference values before the\n * runtime codec converts each value independently.\n *\n * Example 1: a GLM-like model that directly matches the standard coordinates.\n *\n * ```ts\n * resultFormat: {\n *   coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n * }\n * ```\n *\n * Example 2: Qwen 2.5 returns pixel coordinates, but may return a point-like\n * value that needs custom parsing/fallback. The default pixel bbox mapper is\n * bypassed only if custom fallback sizing is required.\n *\n * ```ts\n * resultFormat: {\n *   coordinates: { shape: 'bbox', order: 'xy' },\n *   parseRawLocateValue: parseQwen25RawLocateValue,\n *   mapLocateResultToPixelBbox: normalizeQwen25ResultToPixelBbox,\n * }\n * ```\n *\n * Example 3: a model with a custom raw value shape can keep the standard\n * workflow while replacing parsing and mapping.\n *\n * ```ts\n * resultFormat: {\n *   coordinates: { shape: 'bbox', order: 'xy' },\n *   parseRawLocateValue: (raw) => ({\n *     coordinates: [\n *       Number((raw as any).left),\n *       Number((raw as any).top),\n *       Number((raw as any).right),\n *       Number((raw as any).bottom),\n *     ],\n *     coordinatesMeta: { shape: 'bbox', order: 'xy' },\n *   }),\n *   mapLocateResultToPixelBbox: (result) => result.coordinates,\n * }\n * ```\n */\nexport type LocateResultFormatDefinition = {\n  /**\n   * Common locate result coordinates shorthand. This is the preferred config surface\n   * for normal models because it keeps result type, coordinate system, and\n   * coordinate order in one orthogonal field.\n   */\n  coordinates: LocateResultCoordinates;\n  /**\n   * Parses one raw coordinate value into a `LocateResultValue`. This function\n   * should handle response repair, bbox-vs-point fallback, and the coordinate\n   * metadata that describes the parsed coordinates.\n   */\n  parseRawLocateValue?: RawLocateValueParser;\n  /**\n   * Maps the parsed result into a pixel bbox. Most models should omit this\n   * and let `coordinates` drive the default conversion. Provide it only when point\n   * fallback size, clipping, or coordinate semantics are model-specific.\n   */\n  mapLocateResultToPixelBbox?: LocateResultPixelBboxMapper;\n};\n"],"names":["isBboxLocateResultValue","result","isPointLocateResultValue"],"mappings":"AAmCO,SAASA,wBACdC,MAAyB;IAEzB,OAAOA,AAAiC,WAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC;AAEO,SAASC,yBACdD,MAAyB;IAEzB,OAAOA,AAAiC,YAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC"}
+{"version":3,"file":"ai-model/shared/model-locate-result/types.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/types.ts"],"sourcesContent":["import type { Rect } from '@midscene/shared/types';\nimport type { Bbox } from '../../../types';\n\nexport type { Bbox };\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\nexport type NonEmptyArray<T> = [T, ...T[]];\nexport type RawLocateValue = unknown;\n\nexport type LocateResultPoint = [number, number];\nexport type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'point';\n};\nexport type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'bbox';\n};\n\nexport type LocateResultValue =\n  | {\n      coordinates: LocateResultPoint;\n      coordinatesMeta: PointLocateResultCoordinates;\n    }\n  | {\n      coordinates: LocateResultBbox;\n      coordinatesMeta: BboxLocateResultCoordinates;\n    };\n\nexport type PointLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'point' } }\n>;\nexport type BboxLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'bbox' } }\n>;\n\nexport function isBboxLocateResultValue(\n  result: LocateResultValue,\n): result is BboxLocateResultValue {\n  return result.coordinatesMeta.shape === 'bbox';\n}\n\nexport function isPointLocateResultValue(\n  result: LocateResultValue,\n): result is PointLocateResultValue {\n  return result.coordinatesMeta.shape === 'point';\n}\n\nexport type LocateResultShape = 'bbox' | 'point';\n\nexport interface LocateResultContext {\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  contentSize?: {\n    width: number;\n    height: number;\n  };\n}\n\nexport interface LocateResultPromptSpec {\n  resultKey: string;\n  resultValueSchema: string;\n  resultValueDescription: string;\n  resultNoun: string;\n  resultNounPlural: string;\n  exampleValues: NonEmptyArray<unknown>;\n}\n\nexport interface PixelLocateResult {\n  /** Raw bbox midpoint (or raw point), mapped to the locate image using the configured rounding. */\n  center: LocateResultPoint;\n  /**\n   * Original bbox mapped to locate-image pixels and clipped to content bounds.\n   * Undefined for point results; never derive center or click coordinates from it.\n   */\n  rect?: Rect;\n}\n\nexport interface LocateResultCodec {\n  promptSpec: LocateResultPromptSpec;\n  /** Parse once and independently map the center and optional original bbox. */\n  toPixelResult(\n    input: RawLocateValue,\n    ctx: LocateResultContext,\n  ): PixelLocateResult;\n}\n\nexport type CoordinateRounding = 'round' | 'trunc' | 'none';\n\nexport interface LocateResultCoordinates {\n  shape: LocateResultShape;\n  /** Axis order in the raw coordinates; defaults to xy. */\n  order?: 'xy' | 'yx';\n  /** Normalization range, e.g. 1 or 1000. Omit for model-image pixel coordinates. */\n  normalizedBy?: number;\n  /** Rounding after normalized-to-pixel mapping; defaults to round. Pixel inputs retain their precision. */\n  rounding?: CoordinateRounding;\n}\n\n/** Coordinate metadata with defaults resolved and an explicit rounding policy. */\nexport type ResolvedLocateResultCoordinates =\n  | {\n      shape: 'point';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    }\n  | {\n      shape: 'bbox';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    };\n\nexport type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;\n\n/**\n * Format of one model coordinate value, shared by element and search-area locate.\n * The operation protocol extracts target/reference values from the response;\n * the codec parses each value once and maps it to `{ center, rect }` using the\n * parsed result's coordinatesMeta. Rect is undefined for actual point results.\n * Raw-value parsing and normalized-coordinate rounding are configurable;\n * coordinate mapping is shared.\n */\nexport type LocateResultFormatDefinition = {\n  /**\n   * Expected shape, axis order, and normalization used by the prompt and default\n   * parser. A custom parser may return different metadata for an actual response.\n   */\n  coordinates: LocateResultCoordinates;\n  /**\n   * Optional model-specific parser for nonstandard formats or point/bbox fallback.\n   * Return `{ coordinates, coordinatesMeta }` describing the actual shape, axis\n   * order, and normalization. Preserve coordinate precision; do not round or map\n   * to pixels here. Include an explicit rounding policy in `coordinatesMeta`.\n   * Omit the parser to parse numeric values according to `coordinates`.\n   */\n  parseRawLocateValue?: RawLocateValueParser;\n};\n"],"names":["isBboxLocateResultValue","result","isPointLocateResultValue"],"mappings":"AAoCO,SAASA,wBACdC,MAAyB;IAEzB,OAAOA,AAAiC,WAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC;AAEO,SAASC,yBACdD,MAAyB;IAEzB,OAAOA,AAAiC,YAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC"}

package/dist/es/ai-model/shared/model-locate-result/validation.mjs

@@ -0,0 +1,55 @@
+import { isBboxLocateResultValue } from "./types.mjs";
+function assertLocateResultStructure(result) {
+    if (!result || 'object' != typeof result) throw new Error(`invalid parsed locate result: expected object, got ${JSON.stringify(result)}`);
+    const coordinatesMeta = result.coordinatesMeta;
+    const expectedLength = coordinatesMeta?.shape === 'bbox' ? 4 : coordinatesMeta?.shape === 'point' ? 2 : 0;
+    if (!expectedLength) throw new Error(`invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(coordinatesMeta?.shape)}`);
+    const coordinates = result.coordinates;
+    if (!Array.isArray(coordinates) || coordinates.length !== expectedLength || !coordinates.every((value)=>'number' == typeof value && Number.isFinite(value))) throw new Error(`invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(coordinates)}`);
+    const { order, normalizedBy, rounding } = coordinatesMeta;
+    if (![
+        'round',
+        'trunc',
+        'none'
+    ].includes(rounding)) throw new Error(`invalid locate coordinate rounding: ${rounding}`);
+    if ('xy' !== order && 'yx' !== order) throw new Error(`invalid locate coordinate order: ${JSON.stringify(order)}`);
+    if (void 0 !== normalizedBy && (!Number.isFinite(normalizedBy) || normalizedBy <= 0)) throw new Error(`locate result normalizedBy must be positive and finite: ${normalizedBy}`);
+}
+function resolveCoordinateLimits(result, maxX, maxY) {
+    const resolvedCoordinates = result.coordinatesMeta;
+    const normalizedBy = resolvedCoordinates.normalizedBy;
+    if (void 0 !== normalizedBy) return result.coordinates.map(()=>normalizedBy);
+    if ('bbox' === resolvedCoordinates.shape) return 'yx' === resolvedCoordinates.order ? [
+        maxY,
+        maxX,
+        maxY,
+        maxX
+    ] : [
+        maxX,
+        maxY,
+        maxX,
+        maxY
+    ];
+    return 'yx' === resolvedCoordinates.order ? [
+        maxY,
+        maxX
+    ] : [
+        maxX,
+        maxY
+    ];
+}
+function assertLocateResultCoordinateRangeAndOrder(result, maxX, maxY) {
+    const coordinatesMeta = result.coordinatesMeta;
+    const { normalizedBy } = coordinatesMeta;
+    const limits = resolveCoordinateLimits(result, maxX, maxY);
+    const outOfRange = result.coordinates.some((value, index)=>value < 0 || value > limits[index]);
+    if (outOfRange) {
+        const source = void 0 !== normalizedBy ? `normalized range [0, ${normalizedBy}]` : `image size [0, ${maxX}]x[0, ${maxY}]`;
+        const normalizedInfo = void 0 !== normalizedBy ? ` normalizedBy=${normalizedBy}` : '';
+        throw new Error(`locate result coordinates ${JSON.stringify(result.coordinates)} exceed ${source}. shape=${coordinatesMeta.shape} order=${coordinatesMeta.order}${normalizedInfo} limits=${JSON.stringify(limits)}`);
+    }
+    if (isBboxLocateResultValue(result) && (result.coordinates[2] < result.coordinates[0] || result.coordinates[3] < result.coordinates[1])) throw new Error(`invalid locate bbox coordinate order: ${JSON.stringify(result.coordinates)}`);
+}
+export { assertLocateResultCoordinateRangeAndOrder, assertLocateResultStructure };
+
+//# sourceMappingURL=validation.mjs.map

package/dist/es/ai-model/shared/model-locate-result/validation.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/validation.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/validation.ts"],"sourcesContent":["import { isBboxLocateResultValue } from './types';\nimport type { LocateResultValue } from './types';\n\nexport function assertLocateResultStructure(result: LocateResultValue): void {\n  if (!result || typeof result !== 'object') {\n    throw new Error(\n      `invalid parsed locate result: expected object, got ${JSON.stringify(\n        result,\n      )}`,\n    );\n  }\n\n  const coordinatesMeta = result.coordinatesMeta;\n  const expectedLength =\n    coordinatesMeta?.shape === 'bbox'\n      ? 4\n      : coordinatesMeta?.shape === 'point'\n        ? 2\n        : 0;\n  if (!expectedLength) {\n    throw new Error(\n      `invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(\n        coordinatesMeta?.shape,\n      )}`,\n    );\n  }\n\n  const coordinates = result.coordinates;\n  if (\n    !Array.isArray(coordinates) ||\n    coordinates.length !== expectedLength ||\n    !coordinates.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(\n      `invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(\n        coordinates,\n      )}`,\n    );\n  }\n\n  const { order, normalizedBy, rounding } = coordinatesMeta;\n  if (!['round', 'trunc', 'none'].includes(rounding)) {\n    throw new Error(`invalid locate coordinate rounding: ${rounding}`);\n  }\n  if (order !== 'xy' && order !== 'yx') {\n    throw new Error(\n      `invalid locate coordinate order: ${JSON.stringify(order)}`,\n    );\n  }\n  if (\n    normalizedBy !== undefined &&\n    (!Number.isFinite(normalizedBy) || normalizedBy <= 0)\n  ) {\n    throw new Error(\n      `locate result normalizedBy must be positive and finite: ${normalizedBy}`,\n    );\n  }\n}\n\nfunction resolveCoordinateLimits(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n): number[] {\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  if (normalizedBy !== undefined) {\n    return result.coordinates.map(() => normalizedBy);\n  }\n\n  if (resolvedCoordinates.shape === 'bbox') {\n    return resolvedCoordinates.order === 'yx'\n      ? [maxY, maxX, maxY, maxX]\n      : [maxX, maxY, maxX, maxY];\n  }\n\n  return resolvedCoordinates.order === 'yx' ? [maxY, maxX] : [maxX, maxY];\n}\n\nexport function assertLocateResultCoordinateRangeAndOrder(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n) {\n  const coordinatesMeta = result.coordinatesMeta;\n  const { normalizedBy } = coordinatesMeta;\n  const limits = resolveCoordinateLimits(result, maxX, maxY);\n  const outOfRange = result.coordinates.some(\n    (value, index) => value < 0 || value > limits[index],\n  );\n\n  if (outOfRange) {\n    const source =\n      normalizedBy !== undefined\n        ? `normalized range [0, ${normalizedBy}]`\n        : `image size [0, ${maxX}]x[0, ${maxY}]`;\n    const normalizedInfo =\n      normalizedBy !== undefined ? ` normalizedBy=${normalizedBy}` : '';\n    throw new Error(\n      `locate result coordinates ${JSON.stringify(\n        result.coordinates,\n      )} exceed ${source}. shape=${\n        coordinatesMeta.shape\n      } order=${coordinatesMeta.order}${normalizedInfo} limits=${JSON.stringify(\n        limits,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    (result.coordinates[2] < result.coordinates[0] ||\n      result.coordinates[3] < result.coordinates[1])\n  ) {\n    throw new Error(\n      `invalid locate bbox coordinate order: ${JSON.stringify(result.coordinates)}`,\n    );\n  }\n}\n"],"names":["assertLocateResultStructure","result","Error","JSON","coordinatesMeta","expectedLength","coordinates","Array","value","Number","order","normalizedBy","rounding","undefined","resolveCoordinateLimits","maxX","maxY","resolvedCoordinates","assertLocateResultCoordinateRangeAndOrder","limits","outOfRange","index","source","normalizedInfo","isBboxLocateResultValue"],"mappings":";AAGO,SAASA,4BAA4BC,MAAyB;IACnE,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,QACpB,MAAM,IAAIC,MACR,CAAC,mDAAmD,EAAEC,KAAK,SAAS,CAClEF,SACC;IAIP,MAAMG,kBAAkBH,OAAO,eAAe;IAC9C,MAAMI,iBACJD,iBAAiB,UAAU,SACvB,IACAA,iBAAiB,UAAU,UACzB,IACA;IACR,IAAI,CAACC,gBACH,MAAM,IAAIH,MACR,CAAC,gEAAgE,EAAEC,KAAK,SAAS,CAC/EC,iBAAiB,QAChB;IAIP,MAAME,cAAcL,OAAO,WAAW;IACtC,IACE,CAACM,MAAM,OAAO,CAACD,gBACfA,YAAY,MAAM,KAAKD,kBACvB,CAACC,YAAY,KAAK,CAChB,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SAG1D,MAAM,IAAIN,MACR,CAAC,8BAA8B,EAAEE,gBAAgB,KAAK,CAAC,qBAAqB,EAAEC,eAAe,qBAAqB,EAAEF,KAAK,SAAS,CAChIG,cACC;IAIP,MAAM,EAAEI,KAAK,EAAEC,YAAY,EAAEC,QAAQ,EAAE,GAAGR;IAC1C,IAAI,CAAC;QAAC;QAAS;QAAS;KAAO,CAAC,QAAQ,CAACQ,WACvC,MAAM,IAAIV,MAAM,CAAC,oCAAoC,EAAEU,UAAU;IAEnE,IAAIF,AAAU,SAAVA,SAAkBA,AAAU,SAAVA,OACpB,MAAM,IAAIR,MACR,CAAC,iCAAiC,EAAEC,KAAK,SAAS,CAACO,QAAQ;IAG/D,IACEC,AAAiBE,WAAjBF,gBACC,EAACF,OAAO,QAAQ,CAACE,iBAAiBA,gBAAgB,IAEnD,MAAM,IAAIT,MACR,CAAC,wDAAwD,EAAES,cAAc;AAG/E;AAEA,SAASG,wBACPb,MAAyB,EACzBc,IAAY,EACZC,IAAY;IAEZ,MAAMC,sBAAsBhB,OAAO,eAAe;IAClD,MAAMU,eAAeM,oBAAoB,YAAY;IACrD,IAAIN,AAAiBE,WAAjBF,cACF,OAAOV,OAAO,WAAW,CAAC,GAAG,CAAC,IAAMU;IAGtC,IAAIM,AAA8B,WAA9BA,oBAAoB,KAAK,EAC3B,OAAOA,AAA8B,SAA9BA,oBAAoB,KAAK,GAC5B;QAACD;QAAMD;QAAMC;QAAMD;KAAK,GACxB;QAACA;QAAMC;QAAMD;QAAMC;KAAK;IAG9B,OAAOC,AAA8B,SAA9BA,oBAAoB,KAAK,GAAY;QAACD;QAAMD;KAAK,GAAG;QAACA;QAAMC;KAAK;AACzE;AAEO,SAASE,0CACdjB,MAAyB,EACzBc,IAAY,EACZC,IAAY;IAEZ,MAAMZ,kBAAkBH,OAAO,eAAe;IAC9C,MAAM,EAAEU,YAAY,EAAE,GAAGP;IACzB,MAAMe,SAASL,wBAAwBb,QAAQc,MAAMC;IACrD,MAAMI,aAAanB,OAAO,WAAW,CAAC,IAAI,CACxC,CAACO,OAAOa,QAAUb,QAAQ,KAAKA,QAAQW,MAAM,CAACE,MAAM;IAGtD,IAAID,YAAY;QACd,MAAME,SACJX,AAAiBE,WAAjBF,eACI,CAAC,qBAAqB,EAAEA,aAAa,CAAC,CAAC,GACvC,CAAC,eAAe,EAAEI,KAAK,MAAM,EAAEC,KAAK,CAAC,CAAC;QAC5C,MAAMO,iBACJZ,AAAiBE,WAAjBF,eAA6B,CAAC,cAAc,EAAEA,cAAc,GAAG;QACjE,MAAM,IAAIT,MACR,CAAC,0BAA0B,EAAEC,KAAK,SAAS,CACzCF,OAAO,WAAW,EAClB,QAAQ,EAAEqB,OAAO,QAAQ,EACzBlB,gBAAgB,KAAK,CACtB,OAAO,EAAEA,gBAAgB,KAAK,GAAGmB,eAAe,QAAQ,EAAEpB,KAAK,SAAS,CACvEgB,SACC;IAEP;IAEA,IACEK,wBAAwBvB,WACvBA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAE,IAC5CA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,GAE9C,MAAM,IAAIC,MACR,CAAC,sCAAsC,EAAEC,KAAK,SAAS,CAACF,OAAO,WAAW,GAAG;AAGnF"}

package/dist/es/ai-model/workflows/grounding/locate.mjs

@@ -1,12 +1,10 @@
-import { generateElementByRect } from "@midscene/shared/extractor";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { userPromptToString } from "../../../common.mjs";
 import { buildElementLocateSystemPrompt } from "../../prompt/locate/index.mjs";
 import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
-import { pixelBboxToRect } from "./locate-result-rect.mjs";
-import { mapSearchAreaPixelBboxToOriginalPixelBbox } from "./search-area-mapping.mjs";
+import { mapSearchAreaResultToOriginalResult } from "./search-area-mapping.mjs";
 import { formatLocateModelContext, prepareLocateModelInput } from "./utils.mjs";
 const debugGrounding = getDebug('ai:grounding');
 async function AiLocateElement(options) {
@@ -26,15 +24,14 @@ async function AiLocateElement(options) {
     const locateAdapter = options.modelRuntime.adapter.locate;
     const locateFn = 'custom' === locateAdapter.kind ? locateAdapter.locateFn : genericLocate;
     const locateResponse = await locateFn(locateRequest);
-    const { locatedPixelBbox, rawResponse, rawChoiceMessage, usage, reasoningContent, errors = [] } = locateResponse;
+    const { locatedPixelResult, rawResponse, rawChoiceMessage, usage, reasoningContent, errors = [] } = locateResponse;
     const baseLocateResult = {
         rawResponse,
         rawChoiceMessage,
         usage,
         reasoning_content: reasoningContent
     };
-    if (!locatedPixelBbox) return {
-        rect: void 0,
+    if (!locatedPixelResult) return {
         parseResult: {
             element: void 0,
             errors
@@ -42,12 +39,14 @@ async function AiLocateElement(options) {
         ...baseLocateResult
     };
     try {
-        const rect = pixelBboxToRect(mapSearchAreaPixelBboxToOriginalPixelBbox(locatedPixelBbox, locateOptions.searchConfig?.mapping));
-        debugGrounding('resRect', rect);
+        const mappedResult = mapSearchAreaResultToOriginalResult(locatedPixelResult, locateOptions.searchConfig?.mapping);
+        debugGrounding('center', mappedResult.center);
         return {
-            rect,
             parseResult: {
-                element: generateElementByRect(rect, userPromptToString(targetElementDescription)),
+                element: {
+                    ...mappedResult,
+                    description: userPromptToString(targetElementDescription)
+                },
                 errors: []
             },
             ...baseLocateResult
@@ -55,7 +54,6 @@ async function AiLocateElement(options) {
     } catch (error) {
         const msg = error instanceof Error ? `Failed to parse locate result: ${error.message}` : 'unknown error in locate';
         return {
-            rect: void 0,
             parseResult: {
                 element: void 0,
                 errors: errors.length > 0 ? [
@@ -109,12 +107,12 @@ async function genericLocate(locateRequest) {
                     ] : []
                 };
                 try {
-                    const locatedPixelBbox = resultCodec.toPixelBbox(parsedLocateResult.target, {
+                    const result = resultCodec.toPixelResult(parsedLocateResult.target, {
                         preparedSize: preparedImage.preparedSize,
                         contentSize: preparedImage.contentSize
                     });
                     return {
-                        locatedPixelBbox,
+                        locatedPixelResult: result,
                         rawResponse,
                         rawChoiceMessage: response.rawChoiceMessage,
                         usage: response.usage,

package/dist/es/ai-model/workflows/grounding/locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { pixelBboxToRect } from './locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugGrounding('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          userPromptToString(targetElementDescription),\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const locatedPixelBbox = resultCodec.toPixelBbox(\n            parsedLocateResult.target,\n            {\n              preparedSize: preparedImage.preparedSize,\n              contentSize: preparedImage.contentSize,\n            },\n          );\n          return {\n            locatedPixelBbox,\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","undefined","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;;;AAqBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAMO;QACN,aAAa;YACX,SAASA;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,OAAOC,gBACXC,0CACEV,kBACAT,cAAc,YAAY,EAAE;QAGhCL,eAAe,WAAWsB;QAE1B,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,sBACPH,MACAI,mBAAmBtB;gBAErB,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOO,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMN;YACN,aAAa;gBACX,SAASA;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAES,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGR,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMqB,eAAe3B,QAAQ,YAAY;IACzC,MAAM,EAAE4B,OAAO,EAAE,GAAGD;IACpBxB,OACEyB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBtB;IAGrB,MAAM,EAAEiC,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAa3B,cAAc,WAAW;QACtC,iBAAiBsB,QAAQ,eAAe;QACxC,mBAAmB3B;QACnB,yBAAyB2B,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAa3B,QAAQ,WAAW;oBAChC,4BAA4B6B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMlB,cAAc8B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL/B;oBACA,kBAAkB8B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMjC,mBAAmBmB,YAAY,WAAW,CAC9Ca,mBAAmB,MAAM,EACzB;wBACE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBAEF,OAAO;wBACLxB;wBACAC;wBACA,kBAAkB8B,SAAS,gBAAgB;wBAC3C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMqB,eACJrB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;oBAClD,MAAMuB,UAAU;wBACdH,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEC,cAAc;wBAC3CG,yBAAyBrB;qBAC1B,CACE,MAAM,CAACsB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIvB,MAAMqB,SAAS;wBAAE,OAAOvB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMQ,oBACJ1B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI2B,qBACTD,mBACAR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAa3B,QAAQ,WAAW;YAChC,cAAc,CAACwB;gBACb3B,eACE,uDACA2B,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAO4B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB1B,QAAQ0B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;AAmBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,oCACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,mBAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,OACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,kBAAkB4B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,kBAAkB4B,SAAS,gBAAgB;wBAC3C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,yBAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,qBACTD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs

@@ -62,10 +62,10 @@ function resolvePlanningTapLocator(definition, planner) {
             usage = planningResponse.usage;
             reasoningContent = planningResponse.log;
             debugGrounding('planning-tap-locator rawResponse:', rawResponse);
-            const locatedPixelBbox = definition.getLocatedPixelBbox(planningResponse.actions ?? []);
-            if (!locatedPixelBbox) throw new Error('No locatedPixelBbox found in planner response');
+            const locatedPixelResult = definition.getLocatedPixelResult(planningResponse.actions ?? []);
+            if (!locatedPixelResult) throw new Error('No locatedPixelResult found in planner response');
             return {
-                locatedPixelBbox,
+                locatedPixelResult,
                 rawResponse,
                 rawChoiceMessage,
                 usage,

package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/planning-action-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/planning-action-locate.ts"],"sourcesContent":["import type { DeviceAction } from '@/device';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { z } from 'zod';\nimport { getMidsceneLocationSchema } from '../../../common';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type { ResolvedCustomPlanningDefinition } from '../../model-adapter/custom-planning-types';\nimport type { PlanningTapLocatorDefinition } from '../../model-adapter/types';\nimport { AIResponseParseError } from '../../service-caller/index';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\nimport { ConversationHistory } from '../planning/conversation-history';\nimport { runCustomPlanning } from '../planning/custom-planning';\nimport type { PlanOptions } from '../planning/types';\nimport type { LocateFn, LocateModelResponse, LocateRequest } from './types';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nconst planningActionLocatorActionSpace: DeviceAction[] = [\n  {\n    name: 'Tap',\n    description: 'Tap the element',\n    paramSchema: z.object({\n      locate: getMidsceneLocationSchema(),\n    }),\n    call: async () => undefined,\n  },\n];\n\nasync function buildPlanningTapLocatorPlanOptions(\n  locateRequest: LocateRequest,\n): Promise<PlanOptions> {\n  const { options, locateImage } = locateRequest;\n  const { context } = options;\n\n  return {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        locateImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: {\n        width: locateImage.width,\n        height: locateImage.height,\n      },\n    },\n    actionSpace: planningActionLocatorActionSpace,\n    conversationHistory: new ConversationHistory(),\n    includeLocateInPlanning: true,\n    effort: 'balance',\n  };\n}\n\nexport function resolvePlanningTapLocator<TParsed>(\n  definition: PlanningTapLocatorDefinition,\n  planner: ResolvedCustomPlanningDefinition<TParsed>,\n): LocateFn {\n  const locatorPlanner: ResolvedCustomPlanningDefinition<TParsed> = {\n    ...planner,\n    messages: {\n      ...planner.messages,\n      buildSystemPrompt: definition.buildSystemPrompt,\n      buildUserInstruction: (instruction) => `Tap: ${instruction}`,\n    },\n  };\n\n  return async (locateRequest: LocateRequest): Promise<LocateModelResponse> => {\n    const { targetElementDescription } = locateRequest;\n    assert(\n      targetElementDescription,\n      'cannot find the target element description',\n    );\n\n    let errors: string[] = [];\n    let reasoningContent = '';\n    let rawResponse = '';\n    let rawChoiceMessage: unknown;\n    let usage: LocateModelResponse['usage'];\n\n    try {\n      const locatePlanOptions =\n        await buildPlanningTapLocatorPlanOptions(locateRequest);\n      const planningResponse = await runCustomPlanning(\n        await prepareUserPrompt(targetElementDescription),\n        locatePlanOptions,\n        locatorPlanner,\n      );\n\n      rawResponse = planningResponse.rawResponse ?? '';\n      rawChoiceMessage = planningResponse.rawChoiceMessage;\n      usage = planningResponse.usage;\n      reasoningContent = planningResponse.log;\n\n      debugGrounding('planning-tap-locator rawResponse:', rawResponse);\n\n      const locatedPixelBbox = definition.getLocatedPixelBbox(\n        planningResponse.actions ?? [],\n      );\n\n      if (!locatedPixelBbox) {\n        throw new Error('No locatedPixelBbox found in planner response');\n      }\n\n      return {\n        locatedPixelBbox,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoningContent,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      if (error instanceof AIResponseParseError) {\n        rawResponse = error.rawResponse;\n        rawChoiceMessage = error.rawChoiceMessage;\n        usage = error.usage;\n      }\n      errors = [\n        errorMessage || 'Failed to parse planning tap locator response',\n      ];\n      debugGrounding('planning-tap-locator parse error:', errors[0]);\n    }\n\n    return {\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoningContent,\n      errors,\n    };\n  };\n}\n"],"names":["debugGrounding","getDebug","planningActionLocatorActionSpace","z","getMidsceneLocationSchema","undefined","buildPlanningTapLocatorPlanOptions","locateRequest","options","locateImage","context","ScreenshotItem","ConversationHistory","resolvePlanningTapLocator","definition","planner","locatorPlanner","instruction","targetElementDescription","assert","errors","reasoningContent","rawResponse","rawChoiceMessage","usage","locatePlanOptions","planningResponse","runCustomPlanning","prepareUserPrompt","locatedPixelBbox","Error","error","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;;;AAeA,MAAMA,iBAAiBC,SAAS;AAEhC,MAAMC,mCAAmD;IACvD;QACE,MAAM;QACN,aAAa;QACb,aAAaC,EAAE,MAAM,CAAC;YACpB,QAAQC;QACV;QACA,MAAM,UAAYC;IACpB;CACD;AAED,eAAeC,mCACbC,aAA4B;IAE5B,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGF;IACjC,MAAM,EAAEG,OAAO,EAAE,GAAGF;IAEpB,OAAO;QACL,GAAGA,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYC,eAAe,MAAM,CAC/BF,YAAY,WAAW,EACvBC,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAU;gBACR,OAAOD,YAAY,KAAK;gBACxB,QAAQA,YAAY,MAAM;YAC5B;QACF;QACA,aAAaP;QACb,qBAAqB,IAAIU;QACzB,yBAAyB;QACzB,QAAQ;IACV;AACF;AAEO,SAASC,0BACdC,UAAwC,EACxCC,OAAkD;IAElD,MAAMC,iBAA4D;QAChE,GAAGD,OAAO;QACV,UAAU;YACR,GAAGA,QAAQ,QAAQ;YACnB,mBAAmBD,WAAW,iBAAiB;YAC/C,sBAAsB,CAACG,cAAgB,CAAC,KAAK,EAAEA,aAAa;QAC9D;IACF;IAEA,OAAO,OAAOV;QACZ,MAAM,EAAEW,wBAAwB,EAAE,GAAGX;QACrCY,OACED,0BACA;QAGF,IAAIE,SAAmB,EAAE;QACzB,IAAIC,mBAAmB;QACvB,IAAIC,cAAc;QAClB,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMC,oBACJ,MAAMnB,mCAAmCC;YAC3C,MAAMmB,mBAAmB,MAAMC,kBAC7B,MAAMC,kBAAkBV,2BACxBO,mBACAT;YAGFM,cAAcI,iBAAiB,WAAW,IAAI;YAC9CH,mBAAmBG,iBAAiB,gBAAgB;YACpDF,QAAQE,iBAAiB,KAAK;YAC9BL,mBAAmBK,iBAAiB,GAAG;YAEvC1B,eAAe,qCAAqCsB;YAEpD,MAAMO,mBAAmBf,WAAW,mBAAmB,CACrDY,iBAAiB,OAAO,IAAI,EAAE;YAGhC,IAAI,CAACG,kBACH,MAAM,IAAIC,MAAM;YAGlB,OAAO;gBACLD;gBACAP;gBACAC;gBACAC;gBACAH;YACF;QACF,EAAE,OAAOU,OAAO;YACd,MAAMC,eACJD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;YAClD,IAAIA,iBAAiBG,sBAAsB;gBACzCZ,cAAcS,MAAM,WAAW;gBAC/BR,mBAAmBQ,MAAM,gBAAgB;gBACzCP,QAAQO,MAAM,KAAK;YACrB;YACAX,SAAS;gBACPY,gBAAgB;aACjB;YACDhC,eAAe,qCAAqCoB,MAAM,CAAC,EAAE;QAC/D;QAEA,OAAO;YACLE;YACAC;YACAC;YACAH;YACAD;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/planning-action-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/planning-action-locate.ts"],"sourcesContent":["import type { DeviceAction } from '@/device';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { z } from 'zod';\nimport { getMidsceneLocationSchema } from '../../../common';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type { ResolvedCustomPlanningDefinition } from '../../model-adapter/custom-planning-types';\nimport type { PlanningTapLocatorDefinition } from '../../model-adapter/types';\nimport { AIResponseParseError } from '../../service-caller/index';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\nimport { ConversationHistory } from '../planning/conversation-history';\nimport { runCustomPlanning } from '../planning/custom-planning';\nimport type { PlanOptions } from '../planning/types';\nimport type { LocateFn, LocateModelResponse, LocateRequest } from './types';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nconst planningActionLocatorActionSpace: DeviceAction[] = [\n  {\n    name: 'Tap',\n    description: 'Tap the element',\n    paramSchema: z.object({\n      locate: getMidsceneLocationSchema(),\n    }),\n    call: async () => undefined,\n  },\n];\n\nasync function buildPlanningTapLocatorPlanOptions(\n  locateRequest: LocateRequest,\n): Promise<PlanOptions> {\n  const { options, locateImage } = locateRequest;\n  const { context } = options;\n\n  return {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        locateImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: {\n        width: locateImage.width,\n        height: locateImage.height,\n      },\n    },\n    actionSpace: planningActionLocatorActionSpace,\n    conversationHistory: new ConversationHistory(),\n    includeLocateInPlanning: true,\n    effort: 'balance',\n  };\n}\n\nexport function resolvePlanningTapLocator<TParsed>(\n  definition: PlanningTapLocatorDefinition,\n  planner: ResolvedCustomPlanningDefinition<TParsed>,\n): LocateFn {\n  const locatorPlanner: ResolvedCustomPlanningDefinition<TParsed> = {\n    ...planner,\n    messages: {\n      ...planner.messages,\n      buildSystemPrompt: definition.buildSystemPrompt,\n      buildUserInstruction: (instruction) => `Tap: ${instruction}`,\n    },\n  };\n\n  return async (locateRequest: LocateRequest): Promise<LocateModelResponse> => {\n    const { targetElementDescription } = locateRequest;\n    assert(\n      targetElementDescription,\n      'cannot find the target element description',\n    );\n\n    let errors: string[] = [];\n    let reasoningContent = '';\n    let rawResponse = '';\n    let rawChoiceMessage: unknown;\n    let usage: LocateModelResponse['usage'];\n\n    try {\n      const locatePlanOptions =\n        await buildPlanningTapLocatorPlanOptions(locateRequest);\n      const planningResponse = await runCustomPlanning(\n        await prepareUserPrompt(targetElementDescription),\n        locatePlanOptions,\n        locatorPlanner,\n      );\n\n      rawResponse = planningResponse.rawResponse ?? '';\n      rawChoiceMessage = planningResponse.rawChoiceMessage;\n      usage = planningResponse.usage;\n      reasoningContent = planningResponse.log;\n\n      debugGrounding('planning-tap-locator rawResponse:', rawResponse);\n\n      const locatedPixelResult = definition.getLocatedPixelResult(\n        planningResponse.actions ?? [],\n      );\n\n      if (!locatedPixelResult) {\n        throw new Error('No locatedPixelResult found in planner response');\n      }\n\n      return {\n        locatedPixelResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoningContent,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      if (error instanceof AIResponseParseError) {\n        rawResponse = error.rawResponse;\n        rawChoiceMessage = error.rawChoiceMessage;\n        usage = error.usage;\n      }\n      errors = [\n        errorMessage || 'Failed to parse planning tap locator response',\n      ];\n      debugGrounding('planning-tap-locator parse error:', errors[0]);\n    }\n\n    return {\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoningContent,\n      errors,\n    };\n  };\n}\n"],"names":["debugGrounding","getDebug","planningActionLocatorActionSpace","z","getMidsceneLocationSchema","undefined","buildPlanningTapLocatorPlanOptions","locateRequest","options","locateImage","context","ScreenshotItem","ConversationHistory","resolvePlanningTapLocator","definition","planner","locatorPlanner","instruction","targetElementDescription","assert","errors","reasoningContent","rawResponse","rawChoiceMessage","usage","locatePlanOptions","planningResponse","runCustomPlanning","prepareUserPrompt","locatedPixelResult","Error","error","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;;;AAeA,MAAMA,iBAAiBC,SAAS;AAEhC,MAAMC,mCAAmD;IACvD;QACE,MAAM;QACN,aAAa;QACb,aAAaC,EAAE,MAAM,CAAC;YACpB,QAAQC;QACV;QACA,MAAM,UAAYC;IACpB;CACD;AAED,eAAeC,mCACbC,aAA4B;IAE5B,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGF;IACjC,MAAM,EAAEG,OAAO,EAAE,GAAGF;IAEpB,OAAO;QACL,GAAGA,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYC,eAAe,MAAM,CAC/BF,YAAY,WAAW,EACvBC,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAU;gBACR,OAAOD,YAAY,KAAK;gBACxB,QAAQA,YAAY,MAAM;YAC5B;QACF;QACA,aAAaP;QACb,qBAAqB,IAAIU;QACzB,yBAAyB;QACzB,QAAQ;IACV;AACF;AAEO,SAASC,0BACdC,UAAwC,EACxCC,OAAkD;IAElD,MAAMC,iBAA4D;QAChE,GAAGD,OAAO;QACV,UAAU;YACR,GAAGA,QAAQ,QAAQ;YACnB,mBAAmBD,WAAW,iBAAiB;YAC/C,sBAAsB,CAACG,cAAgB,CAAC,KAAK,EAAEA,aAAa;QAC9D;IACF;IAEA,OAAO,OAAOV;QACZ,MAAM,EAAEW,wBAAwB,EAAE,GAAGX;QACrCY,OACED,0BACA;QAGF,IAAIE,SAAmB,EAAE;QACzB,IAAIC,mBAAmB;QACvB,IAAIC,cAAc;QAClB,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMC,oBACJ,MAAMnB,mCAAmCC;YAC3C,MAAMmB,mBAAmB,MAAMC,kBAC7B,MAAMC,kBAAkBV,2BACxBO,mBACAT;YAGFM,cAAcI,iBAAiB,WAAW,IAAI;YAC9CH,mBAAmBG,iBAAiB,gBAAgB;YACpDF,QAAQE,iBAAiB,KAAK;YAC9BL,mBAAmBK,iBAAiB,GAAG;YAEvC1B,eAAe,qCAAqCsB;YAEpD,MAAMO,qBAAqBf,WAAW,qBAAqB,CACzDY,iBAAiB,OAAO,IAAI,EAAE;YAGhC,IAAI,CAACG,oBACH,MAAM,IAAIC,MAAM;YAGlB,OAAO;gBACLD;gBACAP;gBACAC;gBACAC;gBACAH;YACF;QACF,EAAE,OAAOU,OAAO;YACd,MAAMC,eACJD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;YAClD,IAAIA,iBAAiBG,sBAAsB;gBACzCZ,cAAcS,MAAM,WAAW;gBAC/BR,mBAAmBQ,MAAM,gBAAgB;gBACzCP,QAAQO,MAAM,KAAK;YACrB;YACAX,SAAS;gBACPY,gBAAgB;aACjB;YACDhC,eAAe,qCAAqCoB,MAAM,CAAC,EAAE;QAC/D;QAEA,OAAO;YACLE;YACAC;YACAC;YACAH;YACAD;QACF;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/search-area-mapping.mjs

@@ -1,18 +1,34 @@
-function mapSearchAreaPixelBboxToOriginalPixelBbox([left, top, right, bottom], mapping) {
-    const offset = mapping?.offset ?? {
+function mapSearchAreaResultToOriginalResult(result, mapping) {
+    return {
+        center: mapSearchAreaPointToOriginalPoint(result.center, mapping),
+        ...result.rect ? {
+            rect: mapSearchAreaRectToOriginalRect(result.rect, mapping)
+        } : {}
+    };
+}
+function mapSearchAreaPointToOriginalPoint([x, y], mapping) {
+    const { scale = 1, offset = {
         x: 0,
         y: 0
-    };
-    const scale = mapping?.scale ?? 1;
-    const mapX = (x)=>(1 !== scale ? Math.round(x / scale) : x) + offset.x;
-    const mapY = (y)=>(1 !== scale ? Math.round(y / scale) : y) + offset.y;
+    } } = mapping ?? {};
     return [
-        mapX(left),
-        mapY(top),
-        mapX(right),
-        mapY(bottom)
+        x / scale + offset.x,
+        y / scale + offset.y
     ];
 }
-export { mapSearchAreaPixelBboxToOriginalPixelBbox };
+function mapSearchAreaRectToOriginalRect(rect, mapping) {
+    const [left, top] = mapSearchAreaPointToOriginalPoint([
+        rect.left,
+        rect.top
+    ], mapping);
+    const scale = mapping?.scale ?? 1;
+    return {
+        left,
+        top,
+        width: (rect.width - 1) / scale + 1,
+        height: (rect.height - 1) / scale + 1
+    };
+}
+export { mapSearchAreaPointToOriginalPoint, mapSearchAreaRectToOriginalRect, mapSearchAreaResultToOriginalResult };
 
 //# sourceMappingURL=search-area-mapping.mjs.map

package/dist/es/ai-model/workflows/grounding/search-area-mapping.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/search-area-mapping.mjs","sources":["../../../../../src/ai-model/workflows/grounding/search-area-mapping.ts"],"sourcesContent":["import type { PixelBbox } from '@/ai-model/shared/model-locate-result';\nimport type { SearchAreaImageMapping } from './types';\n\n/** Map a bbox in the prepared search-area image back to screenshot pixels. */\nexport function mapSearchAreaPixelBboxToOriginalPixelBbox(\n  [left, top, right, bottom]: PixelBbox,\n  mapping?: SearchAreaImageMapping,\n): PixelBbox {\n  const offset = mapping?.offset ?? { x: 0, y: 0 };\n  const scale = mapping?.scale ?? 1;\n  const mapX = (x: number) =>\n    (scale !== 1 ? Math.round(x / scale) : x) + offset.x;\n  const mapY = (y: number) =>\n    (scale !== 1 ? Math.round(y / scale) : y) + offset.y;\n\n  return [mapX(left), mapY(top), mapX(right), mapY(bottom)];\n}\n"],"names":["mapSearchAreaPixelBboxToOriginalPixelBbox","left","top","right","bottom","mapping","offset","scale","mapX","x","Math","mapY","y"],"mappings":"AAIO,SAASA,0CACd,CAACC,MAAMC,KAAKC,OAAOC,OAAkB,EACrCC,OAAgC;IAEhC,MAAMC,SAASD,SAAS,UAAU;QAAE,GAAG;QAAG,GAAG;IAAE;IAC/C,MAAME,QAAQF,SAAS,SAAS;IAChC,MAAMG,OAAO,CAACC,IACXF,AAAAA,CAAAA,AAAU,MAAVA,QAAcG,KAAK,KAAK,CAACD,IAAIF,SAASE,CAAAA,IAAKH,OAAO,CAAC;IACtD,MAAMK,OAAO,CAACC,IACXL,AAAAA,CAAAA,AAAU,MAAVA,QAAcG,KAAK,KAAK,CAACE,IAAIL,SAASK,CAAAA,IAAKN,OAAO,CAAC;IAEtD,OAAO;QAACE,KAAKP;QAAOU,KAAKT;QAAMM,KAAKL;QAAQQ,KAAKP;KAAQ;AAC3D"}
+{"version":3,"file":"ai-model/workflows/grounding/search-area-mapping.mjs","sources":["../../../../../src/ai-model/workflows/grounding/search-area-mapping.ts"],"sourcesContent":["import type { Rect } from '@midscene/shared/types';\nimport type { PixelLocateResult } from '../../shared/model-locate-result';\nimport type { SearchAreaImageMapping } from './types';\n\n/** Restore a complete locate result, keeping center independent from bbox metadata. */\nexport function mapSearchAreaResultToOriginalResult(\n  result: PixelLocateResult,\n  mapping?: SearchAreaImageMapping,\n): PixelLocateResult {\n  return {\n    center: mapSearchAreaPointToOriginalPoint(result.center, mapping),\n    ...(result.rect\n      ? { rect: mapSearchAreaRectToOriginalRect(result.rect, mapping) }\n      : {}),\n  };\n}\n\n/** Restore the target point without intermediate rounding. */\nexport function mapSearchAreaPointToOriginalPoint(\n  [x, y]: [number, number],\n  mapping?: SearchAreaImageMapping,\n): [number, number] {\n  const { scale = 1, offset = { x: 0, y: 0 } } = mapping ?? {};\n  return [x / scale + offset.x, y / scale + offset.y];\n}\n\n/** Restore bbox metadata independently from the target point. */\nexport function mapSearchAreaRectToOriginalRect(\n  rect: Rect,\n  mapping?: SearchAreaImageMapping,\n): Rect {\n  const [left, top] = mapSearchAreaPointToOriginalPoint(\n    [rect.left, rect.top],\n    mapping,\n  );\n  const scale = mapping?.scale ?? 1;\n  return {\n    left,\n    top,\n    width: (rect.width - 1) / scale + 1,\n    height: (rect.height - 1) / scale + 1,\n  };\n}\n"],"names":["mapSearchAreaResultToOriginalResult","result","mapping","mapSearchAreaPointToOriginalPoint","mapSearchAreaRectToOriginalRect","x","y","scale","offset","rect","left","top"],"mappings":"AAKO,SAASA,oCACdC,MAAyB,EACzBC,OAAgC;IAEhC,OAAO;QACL,QAAQC,kCAAkCF,OAAO,MAAM,EAAEC;QACzD,GAAID,OAAO,IAAI,GACX;YAAE,MAAMG,gCAAgCH,OAAO,IAAI,EAAEC;QAAS,IAC9D,CAAC,CAAC;IACR;AACF;AAGO,SAASC,kCACd,CAACE,GAAGC,EAAoB,EACxBJ,OAAgC;IAEhC,MAAM,EAAEK,QAAQ,CAAC,EAAEC,SAAS;QAAE,GAAG;QAAG,GAAG;IAAE,CAAC,EAAE,GAAGN,WAAW,CAAC;IAC3D,OAAO;QAACG,IAAIE,QAAQC,OAAO,CAAC;QAAEF,IAAIC,QAAQC,OAAO,CAAC;KAAC;AACrD;AAGO,SAASJ,gCACdK,IAAU,EACVP,OAAgC;IAEhC,MAAM,CAACQ,MAAMC,IAAI,GAAGR,kCAClB;QAACM,KAAK,IAAI;QAAEA,KAAK,GAAG;KAAC,EACrBP;IAEF,MAAMK,QAAQL,SAAS,SAAS;IAChC,OAAO;QACLQ;QACAC;QACA,OAAQF,AAAAA,CAAAA,KAAK,KAAK,GAAG,KAAKF,QAAQ;QAClC,QAASE,AAAAA,CAAAA,KAAK,MAAM,GAAG,KAAKF,QAAQ;IACtC;AACF"}

package/dist/es/ai-model/workflows/grounding/search-area.mjs

@@ -1,4 +1,29 @@
 import { cropByRect, scaleImage } from "@midscene/shared/img";
+function mergeSearchAreaResults(target, references = []) {
+    const bounds = [
+        target,
+        ...references
+    ].map(({ rect, center })=>{
+        const left = rect?.left ?? Math.round(center[0]);
+        const top = rect?.top ?? Math.round(center[1]);
+        return {
+            left,
+            top,
+            right: left + (rect?.width ?? 1),
+            bottom: top + (rect?.height ?? 1)
+        };
+    });
+    const left = Math.min(...bounds.map((bound)=>bound.left));
+    const top = Math.min(...bounds.map((bound)=>bound.top));
+    const right = Math.max(...bounds.map((bound)=>bound.right));
+    const bottom = Math.max(...bounds.map((bound)=>bound.bottom));
+    return {
+        left,
+        top,
+        width: right - left,
+        height: bottom - top
+    };
+}
 function expandSearchArea(rect, screenSize) {
     const minArea = 160000;
     const expandSize = 100;
@@ -50,6 +75,6 @@ async function buildSearchAreaConfig(options) {
         }
     };
 }
-export { buildSearchAreaConfig, expandSearchArea };
+export { buildSearchAreaConfig, expandSearchArea, mergeSearchAreaResults };
 
 //# sourceMappingURL=search-area.mjs.map