@midscene/core 1.16.5 → 1.17.0

package/dist/lib/ai-model/shared/model-locate-result/pixel-mapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/pixel-mapper.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/shared/model-locate-result/pixel-mapper.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { Rect } from '@/types';\nimport { isBboxLocateResultValue } from './types';\nimport type {\n  BboxLocateResultValue,\n  CoordinateRounding,\n  LocateResultPoint,\n  LocateResultValue,\n  PixelBbox,\n} from './types';\n\nfunction clamp(value: number, min: number, max: number) {\n  return Math.min(Math.max(value, min), max);\n}\n\nexport function maxPixelCoordinate(size: number) {\n  return Math.max(size - 1, 0);\n}\n\nexport function normalizedCoordinateToPixel(\n  value: number,\n  normalizedBy: number,\n  size: number,\n  rounding: CoordinateRounding,\n) {\n  const pixel = (value * size) / normalizedBy;\n  switch (rounding) {\n    case 'round':\n      return Math.round(pixel);\n    case 'trunc':\n      return Math.trunc(pixel);\n    case 'none':\n      return pixel;\n    default:\n      throw new Error(`Invalid coordinate rounding: ${rounding}`);\n  }\n}\n\n/** Restrict an already mapped point to the effective content area. */\nexport function clampPixelPointToSize(\n  [x, y]: LocateResultPoint,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  return [\n    clamp(x, 0, maxPixelCoordinate(width)),\n    clamp(y, 0, maxPixelCoordinate(height)),\n  ];\n}\n\n/** Restrict an already mapped bbox to the effective content area. */\nexport function clampPixelBboxToSize(\n  pixelBbox: PixelBbox,\n  size: { width: number; height: number },\n): PixelBbox {\n  const [left, top, right, bottom] = pixelBbox;\n  return [\n    ...clampPixelPointToSize([left, top], size),\n    ...clampPixelPointToSize([right, bottom], size),\n  ];\n}\n\nexport function pixelBboxToRect([left, top, right, bottom]: PixelBbox): Rect {\n  return {\n    left,\n    top,\n    width: right - left + 1,\n    height: bottom - top + 1,\n  };\n}\n\nexport function mapLocateResultToPixelBbox(\n  result: BboxLocateResultValue,\n  { width, height }: { width: number; height: number },\n): PixelBbox {\n  const { normalizedBy, order, rounding } = result.coordinatesMeta;\n  const [first, second, third, fourth] = result.coordinates;\n  const xyBbox: PixelBbox =\n    order === 'yx' ? [second, first, fourth, third] : result.coordinates;\n  return normalizedBy === undefined\n    ? xyBbox\n    : [\n        normalizedCoordinateToPixel(xyBbox[0], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[1], normalizedBy, height, rounding),\n        normalizedCoordinateToPixel(xyBbox[2], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[3], normalizedBy, height, rounding),\n      ];\n}\n\n/** Resolve the target before any rounding or clipping can shift its center. */\nexport function mapLocateResultToPixelPoint(\n  result: LocateResultValue,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  const { order, normalizedBy, rounding } = result.coordinatesMeta;\n  const [first, second] = isBboxLocateResultValue(result)\n    ? [\n        (result.coordinates[0] + result.coordinates[2]) / 2,\n        (result.coordinates[1] + result.coordinates[3]) / 2,\n      ]\n    : result.coordinates;\n  const [x, y] = order === 'yx' ? [second, first] : [first, second];\n  return normalizedBy === undefined\n    ? [x, y]\n    : [\n        normalizedCoordinateToPixel(x, normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(y, normalizedBy, height, rounding),\n      ];\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","clamp","value","min","max","Math","maxPixelCoordinate","size","normalizedCoordinateToPixel","normalizedBy","rounding","pixel","Error","clampPixelPointToSize","x","y","width","height","clampPixelBboxToSize","pixelBbox","left","top","right","bottom","pixelBboxToRect","mapLocateResultToPixelBbox","result","order","first","second","third","fourth","xyBbox","undefined","mapLocateResultToPixelPoint","isBboxLocateResultValue"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACIA,SAASI,MAAMC,KAAa,EAAEC,GAAW,EAAEC,GAAW;IACpD,OAAOC,KAAK,GAAG,CAACA,KAAK,GAAG,CAACH,OAAOC,MAAMC;AACxC;AAEO,SAASE,mBAAmBC,IAAY;IAC7C,OAAOF,KAAK,GAAG,CAACE,OAAO,GAAG;AAC5B;AAEO,SAASC,4BACdN,KAAa,EACbO,YAAoB,EACpBF,IAAY,EACZG,QAA4B;IAE5B,MAAMC,QAAST,QAAQK,OAAQE;IAC/B,OAAQC;QACN,KAAK;YACH,OAAOL,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAON,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAOA;QACT;YACE,MAAM,IAAIC,MAAM,CAAC,6BAA6B,EAAEF,UAAU;IAC9D;AACF;AAGO,SAASG,sBACd,CAACC,GAAGC,EAAqB,EACzB,EAAEC,KAAK,EAAEC,MAAM,EAAqC;IAEpD,OAAO;QACLhB,MAAMa,GAAG,GAAGR,mBAAmBU;QAC/Bf,MAAMc,GAAG,GAAGT,mBAAmBW;KAChC;AACH;AAGO,SAASC,qBACdC,SAAoB,EACpBZ,IAAuC;IAEvC,MAAM,CAACa,MAAMC,KAAKC,OAAOC,OAAO,GAAGJ;IACnC,OAAO;WACFN,sBAAsB;YAACO;YAAMC;SAAI,EAAEd;WACnCM,sBAAsB;YAACS;YAAOC;SAAO,EAAEhB;KAC3C;AACH;AAEO,SAASiB,gBAAgB,CAACJ,MAAMC,KAAKC,OAAOC,OAAkB;IACnE,OAAO;QACLH;QACAC;QACA,OAAOC,QAAQF,OAAO;QACtB,QAAQG,SAASF,MAAM;IACzB;AACF;AAEO,SAASI,2BACdC,MAA6B,EAC7B,EAAEV,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAER,YAAY,EAAEkB,KAAK,EAAEjB,QAAQ,EAAE,GAAGgB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,QAAQC,OAAOC,OAAO,GAAGL,OAAO,WAAW;IACzD,MAAMM,SACJL,AAAU,SAAVA,QAAiB;QAACE;QAAQD;QAAOG;QAAQD;KAAM,GAAGJ,OAAO,WAAW;IACtE,OAAOjB,AAAiBwB,WAAjBxB,eACHuB,SACA;QACExB,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcO,OAAON;QAC5DF,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcQ,QAAQP;QAC7DF,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcO,OAAON;QAC5DF,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcQ,QAAQP;KAC9D;AACP;AAGO,SAASwB,4BACdR,MAAyB,EACzB,EAAEV,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAEU,KAAK,EAAElB,YAAY,EAAEC,QAAQ,EAAE,GAAGgB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,OAAO,GAAGM,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwBT,UAC5C;QACGA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;QACjDA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;KACnD,GACDA,OAAO,WAAW;IACtB,MAAM,CAACZ,GAAGC,EAAE,GAAGY,AAAU,SAAVA,QAAiB;QAACE;QAAQD;KAAM,GAAG;QAACA;QAAOC;KAAO;IACjE,OAAOpB,AAAiBwB,WAAjBxB,eACH;QAACK;QAAGC;KAAE,GACN;QACEP,4BAA4BM,GAAGL,cAAcO,OAAON;QACpDF,4BAA4BO,GAAGN,cAAcQ,QAAQP;KACtD;AACP"}
+{"version":3,"file":"ai-model/shared/model-locate-result/pixel-mapper.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/shared/model-locate-result/pixel-mapper.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { Rect } from '@/types';\nimport { isBboxLocateResultValue } from './types';\nimport type {\n  BboxBoundsType,\n  BboxLocateResultValue,\n  CoordinateRounding,\n  LocateResultPoint,\n  LocateResultValue,\n  PixelBbox,\n} from './types';\n\nfunction clamp(value: number, min: number, max: number) {\n  return Math.min(Math.max(value, min), max);\n}\n\nexport function maxPixelCoordinate(size: number) {\n  return Math.max(size - 1, 0);\n}\n\nexport function normalizedCoordinateToPixel(\n  value: number,\n  normalizedBy: number,\n  size: number,\n  rounding: CoordinateRounding,\n) {\n  const pixel = (value * size) / normalizedBy;\n  switch (rounding) {\n    case 'round':\n      return Math.round(pixel);\n    case 'trunc':\n      return Math.trunc(pixel);\n    case 'none':\n      return pixel;\n    default:\n      throw new Error(`Invalid coordinate rounding: ${rounding}`);\n  }\n}\n\n/** Restrict an already mapped point to the effective content area. */\nexport function clampPixelPointToSize(\n  [x, y]: LocateResultPoint,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  return [\n    clamp(x, 0, maxPixelCoordinate(width)),\n    clamp(y, 0, maxPixelCoordinate(height)),\n  ];\n}\n\n/** Restrict an already mapped bbox to the effective content area. */\nexport function clampPixelBboxToSize(\n  pixelBbox: PixelBbox,\n  size: { width: number; height: number },\n  boundsType: BboxBoundsType = 'inclusive',\n): PixelBbox {\n  const [left, top, right, bottom] = pixelBbox;\n  if (boundsType === 'edges') {\n    return [\n      clamp(left, 0, size.width),\n      clamp(top, 0, size.height),\n      clamp(right, 0, size.width),\n      clamp(bottom, 0, size.height),\n    ];\n  }\n  return [\n    ...clampPixelPointToSize([left, top], size),\n    ...clampPixelPointToSize([right, bottom], size),\n  ];\n}\n\nexport function pixelBboxToRect(\n  [left, top, right, bottom]: PixelBbox,\n  boundsType: BboxBoundsType = 'inclusive',\n): Rect {\n  return {\n    left,\n    top,\n    width: right - left + (boundsType === 'inclusive' ? 1 : 0),\n    height: bottom - top + (boundsType === 'inclusive' ? 1 : 0),\n  };\n}\n\nexport function mapLocateResultToPixelBbox(\n  result: BboxLocateResultValue,\n  { width, height }: { width: number; height: number },\n): PixelBbox {\n  const { normalizedBy, order, rounding } = result.coordinatesMeta;\n  const [first, second, third, fourth] = result.coordinates;\n  const xyBbox: PixelBbox =\n    order === 'yx' ? [second, first, fourth, third] : result.coordinates;\n  return normalizedBy === undefined\n    ? xyBbox\n    : [\n        normalizedCoordinateToPixel(xyBbox[0], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[1], normalizedBy, height, rounding),\n        normalizedCoordinateToPixel(xyBbox[2], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[3], normalizedBy, height, rounding),\n      ];\n}\n\n/** Resolve the target before any rounding or clipping can shift its center. */\nexport function mapLocateResultToPixelPoint(\n  result: LocateResultValue,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  const { order, normalizedBy, rounding } = result.coordinatesMeta;\n  const [first, second] = isBboxLocateResultValue(result)\n    ? [\n        (result.coordinates[0] + result.coordinates[2]) / 2,\n        (result.coordinates[1] + result.coordinates[3]) / 2,\n      ]\n    : result.coordinates;\n  const [x, y] = order === 'yx' ? [second, first] : [first, second];\n  return normalizedBy === undefined\n    ? [x, y]\n    : [\n        normalizedCoordinateToPixel(x, normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(y, normalizedBy, height, rounding),\n      ];\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","clamp","value","min","max","Math","maxPixelCoordinate","size","normalizedCoordinateToPixel","normalizedBy","rounding","pixel","Error","clampPixelPointToSize","x","y","width","height","clampPixelBboxToSize","pixelBbox","boundsType","left","top","right","bottom","pixelBboxToRect","mapLocateResultToPixelBbox","result","order","first","second","third","fourth","xyBbox","undefined","mapLocateResultToPixelPoint","isBboxLocateResultValue"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACKA,SAASI,MAAMC,KAAa,EAAEC,GAAW,EAAEC,GAAW;IACpD,OAAOC,KAAK,GAAG,CAACA,KAAK,GAAG,CAACH,OAAOC,MAAMC;AACxC;AAEO,SAASE,mBAAmBC,IAAY;IAC7C,OAAOF,KAAK,GAAG,CAACE,OAAO,GAAG;AAC5B;AAEO,SAASC,4BACdN,KAAa,EACbO,YAAoB,EACpBF,IAAY,EACZG,QAA4B;IAE5B,MAAMC,QAAST,QAAQK,OAAQE;IAC/B,OAAQC;QACN,KAAK;YACH,OAAOL,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAON,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAOA;QACT;YACE,MAAM,IAAIC,MAAM,CAAC,6BAA6B,EAAEF,UAAU;IAC9D;AACF;AAGO,SAASG,sBACd,CAACC,GAAGC,EAAqB,EACzB,EAAEC,KAAK,EAAEC,MAAM,EAAqC;IAEpD,OAAO;QACLhB,MAAMa,GAAG,GAAGR,mBAAmBU;QAC/Bf,MAAMc,GAAG,GAAGT,mBAAmBW;KAChC;AACH;AAGO,SAASC,qBACdC,SAAoB,EACpBZ,IAAuC,EACvCa,aAA6B,WAAW;IAExC,MAAM,CAACC,MAAMC,KAAKC,OAAOC,OAAO,GAAGL;IACnC,IAAIC,AAAe,YAAfA,YACF,OAAO;QACLnB,MAAMoB,MAAM,GAAGd,KAAK,KAAK;QACzBN,MAAMqB,KAAK,GAAGf,KAAK,MAAM;QACzBN,MAAMsB,OAAO,GAAGhB,KAAK,KAAK;QAC1BN,MAAMuB,QAAQ,GAAGjB,KAAK,MAAM;KAC7B;IAEH,OAAO;WACFM,sBAAsB;YAACQ;YAAMC;SAAI,EAAEf;WACnCM,sBAAsB;YAACU;YAAOC;SAAO,EAAEjB;KAC3C;AACH;AAEO,SAASkB,gBACd,CAACJ,MAAMC,KAAKC,OAAOC,OAAkB,EACrCJ,aAA6B,WAAW;IAExC,OAAO;QACLC;QACAC;QACA,OAAOC,QAAQF,OAAQD,CAAAA,AAAe,gBAAfA,aAA6B,IAAI;QACxD,QAAQI,SAASF,MAAOF,CAAAA,AAAe,gBAAfA,aAA6B,IAAI;IAC3D;AACF;AAEO,SAASM,2BACdC,MAA6B,EAC7B,EAAEX,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAER,YAAY,EAAEmB,KAAK,EAAElB,QAAQ,EAAE,GAAGiB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,QAAQC,OAAOC,OAAO,GAAGL,OAAO,WAAW;IACzD,MAAMM,SACJL,AAAU,SAAVA,QAAiB;QAACE;QAAQD;QAAOG;QAAQD;KAAM,GAAGJ,OAAO,WAAW;IACtE,OAAOlB,AAAiByB,WAAjBzB,eACHwB,SACA;QACEzB,4BAA4ByB,MAAM,CAAC,EAAE,EAAExB,cAAcO,OAAON;QAC5DF,4BAA4ByB,MAAM,CAAC,EAAE,EAAExB,cAAcQ,QAAQP;QAC7DF,4BAA4ByB,MAAM,CAAC,EAAE,EAAExB,cAAcO,OAAON;QAC5DF,4BAA4ByB,MAAM,CAAC,EAAE,EAAExB,cAAcQ,QAAQP;KAC9D;AACP;AAGO,SAASyB,4BACdR,MAAyB,EACzB,EAAEX,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAEW,KAAK,EAAEnB,YAAY,EAAEC,QAAQ,EAAE,GAAGiB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,OAAO,GAAGM,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwBT,UAC5C;QACGA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;QACjDA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;KACnD,GACDA,OAAO,WAAW;IACtB,MAAM,CAACb,GAAGC,EAAE,GAAGa,AAAU,SAAVA,QAAiB;QAACE;QAAQD;KAAM,GAAG;QAACA;QAAOC;KAAO;IACjE,OAAOrB,AAAiByB,WAAjBzB,eACH;QAACK;QAAGC;KAAE,GACN;QACEP,4BAA4BM,GAAGL,cAAcO,OAAON;QACpDF,4BAA4BO,GAAGN,cAAcQ,QAAQP;KACtD;AACP"}

package/dist/lib/ai-model/shared/model-locate-result/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/types.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/shared/model-locate-result/types.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { Rect } from '@midscene/shared/types';\nimport type { Bbox } from '../../../types';\n\nexport type { Bbox };\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\nexport type NonEmptyArray<T> = [T, ...T[]];\nexport type RawLocateValue = unknown;\n\nexport type LocateResultPoint = [number, number];\nexport type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'point';\n};\nexport type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'bbox';\n};\n\nexport type LocateResultValue =\n  | {\n      coordinates: LocateResultPoint;\n      coordinatesMeta: PointLocateResultCoordinates;\n    }\n  | {\n      coordinates: LocateResultBbox;\n      coordinatesMeta: BboxLocateResultCoordinates;\n    };\n\nexport type PointLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'point' } }\n>;\nexport type BboxLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'bbox' } }\n>;\n\nexport function isBboxLocateResultValue(\n  result: LocateResultValue,\n): result is BboxLocateResultValue {\n  return result.coordinatesMeta.shape === 'bbox';\n}\n\nexport function isPointLocateResultValue(\n  result: LocateResultValue,\n): result is PointLocateResultValue {\n  return result.coordinatesMeta.shape === 'point';\n}\n\nexport type LocateResultShape = 'bbox' | 'point';\n\nexport interface LocateResultContext {\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  contentSize?: {\n    width: number;\n    height: number;\n  };\n}\n\nexport interface LocateResultPromptSpec {\n  resultKey: string;\n  /** Parse-only alternatives, tried in order after resultKey. */\n  resultKeyAliases?: string[];\n  resultValueSchema: string;\n  resultValueDescription: string;\n  resultNoun: string;\n  resultNounPlural: string;\n  exampleValues: NonEmptyArray<unknown>;\n}\n\nexport interface PixelLocateResult {\n  /** Raw bbox midpoint (or raw point), mapped to the locate image using the configured rounding. */\n  center: LocateResultPoint;\n  /**\n   * Original bbox mapped to locate-image pixels and clipped to content bounds.\n   * Undefined for point results; never derive center or click coordinates from it.\n   */\n  rect?: Rect;\n}\n\nexport interface LocateResultCodec {\n  promptSpec: LocateResultPromptSpec;\n  /** Parse once and independently map the center and optional original bbox. */\n  toPixelResult(\n    input: RawLocateValue,\n    ctx: LocateResultContext,\n  ): PixelLocateResult;\n}\n\nexport type CoordinateRounding = 'round' | 'trunc' | 'none';\n\nexport interface LocateResultCoordinates {\n  shape: LocateResultShape;\n  /** Axis order in the raw coordinates; defaults to xy. */\n  order?: 'xy' | 'yx';\n  /** Normalization range, e.g. 1 or 1000. Omit for model-image pixel coordinates. */\n  normalizedBy?: number;\n  /** Rounding after normalized-to-pixel mapping; defaults to round. Pixel inputs retain their precision. */\n  rounding?: CoordinateRounding;\n}\n\n/** Coordinate metadata with defaults resolved and an explicit rounding policy. */\nexport type ResolvedLocateResultCoordinates =\n  | {\n      shape: 'point';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    }\n  | {\n      shape: 'bbox';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    };\n\nexport type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;\n\n/**\n * Format of one model coordinate value, shared by element and search-area locate.\n * The operation protocol extracts target/reference values from the response;\n * the codec parses each value once and maps it to `{ center, rect }` using the\n * parsed result's coordinatesMeta. Rect is undefined for actual point results.\n * Raw-value parsing and normalized-coordinate rounding are configurable;\n * coordinate mapping is shared.\n */\nexport type LocateResultFormatDefinition = {\n  /** Response field used in prompts and parsing; defaults to bbox or point. */\n  resultKey?: string;\n  /** Parse-only alternatives, tried in order when the primary field is undefined. */\n  resultKeyAliases?: string[];\n  /**\n   * Expected shape, axis order, and normalization used by the prompt and default\n   * parser. A custom parser may return different metadata for an actual response.\n   */\n  coordinates: LocateResultCoordinates;\n  /**\n   * Optional model-specific parser for nonstandard formats or point/bbox fallback.\n   * Return `{ coordinates, coordinatesMeta }` describing the actual shape, axis\n   * order, and normalization. Preserve coordinate precision; do not round or map\n   * to pixels here. Include an explicit rounding policy in `coordinatesMeta`.\n   * Omit the parser to parse numeric values according to `coordinates`.\n   */\n  parseRawLocateValue?: RawLocateValueParser;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","isBboxLocateResultValue","result","isPointLocateResultValue"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;AC8BO,SAASI,wBACdC,MAAyB;IAEzB,OAAOA,AAAiC,WAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC;AAEO,SAASC,yBACdD,MAAyB;IAEzB,OAAOA,AAAiC,YAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC"}
+{"version":3,"file":"ai-model/shared/model-locate-result/types.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/shared/model-locate-result/types.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { Rect } from '@midscene/shared/types';\nimport type { Bbox } from '../../../types';\n\nexport type { Bbox };\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\nexport type NonEmptyArray<T> = [T, ...T[]];\nexport type RawLocateValue = unknown;\n\nexport type LocateResultPoint = [number, number];\nexport type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'point';\n};\nexport type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'bbox';\n};\n\nexport type LocateResultValue =\n  | {\n      coordinates: LocateResultPoint;\n      coordinatesMeta: PointLocateResultCoordinates;\n    }\n  | {\n      coordinates: LocateResultBbox;\n      coordinatesMeta: BboxLocateResultCoordinates;\n    };\n\nexport type PointLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'point' } }\n>;\nexport type BboxLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'bbox' } }\n>;\n\nexport function isBboxLocateResultValue(\n  result: LocateResultValue,\n): result is BboxLocateResultValue {\n  return result.coordinatesMeta.shape === 'bbox';\n}\n\nexport function isPointLocateResultValue(\n  result: LocateResultValue,\n): result is PointLocateResultValue {\n  return result.coordinatesMeta.shape === 'point';\n}\n\nexport type LocateResultShape = 'bbox' | 'point';\n\nexport interface LocateResultContext {\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  contentSize?: {\n    width: number;\n    height: number;\n  };\n}\n\nexport interface LocateResultPromptSpec {\n  resultKey: string;\n  /** Parse-only alternatives, tried in order after resultKey. */\n  resultKeyAliases?: string[];\n  resultValueSchema: string;\n  resultValueDescription: string;\n  resultNoun: string;\n  resultNounPlural: string;\n  exampleValues: NonEmptyArray<unknown>;\n}\n\nexport interface PixelLocateResult {\n  /** Raw bbox midpoint (or raw point), mapped to the locate image using the configured rounding. */\n  center: LocateResultPoint;\n  /**\n   * Original bbox mapped to locate-image pixels and clipped to content bounds.\n   * Undefined for point results; never derive center or click coordinates from it.\n   */\n  rect?: Rect;\n}\n\nexport interface LocateResultCodec {\n  promptSpec: LocateResultPromptSpec;\n  /** Parse once and independently map the center and optional original bbox. */\n  toPixelResult(\n    input: RawLocateValue,\n    ctx: LocateResultContext,\n  ): PixelLocateResult;\n}\n\nexport type CoordinateRounding = 'round' | 'trunc' | 'none';\n\nexport type BboxBoundsType = 'inclusive' | 'edges';\n\nexport interface LocateResultCoordinates {\n  /** Bbox boundary semantics; defaults to inclusive pixel endpoints. */\n  boundsType?: BboxBoundsType;\n  shape: LocateResultShape;\n  /** Axis order in the raw coordinates; defaults to xy. */\n  order?: 'xy' | 'yx';\n  /** Normalization range, e.g. 1 or 1000. Omit for model-image pixel coordinates. */\n  normalizedBy?: number;\n  /** Rounding after normalized-to-pixel mapping; defaults to round. Pixel inputs retain their precision. */\n  rounding?: CoordinateRounding;\n}\n\n/** Coordinate metadata with defaults resolved and an explicit rounding policy. */\nexport type ResolvedLocateResultCoordinates =\n  | {\n      shape: 'point';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    }\n  | {\n      shape: 'bbox';\n      boundsType?: BboxBoundsType;\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    };\n\nexport type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;\n\n/**\n * Format of one model coordinate value, shared by element and search-area locate.\n * The operation protocol extracts target/reference values from the response;\n * the codec parses each value once and maps it to `{ center, rect }` using the\n * parsed result's coordinatesMeta. Rect is undefined for actual point results.\n * Raw-value parsing and normalized-coordinate rounding are configurable;\n * coordinate mapping is shared.\n */\nexport type LocateResultFormatDefinition = {\n  /** Response field used in prompts and parsing; defaults to bbox or point. */\n  resultKey?: string;\n  /** Parse-only alternatives, tried in order when the primary field is undefined. */\n  resultKeyAliases?: string[];\n  /**\n   * Expected shape, axis order, and normalization used by the prompt and default\n   * parser. A custom parser may return different metadata for an actual response.\n   */\n  coordinates: LocateResultCoordinates;\n  /**\n   * Optional model-specific parser for nonstandard formats or point/bbox fallback.\n   * Return `{ coordinates, coordinatesMeta }` describing the actual shape, axis\n   * order, and normalization. Preserve coordinate precision; do not round or map\n   * to pixels here. Include an explicit rounding policy in `coordinatesMeta`.\n   * Omit the parser to parse numeric values according to `coordinates`.\n   */\n  parseRawLocateValue?: RawLocateValueParser;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","isBboxLocateResultValue","result","isPointLocateResultValue"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;AC8BO,SAASI,wBACdC,MAAyB;IAEzB,OAAOA,AAAiC,WAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC;AAEO,SAASC,yBACdD,MAAyB;IAEzB,OAAOA,AAAiC,YAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC"}

package/dist/lib/ai-model/shared/model-locate-result/validation.js

@@ -35,6 +35,10 @@ function assertLocateResultStructure(result) {
     if (!expectedLength) throw new Error(`invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(coordinatesMeta?.shape)}`);
     const coordinates = result.coordinates;
     if (!Array.isArray(coordinates) || coordinates.length !== expectedLength || !coordinates.every((value)=>'number' == typeof value && Number.isFinite(value))) throw new Error(`invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(coordinates)}`);
+    if ((0, external_types_js_namespaceObject.isBboxLocateResultValue)(result) && void 0 !== result.coordinatesMeta.boundsType && ![
+        'inclusive',
+        'edges'
+    ].includes(result.coordinatesMeta.boundsType)) throw new Error(`invalid locate bbox boundsType: ${result.coordinatesMeta.boundsType}`);
     const { order, normalizedBy, rounding } = coordinatesMeta;
     if (![
         'round',

package/dist/lib/ai-model/shared/model-locate-result/validation.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/validation.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/shared/model-locate-result/validation.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { isBboxLocateResultValue } from './types';\nimport type { LocateResultValue } from './types';\n\nexport function assertLocateResultStructure(result: LocateResultValue): void {\n  if (!result || typeof result !== 'object') {\n    throw new Error(\n      `invalid parsed locate result: expected object, got ${JSON.stringify(\n        result,\n      )}`,\n    );\n  }\n\n  const coordinatesMeta = result.coordinatesMeta;\n  const expectedLength =\n    coordinatesMeta?.shape === 'bbox'\n      ? 4\n      : coordinatesMeta?.shape === 'point'\n        ? 2\n        : 0;\n  if (!expectedLength) {\n    throw new Error(\n      `invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(\n        coordinatesMeta?.shape,\n      )}`,\n    );\n  }\n\n  const coordinates = result.coordinates;\n  if (\n    !Array.isArray(coordinates) ||\n    coordinates.length !== expectedLength ||\n    !coordinates.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(\n      `invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(\n        coordinates,\n      )}`,\n    );\n  }\n\n  const { order, normalizedBy, rounding } = coordinatesMeta;\n  if (!['round', 'trunc', 'none'].includes(rounding)) {\n    throw new Error(`invalid locate coordinate rounding: ${rounding}`);\n  }\n  if (order !== 'xy' && order !== 'yx') {\n    throw new Error(\n      `invalid locate coordinate order: ${JSON.stringify(order)}`,\n    );\n  }\n  if (\n    normalizedBy !== undefined &&\n    (!Number.isFinite(normalizedBy) || normalizedBy <= 0)\n  ) {\n    throw new Error(\n      `locate result normalizedBy must be positive and finite: ${normalizedBy}`,\n    );\n  }\n}\n\nfunction resolveCoordinateLimits(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n): number[] {\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  if (normalizedBy !== undefined) {\n    return result.coordinates.map(() => normalizedBy);\n  }\n\n  if (resolvedCoordinates.shape === 'bbox') {\n    return resolvedCoordinates.order === 'yx'\n      ? [maxY, maxX, maxY, maxX]\n      : [maxX, maxY, maxX, maxY];\n  }\n\n  return resolvedCoordinates.order === 'yx' ? [maxY, maxX] : [maxX, maxY];\n}\n\nexport function assertLocateResultCoordinateRangeAndOrder(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n) {\n  const coordinatesMeta = result.coordinatesMeta;\n  const { normalizedBy } = coordinatesMeta;\n  const limits = resolveCoordinateLimits(result, maxX, maxY);\n  const outOfRange = result.coordinates.some(\n    (value, index) => value < 0 || value > limits[index],\n  );\n\n  if (outOfRange) {\n    const source =\n      normalizedBy !== undefined\n        ? `normalized range [0, ${normalizedBy}]`\n        : `image size [0, ${maxX}]x[0, ${maxY}]`;\n    const normalizedInfo =\n      normalizedBy !== undefined ? ` normalizedBy=${normalizedBy}` : '';\n    throw new Error(\n      `locate result coordinates ${JSON.stringify(\n        result.coordinates,\n      )} exceed ${source}. shape=${\n        coordinatesMeta.shape\n      } order=${coordinatesMeta.order}${normalizedInfo} limits=${JSON.stringify(\n        limits,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    (result.coordinates[2] < result.coordinates[0] ||\n      result.coordinates[3] < result.coordinates[1])\n  ) {\n    throw new Error(\n      `invalid locate bbox coordinate order: ${JSON.stringify(result.coordinates)}`,\n    );\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","assertLocateResultStructure","result","Error","JSON","coordinatesMeta","expectedLength","coordinates","Array","value","Number","order","normalizedBy","rounding","undefined","resolveCoordinateLimits","maxX","maxY","resolvedCoordinates","assertLocateResultCoordinateRangeAndOrder","limits","outOfRange","index","source","normalizedInfo","isBboxLocateResultValue"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACHO,SAASI,4BAA4BC,MAAyB;IACnE,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,QACpB,MAAM,IAAIC,MACR,CAAC,mDAAmD,EAAEC,KAAK,SAAS,CAClEF,SACC;IAIP,MAAMG,kBAAkBH,OAAO,eAAe;IAC9C,MAAMI,iBACJD,iBAAiB,UAAU,SACvB,IACAA,iBAAiB,UAAU,UACzB,IACA;IACR,IAAI,CAACC,gBACH,MAAM,IAAIH,MACR,CAAC,gEAAgE,EAAEC,KAAK,SAAS,CAC/EC,iBAAiB,QAChB;IAIP,MAAME,cAAcL,OAAO,WAAW;IACtC,IACE,CAACM,MAAM,OAAO,CAACD,gBACfA,YAAY,MAAM,KAAKD,kBACvB,CAACC,YAAY,KAAK,CAChB,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SAG1D,MAAM,IAAIN,MACR,CAAC,8BAA8B,EAAEE,gBAAgB,KAAK,CAAC,qBAAqB,EAAEC,eAAe,qBAAqB,EAAEF,KAAK,SAAS,CAChIG,cACC;IAIP,MAAM,EAAEI,KAAK,EAAEC,YAAY,EAAEC,QAAQ,EAAE,GAAGR;IAC1C,IAAI,CAAC;QAAC;QAAS;QAAS;KAAO,CAAC,QAAQ,CAACQ,WACvC,MAAM,IAAIV,MAAM,CAAC,oCAAoC,EAAEU,UAAU;IAEnE,IAAIF,AAAU,SAAVA,SAAkBA,AAAU,SAAVA,OACpB,MAAM,IAAIR,MACR,CAAC,iCAAiC,EAAEC,KAAK,SAAS,CAACO,QAAQ;IAG/D,IACEC,AAAiBE,WAAjBF,gBACC,EAACF,OAAO,QAAQ,CAACE,iBAAiBA,gBAAgB,IAEnD,MAAM,IAAIT,MACR,CAAC,wDAAwD,EAAES,cAAc;AAG/E;AAEA,SAASG,wBACPb,MAAyB,EACzBc,IAAY,EACZC,IAAY;IAEZ,MAAMC,sBAAsBhB,OAAO,eAAe;IAClD,MAAMU,eAAeM,oBAAoB,YAAY;IACrD,IAAIN,AAAiBE,WAAjBF,cACF,OAAOV,OAAO,WAAW,CAAC,GAAG,CAAC,IAAMU;IAGtC,IAAIM,AAA8B,WAA9BA,oBAAoB,KAAK,EAC3B,OAAOA,AAA8B,SAA9BA,oBAAoB,KAAK,GAC5B;QAACD;QAAMD;QAAMC;QAAMD;KAAK,GACxB;QAACA;QAAMC;QAAMD;QAAMC;KAAK;IAG9B,OAAOC,AAA8B,SAA9BA,oBAAoB,KAAK,GAAY;QAACD;QAAMD;KAAK,GAAG;QAACA;QAAMC;KAAK;AACzE;AAEO,SAASE,0CACdjB,MAAyB,EACzBc,IAAY,EACZC,IAAY;IAEZ,MAAMZ,kBAAkBH,OAAO,eAAe;IAC9C,MAAM,EAAEU,YAAY,EAAE,GAAGP;IACzB,MAAMe,SAASL,wBAAwBb,QAAQc,MAAMC;IACrD,MAAMI,aAAanB,OAAO,WAAW,CAAC,IAAI,CACxC,CAACO,OAAOa,QAAUb,QAAQ,KAAKA,QAAQW,MAAM,CAACE,MAAM;IAGtD,IAAID,YAAY;QACd,MAAME,SACJX,AAAiBE,WAAjBF,eACI,CAAC,qBAAqB,EAAEA,aAAa,CAAC,CAAC,GACvC,CAAC,eAAe,EAAEI,KAAK,MAAM,EAAEC,KAAK,CAAC,CAAC;QAC5C,MAAMO,iBACJZ,AAAiBE,WAAjBF,eAA6B,CAAC,cAAc,EAAEA,cAAc,GAAG;QACjE,MAAM,IAAIT,MACR,CAAC,0BAA0B,EAAEC,KAAK,SAAS,CACzCF,OAAO,WAAW,EAClB,QAAQ,EAAEqB,OAAO,QAAQ,EACzBlB,gBAAgB,KAAK,CACtB,OAAO,EAAEA,gBAAgB,KAAK,GAAGmB,eAAe,QAAQ,EAAEpB,KAAK,SAAS,CACvEgB,SACC;IAEP;IAEA,IACEK,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwBvB,WACvBA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAE,IAC5CA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,GAE9C,MAAM,IAAIC,MACR,CAAC,sCAAsC,EAAEC,KAAK,SAAS,CAACF,OAAO,WAAW,GAAG;AAGnF"}
+{"version":3,"file":"ai-model/shared/model-locate-result/validation.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/shared/model-locate-result/validation.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { isBboxLocateResultValue } from './types';\nimport type { LocateResultValue } from './types';\n\nexport function assertLocateResultStructure(result: LocateResultValue): void {\n  if (!result || typeof result !== 'object') {\n    throw new Error(\n      `invalid parsed locate result: expected object, got ${JSON.stringify(\n        result,\n      )}`,\n    );\n  }\n\n  const coordinatesMeta = result.coordinatesMeta;\n  const expectedLength =\n    coordinatesMeta?.shape === 'bbox'\n      ? 4\n      : coordinatesMeta?.shape === 'point'\n        ? 2\n        : 0;\n  if (!expectedLength) {\n    throw new Error(\n      `invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(\n        coordinatesMeta?.shape,\n      )}`,\n    );\n  }\n\n  const coordinates = result.coordinates;\n  if (\n    !Array.isArray(coordinates) ||\n    coordinates.length !== expectedLength ||\n    !coordinates.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(\n      `invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(\n        coordinates,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    result.coordinatesMeta.boundsType !== undefined &&\n    !['inclusive', 'edges'].includes(result.coordinatesMeta.boundsType)\n  ) {\n    throw new Error(\n      `invalid locate bbox boundsType: ${result.coordinatesMeta.boundsType}`,\n    );\n  }\n  const { order, normalizedBy, rounding } = coordinatesMeta;\n  if (!['round', 'trunc', 'none'].includes(rounding)) {\n    throw new Error(`invalid locate coordinate rounding: ${rounding}`);\n  }\n  if (order !== 'xy' && order !== 'yx') {\n    throw new Error(\n      `invalid locate coordinate order: ${JSON.stringify(order)}`,\n    );\n  }\n  if (\n    normalizedBy !== undefined &&\n    (!Number.isFinite(normalizedBy) || normalizedBy <= 0)\n  ) {\n    throw new Error(\n      `locate result normalizedBy must be positive and finite: ${normalizedBy}`,\n    );\n  }\n}\n\nfunction resolveCoordinateLimits(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n): number[] {\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  if (normalizedBy !== undefined) {\n    return result.coordinates.map(() => normalizedBy);\n  }\n\n  if (resolvedCoordinates.shape === 'bbox') {\n    return resolvedCoordinates.order === 'yx'\n      ? [maxY, maxX, maxY, maxX]\n      : [maxX, maxY, maxX, maxY];\n  }\n\n  return resolvedCoordinates.order === 'yx' ? [maxY, maxX] : [maxX, maxY];\n}\n\nexport function assertLocateResultCoordinateRangeAndOrder(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n) {\n  const coordinatesMeta = result.coordinatesMeta;\n  const { normalizedBy } = coordinatesMeta;\n  const limits = resolveCoordinateLimits(result, maxX, maxY);\n  const outOfRange = result.coordinates.some(\n    (value, index) => value < 0 || value > limits[index],\n  );\n\n  if (outOfRange) {\n    const source =\n      normalizedBy !== undefined\n        ? `normalized range [0, ${normalizedBy}]`\n        : `image size [0, ${maxX}]x[0, ${maxY}]`;\n    const normalizedInfo =\n      normalizedBy !== undefined ? ` normalizedBy=${normalizedBy}` : '';\n    throw new Error(\n      `locate result coordinates ${JSON.stringify(\n        result.coordinates,\n      )} exceed ${source}. shape=${\n        coordinatesMeta.shape\n      } order=${coordinatesMeta.order}${normalizedInfo} limits=${JSON.stringify(\n        limits,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    (result.coordinates[2] < result.coordinates[0] ||\n      result.coordinates[3] < result.coordinates[1])\n  ) {\n    throw new Error(\n      `invalid locate bbox coordinate order: ${JSON.stringify(result.coordinates)}`,\n    );\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","assertLocateResultStructure","result","Error","JSON","coordinatesMeta","expectedLength","coordinates","Array","value","Number","isBboxLocateResultValue","undefined","order","normalizedBy","rounding","resolveCoordinateLimits","maxX","maxY","resolvedCoordinates","assertLocateResultCoordinateRangeAndOrder","limits","outOfRange","index","source","normalizedInfo"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACHO,SAASI,4BAA4BC,MAAyB;IACnE,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,QACpB,MAAM,IAAIC,MACR,CAAC,mDAAmD,EAAEC,KAAK,SAAS,CAClEF,SACC;IAIP,MAAMG,kBAAkBH,OAAO,eAAe;IAC9C,MAAMI,iBACJD,iBAAiB,UAAU,SACvB,IACAA,iBAAiB,UAAU,UACzB,IACA;IACR,IAAI,CAACC,gBACH,MAAM,IAAIH,MACR,CAAC,gEAAgE,EAAEC,KAAK,SAAS,CAC/EC,iBAAiB,QAChB;IAIP,MAAME,cAAcL,OAAO,WAAW;IACtC,IACE,CAACM,MAAM,OAAO,CAACD,gBACfA,YAAY,MAAM,KAAKD,kBACvB,CAACC,YAAY,KAAK,CAChB,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SAG1D,MAAM,IAAIN,MACR,CAAC,8BAA8B,EAAEE,gBAAgB,KAAK,CAAC,qBAAqB,EAAEC,eAAe,qBAAqB,EAAEF,KAAK,SAAS,CAChIG,cACC;IAIP,IACEI,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwBT,WACxBA,AAAsCU,WAAtCV,OAAO,eAAe,CAAC,UAAU,IACjC,CAAC;QAAC;QAAa;KAAQ,CAAC,QAAQ,CAACA,OAAO,eAAe,CAAC,UAAU,GAElE,MAAM,IAAIC,MACR,CAAC,gCAAgC,EAAED,OAAO,eAAe,CAAC,UAAU,EAAE;IAG1E,MAAM,EAAEW,KAAK,EAAEC,YAAY,EAAEC,QAAQ,EAAE,GAAGV;IAC1C,IAAI,CAAC;QAAC;QAAS;QAAS;KAAO,CAAC,QAAQ,CAACU,WACvC,MAAM,IAAIZ,MAAM,CAAC,oCAAoC,EAAEY,UAAU;IAEnE,IAAIF,AAAU,SAAVA,SAAkBA,AAAU,SAAVA,OACpB,MAAM,IAAIV,MACR,CAAC,iCAAiC,EAAEC,KAAK,SAAS,CAACS,QAAQ;IAG/D,IACEC,AAAiBF,WAAjBE,gBACC,EAACJ,OAAO,QAAQ,CAACI,iBAAiBA,gBAAgB,IAEnD,MAAM,IAAIX,MACR,CAAC,wDAAwD,EAAEW,cAAc;AAG/E;AAEA,SAASE,wBACPd,MAAyB,EACzBe,IAAY,EACZC,IAAY;IAEZ,MAAMC,sBAAsBjB,OAAO,eAAe;IAClD,MAAMY,eAAeK,oBAAoB,YAAY;IACrD,IAAIL,AAAiBF,WAAjBE,cACF,OAAOZ,OAAO,WAAW,CAAC,GAAG,CAAC,IAAMY;IAGtC,IAAIK,AAA8B,WAA9BA,oBAAoB,KAAK,EAC3B,OAAOA,AAA8B,SAA9BA,oBAAoB,KAAK,GAC5B;QAACD;QAAMD;QAAMC;QAAMD;KAAK,GACxB;QAACA;QAAMC;QAAMD;QAAMC;KAAK;IAG9B,OAAOC,AAA8B,SAA9BA,oBAAoB,KAAK,GAAY;QAACD;QAAMD;KAAK,GAAG;QAACA;QAAMC;KAAK;AACzE;AAEO,SAASE,0CACdlB,MAAyB,EACzBe,IAAY,EACZC,IAAY;IAEZ,MAAMb,kBAAkBH,OAAO,eAAe;IAC9C,MAAM,EAAEY,YAAY,EAAE,GAAGT;IACzB,MAAMgB,SAASL,wBAAwBd,QAAQe,MAAMC;IACrD,MAAMI,aAAapB,OAAO,WAAW,CAAC,IAAI,CACxC,CAACO,OAAOc,QAAUd,QAAQ,KAAKA,QAAQY,MAAM,CAACE,MAAM;IAGtD,IAAID,YAAY;QACd,MAAME,SACJV,AAAiBF,WAAjBE,eACI,CAAC,qBAAqB,EAAEA,aAAa,CAAC,CAAC,GACvC,CAAC,eAAe,EAAEG,KAAK,MAAM,EAAEC,KAAK,CAAC,CAAC;QAC5C,MAAMO,iBACJX,AAAiBF,WAAjBE,eAA6B,CAAC,cAAc,EAAEA,cAAc,GAAG;QACjE,MAAM,IAAIX,MACR,CAAC,0BAA0B,EAAEC,KAAK,SAAS,CACzCF,OAAO,WAAW,EAClB,QAAQ,EAAEsB,OAAO,QAAQ,EACzBnB,gBAAgB,KAAK,CACtB,OAAO,EAAEA,gBAAgB,KAAK,GAAGoB,eAAe,QAAQ,EAAErB,KAAK,SAAS,CACvEiB,SACC;IAEP;IAEA,IACEV,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwBT,WACvBA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAE,IAC5CA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,GAE9C,MAAM,IAAIC,MACR,CAAC,sCAAsC,EAAEC,KAAK,SAAS,CAACF,OAAO,WAAW,GAAG;AAGnF"}

package/dist/lib/ai-model/workflows/grounding/locate.js

@@ -118,7 +118,7 @@ async function genericLocate(locateRequest) {
     });
     try {
         return await (0, semantic_retry_js_namespaceObject.callAiAndParseWithRetry)({
-            callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
+            callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callGenerativeAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
                     abortSignal: options.abortSignal,
                     expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,
                     semanticRetryAttempt: retryAttempt

package/dist/lib/ai-model/workflows/grounding/locate.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACaA,MAAMI,iBAAiBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,AAAAA,IAAAA,gDAAAA,mCAAAA,AAAAA,EACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,IAAAA,sBAAAA,MAAAA,AAAAA,EACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,AAAAA,IAAAA,yBAAAA,8BAAAA,AAAAA,EAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,iDAAAA,oBAAoBA,CAC7BD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACgBA,MAAMI,iBAAiBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,AAAAA,IAAAA,gDAAAA,mCAAAA,AAAAA,EACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,IAAAA,sBAAAA,MAAAA,AAAAA,EACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,AAAAA,IAAAA,yBAAAA,8BAAAA,AAAAA,EAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,iDAAAA,oBAAoBA,CAC7BD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}

package/dist/lib/ai-model/workflows/grounding/section-locate.js

@@ -64,7 +64,7 @@ async function AiLocateSection(options) {
     let parsedResult;
     try {
         parsedResult = await (0, semantic_retry_js_namespaceObject.callAiAndParseWithRetry)({
-            callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
+            callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callGenerativeAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
                     abortSignal: options.abortSignal,
                     expectedJsonObjectResponse: searchAreaProtocol.expectedJsonObjectResponse,
                     semanticRetryAttempt: retryAttempt

package/dist/lib/ai-model/workflows/grounding/section-locate.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/section-locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACeA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,IAAAA,sBAAAA,MAAAA,AAAAA,EACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,AAAAA,IAAAA,yBAAAA,iCAAAA,AAAAA,EAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,AAAAA,IAAAA,wCAAAA,sBAAAA,AAAAA,EAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,iDAAAA,oBAAoBA,CAC7BD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,AAAAA,IAAAA,wCAAAA,gBAAAA,AAAAA,EAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,AAAAA,IAAAA,wCAAAA,qBAAAA,AAAAA,EAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/section-locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACkBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,IAAAA,sBAAAA,MAAAA,AAAAA,EACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,AAAAA,IAAAA,yBAAAA,iCAAAA,AAAAA,EAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,AAAAA,IAAAA,wCAAAA,sBAAAA,AAAAA,EAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,iDAAAA,oBAAoBA,CAC7BD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,AAAAA,IAAAA,wCAAAA,gBAAAA,AAAAA,EAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,AAAAA,IAAAA,wCAAAA,qBAAAA,AAAAA,EAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}

package/dist/lib/ai-model/workflows/insight/boolean.js

@@ -31,7 +31,7 @@ async function extractBoolean(options) {
     const { booleanFn } = options.modelRuntime.adapter.insight;
     (0, utils_namespaceObject.assert)(booleanFn, 'A boolean adapter is required');
     const keys = 'object' == typeof options.dataQuery ? Object.keys(options.dataQuery) : [];
-    (0, utils_namespaceObject.assert)(1 === keys.length && ('Boolean' === keys[0] || 'StatementIsTruthy' === keys[0]), 'This model only supports boolean insight tasks. Use the default model for extraction.');
+    (0, utils_namespaceObject.assert)(1 === keys.length, 'Boolean insight tasks require exactly one result field.');
     const result = await booleanFn(options);
     return {
         parseResult: {

package/dist/lib/ai-model/workflows/insight/boolean.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/boolean.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/boolean.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIDataExtractionResponse,\n  AIUsageInfo,\n  RawAssistantOutput,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport type { ExtractOptions } from './extraction';\n\nexport type BooleanFn = (options: ExtractOptions) => Promise<{\n  value: boolean;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}>;\n\nexport async function extractBoolean<T>(options: ExtractOptions): Promise<{\n  parseResult: AIDataExtractionResponse<T>;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  rawAssistantOutput?: RawAssistantOutput;\n  reasoning_content?: string;\n}> {\n  const { booleanFn } = options.modelRuntime.adapter.insight;\n  assert(booleanFn, 'A boolean adapter is required');\n  const keys =\n    typeof options.dataQuery === 'object' ? Object.keys(options.dataQuery) : [];\n  assert(\n    keys.length === 1 &&\n      (keys[0] === 'Boolean' || keys[0] === 'StatementIsTruthy'),\n    'This model only supports boolean insight tasks. Use the default model for extraction.',\n  );\n  const result = await booleanFn(options);\n  return {\n    parseResult: {\n      data: { [keys[0]]: result.value } as T,\n    },\n    rawResponse: result.rawResponse,\n    usage: result.usage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","extractBoolean","options","booleanFn","assert","keys","result"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACQO,eAAeI,eAAkBC,OAAuB;IAO7D,MAAM,EAAEC,SAAS,EAAE,GAAGD,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO;IAC1DE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;IAClB,MAAME,OACJ,AAA6B,YAA7B,OAAOH,QAAQ,SAAS,GAAgBL,OAAO,IAAI,CAACK,QAAQ,SAAS,IAAI,EAAE;IAC7EE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,AAAgB,MAAhBA,KAAK,MAAM,IACRA,CAAAA,AAAY,cAAZA,IAAI,CAAC,EAAE,IAAkBA,AAAY,wBAAZA,IAAI,CAAC,EAAE,AAAuB,GAC1D;IAEF,MAAMC,SAAS,MAAMH,UAAUD;IAC/B,OAAO;QACL,aAAa;YACX,MAAM;gBAAE,CAACG,IAAI,CAAC,EAAE,CAAC,EAAEC,OAAO,KAAK;YAAC;QAClC;QACA,aAAaA,OAAO,WAAW;QAC/B,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/boolean.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/boolean.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIDataExtractionResponse,\n  AIUsageInfo,\n  RawAssistantOutput,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport type { ExtractOptions } from './extraction';\n\nexport type BooleanFn = (options: ExtractOptions) => Promise<{\n  value: boolean;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}>;\n\nexport async function extractBoolean<T>(options: ExtractOptions): Promise<{\n  parseResult: AIDataExtractionResponse<T>;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  rawAssistantOutput?: RawAssistantOutput;\n  reasoning_content?: string;\n}> {\n  const { booleanFn } = options.modelRuntime.adapter.insight;\n  assert(booleanFn, 'A boolean adapter is required');\n  const keys =\n    typeof options.dataQuery === 'object' ? Object.keys(options.dataQuery) : [];\n  assert(\n    keys.length === 1,\n    'Boolean insight tasks require exactly one result field.',\n  );\n  const result = await booleanFn(options);\n  return {\n    parseResult: {\n      data: { [keys[0]]: result.value } as T,\n    },\n    rawResponse: result.rawResponse,\n    usage: result.usage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","extractBoolean","options","booleanFn","assert","keys","result"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACQO,eAAeI,eAAkBC,OAAuB;IAO7D,MAAM,EAAEC,SAAS,EAAE,GAAGD,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO;IAC1DE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;IAClB,MAAME,OACJ,AAA6B,YAA7B,OAAOH,QAAQ,SAAS,GAAgBL,OAAO,IAAI,CAACK,QAAQ,SAAS,IAAI,EAAE;IAC7EE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,AAAgB,MAAhBA,KAAK,MAAM,EACX;IAEF,MAAMC,SAAS,MAAMH,UAAUD;IAC/B,OAAO;QACL,aAAa;YACX,MAAM;gBAAE,CAACG,IAAI,CAAC,EAAE,CAAC,EAAEC,OAAO,KAAK;YAAC;QAClC;QACA,aAAaA,OAAO,WAAW;QAC/B,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/lib/ai-model/workflows/insight/extraction.js

@@ -36,7 +36,7 @@ const external_boolean_js_namespaceObject = require("./boolean.js");
 const external_insight_response_parser_js_namespaceObject = require("./insight-response-parser.js");
 const debugInsight = (0, logger_namespaceObject.getDebug)('ai:insight');
 async function AiExtractElementInfo(options) {
-    if (options.modelRuntime.adapter.insight.booleanFn) return (0, external_boolean_js_namespaceObject.extractBoolean)(options);
+    if ('boolean' === options.taskType && options.modelRuntime.adapter.insight.booleanFn) return (0, external_boolean_js_namespaceObject.extractBoolean)(options);
     const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } = options;
     const insightProtocol = modelRuntime.adapter.insight.protocol;
     const systemPrompt = (0, index_js_namespaceObject.buildInsightSystemPrompt)({
@@ -95,7 +95,7 @@ async function AiExtractElementInfo(options) {
         msgs.push(...addOns);
     }
     return (0, semantic_retry_js_namespaceObject.callAiAndParseWithRetry)({
-        callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(msgs, previousParseError), modelRuntime, {
+        callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callGenerativeAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(msgs, previousParseError), modelRuntime, {
                 abortSignal: options.abortSignal,
                 semanticRetryAttempt: retryAttempt
             }),

package/dist/lib/ai-model/workflows/insight/extraction.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { extractBoolean } from './boolean';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport interface ExtractOptions {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}\n\nexport async function AiExtractElementInfo<T>(options: ExtractOptions) {\n  if (options.modelRuntime.adapter.insight.booleanFn) {\n    return extractBoolean<T>(options);\n  }\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInsight","getDebug","AiExtractElementInfo","options","extractBoolean","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACkBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAYvB,eAAeC,qBAAwBC,OAAuB;IACnE,IAAIA,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAChD,OAAOC,AAAAA,IAAAA,oCAAAA,cAAAA,AAAAA,EAAkBD;IAE3B,MAAM,EAAEE,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEN;IACF,MAAMO,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAC5Bd,QAAQ,eAAe,IAAI,IAC3BE,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,AAAAA,IAAAA,qCAAAA,0BAAAA,AAAAA,EAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaN,QAAQ,WAAW;gBAChC,sBAAsBuB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,AAAAA,IAAAA,oDAAAA,oBAAAA,AAAAA,EAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaN,QAAQ,WAAW;QAChC,cAAc,CAACuC;YACb1C,aACE,iDACA0C,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { extractBoolean } from './boolean';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport interface ExtractOptions {\n  /** Explicit task type; never infer from query keys. */\n  taskType: 'boolean' | 'extract';\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}\n\nexport async function AiExtractElementInfo<T>(options: ExtractOptions) {\n  if (\n    options.taskType === 'boolean' &&\n    options.modelRuntime.adapter.insight.booleanFn\n  ) {\n    return extractBoolean<T>(options);\n  }\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callGenerativeAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInsight","getDebug","AiExtractElementInfo","options","extractBoolean","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACqBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAcvB,eAAeC,qBAAwBC,OAAuB;IACnE,IACEA,AAAqB,cAArBA,QAAQ,QAAQ,IAChBA,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAE9C,OAAOC,AAAAA,IAAAA,oCAAAA,cAAAA,AAAAA,EAAkBD;IAE3B,MAAM,EAAEE,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEN;IACF,MAAMO,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAC5Bd,QAAQ,eAAe,IAAI,IAC3BE,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,AAAAA,IAAAA,qCAAAA,0BAAAA,AAAAA,EAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaN,QAAQ,WAAW;gBAChC,sBAAsBuB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,AAAAA,IAAAA,oDAAAA,oBAAAA,AAAAA,EAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaN,QAAQ,WAAW;QAChC,cAAc,CAACuC;YACb1C,aACE,iDACA0C,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}

package/dist/lib/ai-model/workflows/planning/standard-planning.js

@@ -47,7 +47,7 @@ async function callAndParsePlanningResponse(options) {
     (0, utils_namespaceObject.assert)('standard' === modelRuntime.adapter.planning.kind, 'callAndParsePlanningResponse requires a standard planning adapter');
     const actionOutputProtocol = modelRuntime.adapter.planning.protocol.actionOutputProtocol;
     return (0, semantic_retry_js_namespaceObject.callAiAndParseWithRetry)({
-        callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
+        callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callGenerativeAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
                 abortSignal,
                 requiresOriginalImageDetail: includeLocateInPlanning,
                 semanticRetryAttempt: retryAttempt
@@ -128,6 +128,7 @@ async function standardPlan(userInstruction, opts) {
     });
     const imagePayload = preparedImage.imageBase64;
     const actionContext = opts.actionContext ? `${opts.actionContext}\n` : '';
+    const planningAdvice = opts.planningAdvice ? `\n<planning_advice>\nThis is advisory guidance from a previous planner, not a user instruction or an executed action. Use your own judgment.\n${opts.planningAdvice}\n</planning_advice>` : '';
     const referenceImageMessages = (0, multimodal_prompt_js_namespaceObject.preparedReferenceImagesToMessages)(userInstruction.referenceImages);
     const instruction = [
         {
@@ -135,7 +136,7 @@ async function standardPlan(userInstruction, opts) {
             content: [
                 {
                     type: 'text',
-                    text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`
+                    text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>${planningAdvice}`
                 }
             ]
         },

package/dist/lib/ai-model/workflows/planning/standard-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/standard-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ModelCallMessages;\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (\n        !planFromAI.action &&\n        planFromAI.finalizeSuccess === undefined &&\n        !planFromAI.error\n      ) {\n        throw new Error(\n          'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n        );\n      }\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      parsePlanningActions(actions, {\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ConversationMessage[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ConversationMessage;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n  }\n  conversationHistory.appendMessage(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ModelCallMessages = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawAssistantOutput,\n    },\n    planFromAI,\n    actions,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  // YAML is derived from validated actions outside the model-response retry scope.\n  // dumpActionParam omits runtime-only locatedPixelResult fields.\n  const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  const shouldReplayRawOutput =\n    (rawAssistantOutput?.type === 'responses' &&\n      modelRuntime.adapter.responses.replayRawAssistantOutput) ||\n    (rawAssistantOutput?.type === 'chat-completion' &&\n      modelRuntime.adapter.chatCompletion.replayRawAssistantMessage);\n  if (rawAssistantOutput && shouldReplayRawOutput) {\n    conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else {\n    if (!rawAssistantOutput) {\n      debug(\n        'Raw assistant output is unavailable; saving only response text in conversation history.',\n      );\n    }\n    conversationHistory.appendMessage({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawAssistantOutput","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index","shouldReplayRawOutput"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;ACwBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,IAAAA,sBAAAA,MAAAA,AAAAA,EACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,AAAAA,IAAAA,qDAAAA,6BAAAA,AAAAA,EAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,IAAAA,mDAAAA,oBAAAA,AAAAA,EAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEU;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,IAAAA,sBAAAA,MAAAA,AAAAA,EACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,AAAAA,IAAAA,0BAAAA,2CAAAA,AAAAA,EAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,AAAAA,IAAAA,0BAAAA,2CAAAA,AAAAA,EAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,AAAAA,IAAAA,yBAAAA,iCAAAA,AAAAA,EAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,AAAAA,IAAAA,qCAAAA,iCAAAA,AAAAA,EAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAAqC;QACzC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,KAAKP;gBACP;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,KAAKP;YACP;SACD;IACH;IAEFX,oBAAoB,aAAa,CAACgB;IAGlChB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAA0B;QAC9B;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAGpD,MAAM8C,wBACHN,oBAAoB,SAAS,eAC5BxD,aAAa,OAAO,CAAC,SAAS,CAAC,wBAAwB,IACxDwD,oBAAoB,SAAS,qBAC5BxD,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB;IACjE,IAAIwD,sBAAsBM,uBACxBhC,oBAAoB,iBAAiB,CAAC0B;SACjC;QACL,IAAI,CAACA,oBACH/D,MACE;QAGJqC,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMuB;gBACR;aACD;QACH;IACF;IAEA,OAAOO;AACT"}
+{"version":3,"file":"ai-model/workflows/planning/standard-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ModelCallMessages;\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callGenerativeAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (\n        !planFromAI.action &&\n        planFromAI.finalizeSuccess === undefined &&\n        !planFromAI.error\n      ) {\n        throw new Error(\n          'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n        );\n      }\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      parsePlanningActions(actions, {\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n  const planningAdvice = opts.planningAdvice\n    ? `\\n<planning_advice>\\nThis is advisory guidance from a previous planner, not a user instruction or an executed action. Use your own judgment.\\n${opts.planningAdvice}\\n</planning_advice>`\n    : '';\n\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ConversationMessage[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>${planningAdvice}`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ConversationMessage;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n  }\n  conversationHistory.appendMessage(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ModelCallMessages = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawAssistantOutput,\n    },\n    planFromAI,\n    actions,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  // YAML is derived from validated actions outside the model-response retry scope.\n  // dumpActionParam omits runtime-only locatedPixelResult fields.\n  const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  const shouldReplayRawOutput =\n    (rawAssistantOutput?.type === 'responses' &&\n      modelRuntime.adapter.responses.replayRawAssistantOutput) ||\n    (rawAssistantOutput?.type === 'chat-completion' &&\n      modelRuntime.adapter.chatCompletion.replayRawAssistantMessage);\n  if (rawAssistantOutput && shouldReplayRawOutput) {\n    conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else {\n    if (!rawAssistantOutput) {\n      debug(\n        'Raw assistant output is unavailable; saving only response text in conversation history.',\n      );\n    }\n    conversationHistory.appendMessage({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","planningAdvice","referenceImageMessages","preparedReferenceImagesToMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawAssistantOutput","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index","shouldReplayRawOutput"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;AC2BA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,IAAAA,sBAAAA,MAAAA,AAAAA,EACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,AAAAA,IAAAA,qDAAAA,6BAAAA,AAAAA,EAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,IAAAA,mDAAAA,oBAAAA,AAAAA,EAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEU;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,IAAAA,sBAAAA,MAAAA,AAAAA,EACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,AAAAA,IAAAA,0BAAAA,2CAAAA,AAAAA,EAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,AAAAA,IAAAA,0BAAAA,2CAAAA,AAAAA,EAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,AAAAA,IAAAA,yBAAAA,iCAAAA,AAAAA,EAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IACvE,MAAMe,iBAAiBf,KAAK,cAAc,GACtC,CAAC,8IAA8I,EAAEA,KAAK,cAAc,CAAC,oBAAoB,CAAC,GAC1L;IAEJ,MAAMgB,yBAAyBC,AAAAA,IAAAA,qCAAAA,iCAAAA,AAAAA,EAC7BlB,gBAAgB,eAAe;IAEjC,MAAMmB,cAAqC;QACzC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGJ,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,EAAEgB,gBAAgB;gBACvG;aACD;QACH;WACGC;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBZ,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMmB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BlB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMsD,eAAepB,oBAAoB,cAAc;IACvD,MAAMqB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIpB,oBAAoB,sBAAsB,EAAE;QAC9CiB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGjB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEqB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,KAAKR;gBACP;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEiB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,KAAKR;YACP;SACD;IACH;IAEFX,oBAAoB,aAAa,CAACiB;IAGlCjB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMsB,aAAatB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMyB,OAA0B;QAC9B;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,EACDzC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUwD;QACVrD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMmD,WAAWC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBvC,SAASQ,KAAK,WAAW;IAEjE,IAAIgC,yBAAyB;IAG7B,IAAI5C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNmE,yBAAyB;QAEzB,IAAIxB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM+B,cAAkC;QACtC,GAAG7C,UAAU;QACbI;QACAkC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEApD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM8C,SAAS9C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAACgC;QAI5C,IAAI9C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAGpD,MAAM+C,wBACHN,oBAAoB,SAAS,eAC5BzD,aAAa,OAAO,CAAC,SAAS,CAAC,wBAAwB,IACxDyD,oBAAoB,SAAS,qBAC5BzD,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB;IACjE,IAAIyD,sBAAsBM,uBACxBjC,oBAAoB,iBAAiB,CAAC2B;SACjC;QACL,IAAI,CAACA,oBACHhE,MACE;QAGJqC,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMwB;gBACR;aACD;QACH;IACF;IAEA,OAAOO;AACT"}

package/dist/lib/ai-model/workflows/recorder-generation/playwright.js

@@ -141,7 +141,7 @@ ${constants_namespaceObject.PLAYWRIGHT_EXAMPLE_CODE}`;
             content: messageContent
         }
     ];
-    if (options.stream && options.onChunk) return await (0, external_service_caller_index_js_namespaceObject.callAI)(prompt, modelRuntime, {
+    if (options.stream && options.onChunk) return await (0, external_service_caller_index_js_namespaceObject.callGenerativeAI)(prompt, modelRuntime, {
         stream: true,
         onChunk: options.onChunk
     });