@midscene/core 1.16.2 → 1.16.6-beta-20261023073257.0

package/dist/lib/ai-model/models/jev/image.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/jev/image.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/jev/image.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { Buffer } from 'node:buffer';\nimport {\n  imageInfoOfBase64,\n  padBase64ImageToMinimumSize,\n  parseBase64,\n  preProcessImageUrl,\n  resizeBase64ImageToJpeg,\n} from '@midscene/shared/img';\nimport { assert } from '@midscene/shared/utils';\nimport type { PixelLocateResult } from '../../shared/model-locate-result';\n\nexport async function prepareJevImage(url: string) {\n  const base64 = await preProcessImageUrl(url, true);\n  const source = await imageInfoOfBase64(base64);\n  const scale = Math.min(1, 1568 / source.width, 1568 / source.height);\n  const contentSize = {\n    width: Math.max(1, Math.round(source.width * scale)),\n    height: Math.max(1, Math.round(source.height * scale)),\n  };\n  const resized = await resizeBase64ImageToJpeg(base64, {\n    targetSize: contentSize,\n  });\n  const prepared = await padBase64ImageToMinimumSize(resized, 280);\n  assert(\n    Buffer.from(parseBase64(prepared.imageBase64).body, 'base64').length <=\n      16 * 1024 * 1024,\n    'Jev image exceeds 16 MiB',\n  );\n  return {\n    ...prepared,\n    contentSize,\n    sourceSize: { width: source.width, height: source.height },\n  };\n}\n\nexport function mapJevBox(\n  box: { left: number; top: number; right: number; bottom: number },\n  image: Awaited<ReturnType<typeof prepareJevImage>>,\n  targetSize: { width: number; height: number },\n): PixelLocateResult {\n  // Jev returns fractions of the padded image. Undo padding and scale before\n  // the grounding workflow maps a DeepLocate crop back to the full screenshot.\n  const left =\n    (box.left * image.width * targetSize.width) / image.contentSize.width;\n  const top =\n    (box.top * image.height * targetSize.height) / image.contentSize.height;\n  const right =\n    (box.right * image.width * targetSize.width) / image.contentSize.width;\n  const bottom =\n    (box.bottom * image.height * targetSize.height) / image.contentSize.height;\n  const center: [number, number] = [(left + right) / 2, (top + bottom) / 2];\n  assert(\n    center[0] >= 0 &&\n      center[0] < targetSize.width &&\n      center[1] >= 0 &&\n      center[1] < targetSize.height,\n    'Jev returned a target in image padding, outside the screenshot',\n  );\n  return {\n    center,\n    rect: { left, top, width: right - left, height: bottom - top },\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","prepareJevImage","url","base64","preProcessImageUrl","source","imageInfoOfBase64","scale","Math","contentSize","resized","resizeBase64ImageToJpeg","prepared","padBase64ImageToMinimumSize","assert","Buffer","parseBase64","mapJevBox","box","image","targetSize","left","top","right","bottom","center"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACKO,eAAeI,gBAAgBC,GAAW;IAC/C,MAAMC,SAAS,MAAMC,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EAAmBF,KAAK;IAC7C,MAAMG,SAAS,MAAMC,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EAAkBH;IACvC,MAAMI,QAAQC,KAAK,GAAG,CAAC,GAAG,OAAOH,OAAO,KAAK,EAAE,OAAOA,OAAO,MAAM;IACnE,MAAMI,cAAc;QAClB,OAAOD,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,OAAO,KAAK,GAAGE;QAC7C,QAAQC,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,OAAO,MAAM,GAAGE;IACjD;IACA,MAAMG,UAAU,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwBR,QAAQ;QACpD,YAAYM;IACd;IACA,MAAMG,WAAW,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BH,SAAS;IAC5DI,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,qCAAAA,MAAAA,CAAAA,IAAW,CAACC,AAAAA,IAAAA,oBAAAA,WAAAA,AAAAA,EAAYJ,SAAS,WAAW,EAAE,IAAI,EAAE,UAAU,MAAM,IAClE,UACF;IAEF,OAAO;QACL,GAAGA,QAAQ;QACXH;QACA,YAAY;YAAE,OAAOJ,OAAO,KAAK;YAAE,QAAQA,OAAO,MAAM;QAAC;IAC3D;AACF;AAEO,SAASY,UACdC,GAAiE,EACjEC,KAAkD,EAClDC,UAA6C;IAI7C,MAAMC,OACHH,IAAI,IAAI,GAAGC,MAAM,KAAK,GAAGC,WAAW,KAAK,GAAID,MAAM,WAAW,CAAC,KAAK;IACvE,MAAMG,MACHJ,IAAI,GAAG,GAAGC,MAAM,MAAM,GAAGC,WAAW,MAAM,GAAID,MAAM,WAAW,CAAC,MAAM;IACzE,MAAMI,QACHL,IAAI,KAAK,GAAGC,MAAM,KAAK,GAAGC,WAAW,KAAK,GAAID,MAAM,WAAW,CAAC,KAAK;IACxE,MAAMK,SACHN,IAAI,MAAM,GAAGC,MAAM,MAAM,GAAGC,WAAW,MAAM,GAAID,MAAM,WAAW,CAAC,MAAM;IAC5E,MAAMM,SAA2B;QAAEJ,CAAAA,OAAOE,KAAI,IAAK;QAAID,CAAAA,MAAME,MAAK,IAAK;KAAE;IACzEV,IAAAA,sBAAAA,MAAAA,AAAAA,EACEW,MAAM,CAAC,EAAE,IAAI,KACXA,MAAM,CAAC,EAAE,GAAGL,WAAW,KAAK,IAC5BK,MAAM,CAAC,EAAE,IAAI,KACbA,MAAM,CAAC,EAAE,GAAGL,WAAW,MAAM,EAC/B;IAEF,OAAO;QACLK;QACA,MAAM;YAAEJ;YAAMC;YAAK,OAAOC,QAAQF;YAAM,QAAQG,SAASF;QAAI;IAC/D;AACF"}
+{"version":3,"file":"ai-model/models/jev/image.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/jev/image.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { Buffer } from 'node:buffer';\nimport { parseBase64, preProcessImageUrl } from '@midscene/shared/img';\nimport { assert } from '@midscene/shared/utils';\nimport { createLocateResultCodec } from '../../shared/model-locate-result';\nimport type { PixelLocateResult } from '../../shared/model-locate-result';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\n\nexport async function prepareJevImage(url: string) {\n  const base64 = await preProcessImageUrl(url, true);\n  assert(\n    Buffer.from(parseBase64(base64).body, 'base64').length <= 16 * 1024 * 1024,\n    'Jev image exceeds 16 MiB',\n  );\n  return { imageBase64: base64 };\n}\n\nconst jevBoxCodec = createLocateResultCodec({\n  coordinates: {\n    shape: 'bbox',\n    normalizedBy: 1,\n    rounding: 'none',\n    boundsType: 'edges',\n  },\n});\n\nexport function mapJevBox(\n  box: { left: number; top: number; right: number; bottom: number },\n  targetSize: { width: number; height: number },\n): PixelLocateResult {\n  return jevBoxCodec.toPixelResult([box.left, box.top, box.right, box.bottom], {\n    preparedSize: targetSize,\n  });\n}\n\nexport async function prepareReferences(\n  prompt: Parameters<typeof prepareUserPrompt>[0],\n) {\n  const { referenceImages } = await prepareUserPrompt(prompt);\n  const prepared = await Promise.all(\n    referenceImages.map(async (image, index) => ({\n      key: `reference_${index}`,\n      name: image.name,\n      image: await prepareJevImage(image.url),\n    })),\n  );\n  return {\n    references: prepared.map(({ key, name }) => ({\n      image: `\\`${key}\\``,\n      name,\n    })),\n    images: Object.fromEntries(\n      prepared.map(({ key, image }) => [key, image.imageBase64]),\n    ),\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","prepareJevImage","url","base64","preProcessImageUrl","assert","Buffer","parseBase64","jevBoxCodec","createLocateResultCodec","mapJevBox","box","targetSize","prepareReferences","prompt","referenceImages","prepareUserPrompt","prepared","Promise","image","index","name"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACCO,eAAeI,gBAAgBC,GAAW;IAC/C,MAAMC,SAAS,MAAMC,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EAAmBF,KAAK;IAC7CG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,qCAAAA,MAAAA,CAAAA,IAAW,CAACC,AAAAA,IAAAA,oBAAAA,WAAAA,AAAAA,EAAYJ,QAAQ,IAAI,EAAE,UAAU,MAAM,IAAI,UAC1D;IAEF,OAAO;QAAE,aAAaA;IAAO;AAC/B;AAEA,MAAMK,cAAcC,AAAAA,IAAAA,yBAAAA,uBAAAA,AAAAA,EAAwB;IAC1C,aAAa;QACX,OAAO;QACP,cAAc;QACd,UAAU;QACV,YAAY;IACd;AACF;AAEO,SAASC,UACdC,GAAiE,EACjEC,UAA6C;IAE7C,OAAOJ,YAAY,aAAa,CAAC;QAACG,IAAI,IAAI;QAAEA,IAAI,GAAG;QAAEA,IAAI,KAAK;QAAEA,IAAI,MAAM;KAAC,EAAE;QAC3E,cAAcC;IAChB;AACF;AAEO,eAAeC,kBACpBC,MAA+C;IAE/C,MAAM,EAAEC,eAAe,EAAE,GAAG,MAAMC,AAAAA,IAAAA,qCAAAA,iBAAAA,AAAAA,EAAkBF;IACpD,MAAMG,WAAW,MAAMC,QAAQ,GAAG,CAChCH,gBAAgB,GAAG,CAAC,OAAOI,OAAOC,QAAW;YAC3C,KAAK,CAAC,UAAU,EAAEA,OAAO;YACzB,MAAMD,MAAM,IAAI;YAChB,OAAO,MAAMlB,gBAAgBkB,MAAM,GAAG;QACxC;IAEF,OAAO;QACL,YAAYF,SAAS,GAAG,CAAC,CAAC,EAAErB,GAAG,EAAEyB,IAAI,EAAE,GAAM;gBAC3C,OAAO,CAAC,EAAE,EAAEzB,IAAI,EAAE,CAAC;gBACnByB;YACF;QACA,QAAQxB,OAAO,WAAW,CACxBoB,SAAS,GAAG,CAAC,CAAC,EAAErB,GAAG,EAAEuB,KAAK,EAAE,GAAK;gBAACvB;gBAAKuB,MAAM,WAAW;aAAC;IAE7D;AACF"}

package/dist/lib/ai-model/models/jev/locate.js

@@ -0,0 +1,82 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    jevLocate: ()=>jevLocate
+});
+const external_common_js_namespaceObject = require("../../../common.js");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const index_js_namespaceObject = require("../../service-caller/index.js");
+const schema_js_namespaceObject = require("../../service-caller/system-one/schema.js");
+const external_image_js_namespaceObject = require("./image.js");
+const jevLocate = async ({ targetElementDescription, locateImage, options })=>{
+    const image = await (0, external_image_js_namespaceObject.prepareJevImage)(locateImage.imageBase64);
+    const references = await (0, external_image_js_namespaceObject.prepareReferences)(targetElementDescription);
+    const target = (0, external_common_js_namespaceObject.userPromptToString)(targetElementDescription);
+    const response = await (0, index_js_namespaceObject.callSystemOneAI)({
+        state: {
+            target,
+            references: references.references
+        },
+        images: {
+            screenshot: image.imageBase64,
+            ...references.images
+        },
+        questions: {
+            present: {
+                type: 'noul',
+                instructions: 'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.'
+            },
+            target: {
+                type: 'box',
+                instructions: 'The single target described in `target`, in `screenshot`.'
+            }
+        }
+    }, options.modelRuntime, {
+        abortSignal: options.abortSignal
+    });
+    const { answers } = schema_js_namespaceObject.systemOneResponseSchema.parse(JSON.parse(response.content));
+    (0, utils_namespaceObject.assert)('noul' === answers.present.type && 'box' === answers.target.type, 'Invalid Jev locate answers');
+    return {
+        rawResponse: response.content,
+        usage: response.usage,
+        ...answers.present.noul >= 0.5 ? {
+            locatedPixelResult: (0, external_image_js_namespaceObject.mapJevBox)(answers.target.box, locateImage)
+        } : {
+            errors: [
+                'The target is not present in the screenshot'
+            ]
+        }
+    };
+};
+exports.jevLocate = __webpack_exports__.jevLocate;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "jevLocate"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=locate.js.map

package/dist/lib/ai-model/models/jev/locate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/jev/locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { userPromptToString } from '@/common';\nimport { assert } from '@midscene/shared/utils';\nimport { callSystemOneAI } from '../../service-caller';\nimport { systemOneResponseSchema } from '../../service-caller/system-one/schema';\nimport type { LocateFn } from '../../workflows/grounding/types';\nimport { mapJevBox, prepareJevImage, prepareReferences } from './image';\n\nexport const jevLocate: LocateFn = async ({\n  targetElementDescription,\n  locateImage,\n  options,\n}) => {\n  const image = await prepareJevImage(locateImage.imageBase64);\n  const references = await prepareReferences(targetElementDescription);\n  const target = userPromptToString(targetElementDescription);\n  const response = await callSystemOneAI(\n    {\n      state: { target, references: references.references },\n      images: { screenshot: image.imageBase64, ...references.images },\n      questions: {\n        present: {\n          type: 'noul',\n          instructions:\n            'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.',\n        },\n        target: {\n          type: 'box',\n          instructions:\n            'The single target described in `target`, in `screenshot`.',\n        },\n      },\n    },\n    options.modelRuntime,\n    { abortSignal: options.abortSignal },\n  );\n  const { answers } = systemOneResponseSchema.parse(\n    JSON.parse(response.content),\n  );\n  assert(\n    answers.present.type === 'noul' && answers.target.type === 'box',\n    'Invalid Jev locate answers',\n  );\n  return {\n    rawResponse: response.content,\n    usage: response.usage,\n    ...(answers.present.noul >= 0.5\n      ? {\n          locatedPixelResult: mapJevBox(answers.target.box, locateImage),\n        }\n      : { errors: ['The target is not present in the screenshot'] }),\n  };\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","jevLocate","targetElementDescription","locateImage","options","image","prepareJevImage","references","prepareReferences","target","userPromptToString","response","callSystemOneAI","answers","systemOneResponseSchema","JSON","assert","mapJevBox"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACCO,MAAMI,YAAsB,OAAO,EACxCC,wBAAwB,EACxBC,WAAW,EACXC,OAAO,EACR;IACC,MAAMC,QAAQ,MAAMC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBH,YAAY,WAAW;IAC3D,MAAMI,aAAa,MAAMC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkBN;IAC3C,MAAMO,SAASC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBR;IAClC,MAAMS,WAAW,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EACrB;QACE,OAAO;YAAEH;YAAQ,YAAYF,WAAW,UAAU;QAAC;QACnD,QAAQ;YAAE,YAAYF,MAAM,WAAW;YAAE,GAAGE,WAAW,MAAM;QAAC;QAC9D,WAAW;YACT,SAAS;gBACP,MAAM;gBACN,cACE;YACJ;YACA,QAAQ;gBACN,MAAM;gBACN,cACE;YACJ;QACF;IACF,GACAH,QAAQ,YAAY,EACpB;QAAE,aAAaA,QAAQ,WAAW;IAAC;IAErC,MAAM,EAAES,OAAO,EAAE,GAAGC,0BAAAA,uBAAAA,CAAAA,KAA6B,CAC/CC,KAAK,KAAK,CAACJ,SAAS,OAAO;IAE7BK,IAAAA,sBAAAA,MAAAA,AAAAA,EACEH,AAAyB,WAAzBA,QAAQ,OAAO,CAAC,IAAI,IAAeA,AAAwB,UAAxBA,QAAQ,MAAM,CAAC,IAAI,EACtD;IAEF,OAAO;QACL,aAAaF,SAAS,OAAO;QAC7B,OAAOA,SAAS,KAAK;QACrB,GAAIE,QAAQ,OAAO,CAAC,IAAI,IAAI,MACxB;YACE,oBAAoBI,AAAAA,IAAAA,kCAAAA,SAAAA,AAAAA,EAAUJ,QAAQ,MAAM,CAAC,GAAG,EAAEV;QACpD,IACA;YAAE,QAAQ;gBAAC;aAA8C;QAAC,CAAC;IACjE;AACF"}

package/dist/lib/ai-model/models/jev/planning-action.js

@@ -0,0 +1,69 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    buildJevAction: ()=>buildJevAction
+});
+const external_common_js_namespaceObject = require("../../../common.js");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const external_image_js_namespaceObject = require("./image.js");
+function buildJevAction(decision, context, instruction, options) {
+    const { choice } = decision;
+    if ('complete' === choice.kind) return [];
+    (0, utils_namespaceObject.assert)('action' === choice.kind, 'Delegated actions must use the default planner');
+    const action = choice.action;
+    const fields = action.paramSchema ? (0, external_common_js_namespaceObject.findAllMidsceneLocatorField)(action.paramSchema) : [];
+    const locatedPixelResult = (()=>{
+        if (!options.includeLocateInPlanning || 1 !== fields.length) return;
+        (0, utils_namespaceObject.assert)(decision.target?.type === 'box', 'Missing Jev planning target box');
+        return (0, external_image_js_namespaceObject.mapJevBox)(decision.target.box, options.context.shotSize);
+    })();
+    const param = Object.fromEntries(fields.map((field)=>[
+            field,
+            {
+                prompt: `Locate the target for parameter ${JSON.stringify(field)} of action ${JSON.stringify(action.name)} (${action.description ?? action.name}) for the NEXT step toward this goal. Use the current screenshot as authoritative and consider execution feedback. Task context: ${JSON.stringify(context.state)}`,
+                images: instruction.referenceImages,
+                ...locatedPixelResult ? {
+                    locatedPixelResult
+                } : {}
+            }
+        ]));
+    (0, external_common_js_namespaceObject.parseActionParam)(param, action.paramSchema);
+    return [
+        {
+            type: action.name,
+            param
+        }
+    ];
+}
+exports.buildJevAction = __webpack_exports__.buildJevAction;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "buildJevAction"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=planning-action.js.map

package/dist/lib/ai-model/models/jev/planning-action.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/planning-action.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/jev/planning-action.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField, parseActionParam } from '@/common';\nimport type { PlanningAction } from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport type { PlanFn, PlanOptions } from '../../workflows/planning/types';\nimport { mapJevBox } from './image';\nimport type { JevPlanningContext } from './planning-context';\nimport type { JevActionDecision } from './planning-decision';\n\nexport function buildJevAction(\n  decision: JevActionDecision,\n  context: JevPlanningContext,\n  instruction: Parameters<PlanFn>[0],\n  options: PlanOptions,\n): PlanningAction[] {\n  const { choice } = decision;\n  if (choice.kind === 'complete') {\n    return [];\n  }\n  assert(\n    choice.kind === 'action',\n    'Delegated actions must use the default planner',\n  );\n  const action = choice.action;\n  const fields = action.paramSchema\n    ? findAllMidsceneLocatorField(action.paramSchema)\n    : [];\n  const locatedPixelResult = (() => {\n    if (!options.includeLocateInPlanning || fields.length !== 1) {\n      return undefined;\n    }\n    assert(decision.target?.type === 'box', 'Missing Jev planning target box');\n    return mapJevBox(decision.target.box, options.context.shotSize);\n  })();\n  const param = Object.fromEntries(\n    fields.map((field) => [\n      field,\n      {\n        prompt: `Locate the target for parameter ${JSON.stringify(field)} of action ${JSON.stringify(action.name)} (${action.description ?? action.name}) for the NEXT step toward this goal. Use the current screenshot as authoritative and consider execution feedback. Task context: ${JSON.stringify(context.state)}`,\n        images: instruction.referenceImages,\n        ...(locatedPixelResult ? { locatedPixelResult } : {}),\n      },\n    ]),\n  );\n  // Validate declared direct-action support before handing the plan to execution.\n  parseActionParam(param, action.paramSchema);\n  return [{ type: action.name, param }];\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","buildJevAction","decision","context","instruction","options","choice","assert","action","fields","findAllMidsceneLocatorField","locatedPixelResult","mapJevBox","param","field","JSON","parseActionParam"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACEO,SAASI,eACdC,QAA2B,EAC3BC,OAA2B,EAC3BC,WAAkC,EAClCC,OAAoB;IAEpB,MAAM,EAAEC,MAAM,EAAE,GAAGJ;IACnB,IAAII,AAAgB,eAAhBA,OAAO,IAAI,EACb,OAAO,EAAE;IAEXC,IAAAA,sBAAAA,MAAAA,AAAAA,EACED,AAAgB,aAAhBA,OAAO,IAAI,EACX;IAEF,MAAME,SAASF,OAAO,MAAM;IAC5B,MAAMG,SAASD,OAAO,WAAW,GAC7BE,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BF,OAAO,WAAW,IAC9C,EAAE;IACN,MAAMG,qBAAsB,AAAC;QAC3B,IAAI,CAACN,QAAQ,uBAAuB,IAAII,AAAkB,MAAlBA,OAAO,MAAM,EACnD;QAEFF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOL,SAAS,MAAM,EAAE,SAAS,OAAO;QACxC,OAAOU,AAAAA,IAAAA,kCAAAA,SAAAA,AAAAA,EAAUV,SAAS,MAAM,CAAC,GAAG,EAAEG,QAAQ,OAAO,CAAC,QAAQ;IAChE;IACA,MAAMQ,QAAQhB,OAAO,WAAW,CAC9BY,OAAO,GAAG,CAAC,CAACK,QAAU;YACpBA;YACA;gBACE,QAAQ,CAAC,gCAAgC,EAAEC,KAAK,SAAS,CAACD,OAAO,WAAW,EAAEC,KAAK,SAAS,CAACP,OAAO,IAAI,EAAE,EAAE,EAAEA,OAAO,WAAW,IAAIA,OAAO,IAAI,CAAC,iIAAiI,EAAEO,KAAK,SAAS,CAACZ,QAAQ,KAAK,GAAG;gBAClT,QAAQC,YAAY,eAAe;gBACnC,GAAIO,qBAAqB;oBAAEA;gBAAmB,IAAI,CAAC,CAAC;YACtD;SACD;IAGHK,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBH,OAAOL,OAAO,WAAW;IAC1C,OAAO;QAAC;YAAE,MAAMA,OAAO,IAAI;YAAEK;QAAM;KAAE;AACvC"}

package/dist/lib/ai-model/models/jev/planning-context.js

@@ -0,0 +1,72 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    prepareJevPlanningContext: ()=>prepareJevPlanningContext
+});
+const external_image_js_namespaceObject = require("./image.js");
+async function prepareJevPlanningContext(instruction, options) {
+    const history = options.conversationHistory;
+    const image = await (0, external_image_js_namespaceObject.prepareJevImage)(options.context.screenshot.base64);
+    const references = await Promise.all(instruction.referenceImages.map(async (reference, index)=>({
+            key: `reference_${index}`,
+            name: reference.name,
+            image: await (0, external_image_js_namespaceObject.prepareJevImage)(reference.url)
+        })));
+    const state = {
+        goal: instruction.text,
+        context: options.actionContext,
+        history: history.historicalLogsToText(),
+        memory: history.memoriesToText(),
+        feedback: history.pendingFeedbackMessage,
+        references: references.map(({ key, name })=>({
+                image: `\`${key}\``,
+                name
+            })),
+        availableActions: options.actionSpace.map(({ name, description })=>({
+                name,
+                description
+            }))
+    };
+    return {
+        state,
+        images: {
+            screenshot: image.imageBase64,
+            ...Object.fromEntries(references.map(({ key, image })=>[
+                    key,
+                    image.imageBase64
+                ]))
+        }
+    };
+}
+exports.prepareJevPlanningContext = __webpack_exports__.prepareJevPlanningContext;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "prepareJevPlanningContext"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=planning-context.js.map

package/dist/lib/ai-model/models/jev/planning-context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/planning-context.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/jev/planning-context.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanFn, PlanOptions } from '../../workflows/planning/types';\nimport { prepareJevImage } from './image';\n\nexport async function prepareJevPlanningContext(\n  instruction: Parameters<PlanFn>[0],\n  options: PlanOptions,\n) {\n  const history = options.conversationHistory;\n  const image = await prepareJevImage(options.context.screenshot.base64);\n  const references = await Promise.all(\n    instruction.referenceImages.map(async (reference, index) => ({\n      key: `reference_${index}`,\n      name: reference.name,\n      image: await prepareJevImage(reference.url),\n    })),\n  );\n  const state = {\n    goal: instruction.text,\n    context: options.actionContext,\n    history: history.historicalLogsToText(),\n    memory: history.memoriesToText(),\n    feedback: history.pendingFeedbackMessage,\n    references: references.map(({ key, name }) => ({\n      image: `\\`${key}\\``,\n      name,\n    })),\n    availableActions: options.actionSpace.map(({ name, description }) => ({\n      name,\n      description,\n    })),\n  };\n  return {\n    state,\n    images: {\n      screenshot: image.imageBase64,\n      ...Object.fromEntries(\n        references.map(({ key, image }) => [key, image.imageBase64]),\n      ),\n    },\n  };\n}\n\nexport type JevPlanningContext = Awaited<\n  ReturnType<typeof prepareJevPlanningContext>\n>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","prepareJevPlanningContext","instruction","options","history","image","prepareJevImage","references","Promise","reference","index","state","name","description"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACHO,eAAeI,0BACpBC,WAAkC,EAClCC,OAAoB;IAEpB,MAAMC,UAAUD,QAAQ,mBAAmB;IAC3C,MAAME,QAAQ,MAAMC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBH,QAAQ,OAAO,CAAC,UAAU,CAAC,MAAM;IACrE,MAAMI,aAAa,MAAMC,QAAQ,GAAG,CAClCN,YAAY,eAAe,CAAC,GAAG,CAAC,OAAOO,WAAWC,QAAW;YAC3D,KAAK,CAAC,UAAU,EAAEA,OAAO;YACzB,MAAMD,UAAU,IAAI;YACpB,OAAO,MAAMH,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBG,UAAU,GAAG;QAC5C;IAEF,MAAME,QAAQ;QACZ,MAAMT,YAAY,IAAI;QACtB,SAASC,QAAQ,aAAa;QAC9B,SAASC,QAAQ,oBAAoB;QACrC,QAAQA,QAAQ,cAAc;QAC9B,UAAUA,QAAQ,sBAAsB;QACxC,YAAYG,WAAW,GAAG,CAAC,CAAC,EAAEX,GAAG,EAAEgB,IAAI,EAAE,GAAM;gBAC7C,OAAO,CAAC,EAAE,EAAEhB,IAAI,EAAE,CAAC;gBACnBgB;YACF;QACA,kBAAkBT,QAAQ,WAAW,CAAC,GAAG,CAAC,CAAC,EAAES,IAAI,EAAEC,WAAW,EAAE,GAAM;gBACpED;gBACAC;YACF;IACF;IACA,OAAO;QACLF;QACA,QAAQ;YACN,YAAYN,MAAM,WAAW;YAC7B,GAAGR,OAAO,WAAW,CACnBU,WAAW,GAAG,CAAC,CAAC,EAAEX,GAAG,EAAES,KAAK,EAAE,GAAK;oBAACT;oBAAKS,MAAM,WAAW;iBAAC,EAC5D;QACH;IACF;AACF"}

package/dist/lib/ai-model/models/jev/planning-decision.js

@@ -0,0 +1,102 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    decideJevAction: ()=>decideJevAction
+});
+const external_common_js_namespaceObject = require("../../../common.js");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const index_js_namespaceObject = require("../../service-caller/index.js");
+const schema_js_namespaceObject = require("../../service-caller/system-one/schema.js");
+async function decideJevAction(context, options) {
+    const criteria = {
+        'control:complete': 'The entire user goal is already satisfied, confirmed by the current screenshot and execution feedback. No text answer is requested.',
+        'control:delegate': 'The next step needs generated output, an explanation of why the task cannot be completed, or something not covered by the action choices. Delegate to the default model.'
+    };
+    const choices = new Map([
+        [
+            'control:complete',
+            {
+                kind: 'complete'
+            }
+        ],
+        [
+            'control:delegate',
+            {
+                kind: 'delegate'
+            }
+        ]
+    ]);
+    for (const action of options.actionSpace){
+        const kind = action.supportedBySystemOne ? 'action' : 'delegate';
+        const key = `${kind}:${action.name}`;
+        (0, utils_namespaceObject.assert)(!choices.has(key), `Duplicate Jev planning choice: ${key}`);
+        choices.set(key, {
+            kind,
+            action
+        });
+        criteria[key] = `${action.name}: ${action.description ?? action.name}. ${'action' === kind ? 'Perform this action.' : 'Suggest this action and delegate the next planning step, including its parameters, to the default model.'}`;
+    }
+    (0, utils_namespaceObject.assert)(choices.size <= 255, 'Jev Choice supports at most 255 options');
+    const locateInPlanning = options.includeLocateInPlanning && options.actionSpace.some((action)=>action.supportedBySystemOne && action.paramSchema && 1 === (0, external_common_js_namespaceObject.findAllMidsceneLocatorField)(action.paramSchema).length);
+    const response = await (0, index_js_namespaceObject.callSystemOneAI)({
+        ...context,
+        questions: {
+            ...locateInPlanning ? {
+                target: {
+                    type: 'box',
+                    instructions: 'Identify the single UI element to interact with in the NEXT step toward `goal`, using the current `screenshot`, context, history and feedback. Return its bounding box in `screenshot`. This is the target of the next direct action (such as Tap, DoubleClick, Hover or ClearInput), not the final goal of the entire task. Reference images describe targets, not the current page. If the next step needs no target or must be delegated, this box is unused.'
+                }
+            } : {},
+            action: {
+                type: 'choice',
+                instructions: 'Choose exactly the next step toward `goal` using the current `screenshot`, context, history and feedback. History records proposed actions; feedback and current screenshot establish what actually happened. Do not repeat completed steps. Reference images describe targets, not the current page. A delegate choice means the named action is likely needed; it has not been executed. Choose control:complete only when no requested work or text answer remains.',
+                criteria
+            }
+        }
+    }, options.modelRuntime, {
+        abortSignal: options.abortSignal
+    });
+    const { answers } = schema_js_namespaceObject.systemOneResponseSchema.parse(JSON.parse(response.content));
+    (0, utils_namespaceObject.assert)('choice' === answers.action.type, 'Invalid Jev planning action');
+    const selected = answers.action.choice;
+    const choice = choices.get(selected);
+    (0, utils_namespaceObject.assert)(choice, `Unknown Jev planning choice: ${selected}`);
+    return {
+        selected,
+        choice,
+        response,
+        target: locateInPlanning ? answers.target : void 0
+    };
+}
+exports.decideJevAction = __webpack_exports__.decideJevAction;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "decideJevAction"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=planning-decision.js.map

package/dist/lib/ai-model/models/jev/planning-decision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/planning-decision.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/jev/planning-decision.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport { callSystemOneAI } from '../../service-caller';\nimport { systemOneResponseSchema } from '../../service-caller/system-one/schema';\nimport type { PlanOptions } from '../../workflows/planning/types';\nimport type { JevPlanningContext } from './planning-context';\n\ntype JevPlanningChoice =\n  | { kind: 'complete' }\n  | { kind: 'delegate'; action?: DeviceAction }\n  | { kind: 'action'; action: DeviceAction };\n\nexport async function decideJevAction(\n  context: JevPlanningContext,\n  options: PlanOptions,\n) {\n  const criteria: Record<string, string> = {\n    'control:complete':\n      'The entire user goal is already satisfied, confirmed by the current screenshot and execution feedback. No text answer is requested.',\n    'control:delegate':\n      'The next step needs generated output, an explanation of why the task cannot be completed, or something not covered by the action choices. Delegate to the default model.',\n  };\n  const choices = new Map<string, JevPlanningChoice>([\n    ['control:complete', { kind: 'complete' }],\n    ['control:delegate', { kind: 'delegate' }],\n  ]);\n  for (const action of options.actionSpace) {\n    const kind = action.supportedBySystemOne ? 'action' : 'delegate';\n    const key = `${kind}:${action.name}`;\n    assert(!choices.has(key), `Duplicate Jev planning choice: ${key}`);\n    choices.set(key, { kind, action });\n    criteria[key] = `${action.name}: ${action.description ?? action.name}. ${\n      kind === 'action'\n        ? 'Perform this action.'\n        : 'Suggest this action and delegate the next planning step, including its parameters, to the default model.'\n    }`;\n  }\n  assert(choices.size <= 255, 'Jev Choice supports at most 255 options');\n  const locateInPlanning =\n    options.includeLocateInPlanning &&\n    options.actionSpace.some(\n      (action) =>\n        action.supportedBySystemOne &&\n        action.paramSchema &&\n        findAllMidsceneLocatorField(action.paramSchema).length === 1,\n    );\n  const response = await callSystemOneAI(\n    {\n      ...context,\n      questions: {\n        ...(locateInPlanning\n          ? {\n              target: {\n                type: 'box' as const,\n                instructions:\n                  'Identify the single UI element to interact with in the NEXT step toward `goal`, using the current `screenshot`, context, history and feedback. Return its bounding box in `screenshot`. This is the target of the next direct action (such as Tap, DoubleClick, Hover or ClearInput), not the final goal of the entire task. Reference images describe targets, not the current page. If the next step needs no target or must be delegated, this box is unused.',\n              },\n            }\n          : {}),\n        action: {\n          type: 'choice',\n          instructions:\n            'Choose exactly the next step toward `goal` using the current `screenshot`, context, history and feedback. History records proposed actions; feedback and current screenshot establish what actually happened. Do not repeat completed steps. Reference images describe targets, not the current page. A delegate choice means the named action is likely needed; it has not been executed. Choose control:complete only when no requested work or text answer remains.',\n          criteria,\n        },\n      },\n    },\n    options.modelRuntime,\n    { abortSignal: options.abortSignal },\n  );\n  const { answers } = systemOneResponseSchema.parse(\n    JSON.parse(response.content),\n  );\n  assert(answers.action.type === 'choice', 'Invalid Jev planning action');\n  const selected = answers.action.choice;\n  const choice = choices.get(selected);\n  assert(choice, `Unknown Jev planning choice: ${selected}`);\n  return {\n    selected,\n    choice,\n    response,\n    target: locateInPlanning ? answers.target : undefined,\n  };\n}\n\nexport type JevActionDecision = Awaited<ReturnType<typeof decideJevAction>>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","decideJevAction","context","options","criteria","choices","Map","action","kind","assert","locateInPlanning","findAllMidsceneLocatorField","response","callSystemOneAI","answers","systemOneResponseSchema","JSON","selected","choice","undefined"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACOO,eAAeI,gBACpBC,OAA2B,EAC3BC,OAAoB;IAEpB,MAAMC,WAAmC;QACvC,oBACE;QACF,oBACE;IACJ;IACA,MAAMC,UAAU,IAAIC,IAA+B;QACjD;YAAC;YAAoB;gBAAE,MAAM;YAAW;SAAE;QAC1C;YAAC;YAAoB;gBAAE,MAAM;YAAW;SAAE;KAC3C;IACD,KAAK,MAAMC,UAAUJ,QAAQ,WAAW,CAAE;QACxC,MAAMK,OAAOD,OAAO,oBAAoB,GAAG,WAAW;QACtD,MAAMX,MAAM,GAAGY,KAAK,CAAC,EAAED,OAAO,IAAI,EAAE;QACpCE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACJ,QAAQ,GAAG,CAACT,MAAM,CAAC,+BAA+B,EAAEA,KAAK;QACjES,QAAQ,GAAG,CAACT,KAAK;YAAEY;YAAMD;QAAO;QAChCH,QAAQ,CAACR,IAAI,GAAG,GAAGW,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAIA,OAAO,IAAI,CAAC,EAAE,EACrEC,AAAS,aAATA,OACI,yBACA,4GACJ;IACJ;IACAC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,QAAQ,IAAI,IAAI,KAAK;IAC5B,MAAMK,mBACJP,QAAQ,uBAAuB,IAC/BA,QAAQ,WAAW,CAAC,IAAI,CACtB,CAACI,SACCA,OAAO,oBAAoB,IAC3BA,OAAO,WAAW,IAClBI,AAA2D,MAA3DA,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BJ,OAAO,WAAW,EAAE,MAAM;IAE5D,MAAMK,WAAW,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EACrB;QACE,GAAGX,OAAO;QACV,WAAW;YACT,GAAIQ,mBACA;gBACE,QAAQ;oBACN,MAAM;oBACN,cACE;gBACJ;YACF,IACA,CAAC,CAAC;YACN,QAAQ;gBACN,MAAM;gBACN,cACE;gBACFN;YACF;QACF;IACF,GACAD,QAAQ,YAAY,EACpB;QAAE,aAAaA,QAAQ,WAAW;IAAC;IAErC,MAAM,EAAEW,OAAO,EAAE,GAAGC,0BAAAA,uBAAAA,CAAAA,KAA6B,CAC/CC,KAAK,KAAK,CAACJ,SAAS,OAAO;IAE7BH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOK,AAAwB,aAAxBA,QAAQ,MAAM,CAAC,IAAI,EAAe;IACzC,MAAMG,WAAWH,QAAQ,MAAM,CAAC,MAAM;IACtC,MAAMI,SAASb,QAAQ,GAAG,CAACY;IAC3BR,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOS,QAAQ,CAAC,6BAA6B,EAAED,UAAU;IACzD,OAAO;QACLA;QACAC;QACAN;QACA,QAAQF,mBAAmBI,QAAQ,MAAM,GAAGK;IAC9C;AACF"}

package/dist/lib/ai-model/models/jev/planning.js

@@ -27,139 +27,27 @@ __webpack_require__.d(__webpack_exports__, {
     jevPlan: ()=>jevPlan
 });
 const external_common_js_namespaceObject = require("../../../common.js");
-const utils_namespaceObject = require("@midscene/shared/utils");
-const index_js_namespaceObject = require("../../service-caller/index.js");
-const system_one_js_namespaceObject = require("../../service-caller/system-one.js");
-const standard_planning_js_namespaceObject = require("../../workflows/planning/standard-planning.js");
-const external_image_js_namespaceObject = require("./image.js");
+const external_planning_action_js_namespaceObject = require("./planning-action.js");
+const external_planning_context_js_namespaceObject = require("./planning-context.js");
+const external_planning_decision_js_namespaceObject = require("./planning-decision.js");
 const jevPlan = async (instruction, options)=>{
+    if (options.actionSpace.length + 2 > 255) throw new Error('Jev planning supports at most 253 actions (255 choices including completion and delegation).');
     const history = options.conversationHistory;
-    const delegate = async (reason)=>{
-        (0, utils_namespaceObject.assert)(options.getDefaultModelRuntime, 'Default model runtime is required for Jev planning delegation');
-        const base = options.getDefaultModelRuntime();
-        (0, utils_namespaceObject.assert)('jev' !== base.config.modelFamily, 'Jev cannot delegate planning to itself');
-        const runtime = {
-            ...base,
-            config: {
-                ...base.config,
-                intent: 'planning'
-            },
-            executionId: options.modelRuntime.executionId,
-            onUsage: base.onUsage ? (usage)=>base.onUsage?.({
-                    ...usage,
-                    intent: 'planning'
-                }) : void 0,
-            modelRouting: {
-                requestedModel: options.modelRuntime.config.modelName,
-                requestedSlot: options.modelRuntime.config.slot,
-                task: 'planning',
-                reason
-            }
-        };
-        const planner = 'custom' === runtime.adapter.planning.kind ? runtime.adapter.planning.planFn : standard_planning_js_namespaceObject.standardPlan;
-        return planner(instruction, {
-            ...options,
-            modelRuntime: runtime,
-            includeLocateInPlanning: false
-        });
-    };
-    const image = await (0, external_image_js_namespaceObject.prepareJevImage)(options.context.screenshot.base64);
-    const references = await Promise.all(instruction.referenceImages.map(async (reference, index)=>({
-            key: `reference_${index}`,
-            name: reference.name,
-            image: await (0, external_image_js_namespaceObject.prepareJevImage)(reference.url)
-        })));
-    const state = {
-        goal: instruction.text,
-        context: options.actionContext,
-        history: history.historicalLogsToText(),
-        memory: history.memoriesToText(),
-        feedback: history.pendingFeedbackMessage,
-        references: references.map(({ key, name })=>({
-                image: `\`${key}\``,
-                name
-            })),
-        availableActions: options.actionSpace.map(({ name, description })=>({
-                name,
-                description
-            }))
-    };
-    const criteria = {
-        Complete: 'The entire user goal is already satisfied, confirmed by the current screenshot and execution feedback. No text answer is requested.',
-        Delegate: 'The next step requires text input, generated output, an action or parameters not covered by the other choices. Also use for impossible tasks needing explanation.'
-    };
-    for (const action of options.actionSpace)if ('Tap' === action.name || 'Scroll' === action.name || 'Sleep' === action.name) criteria[action.name] = 'Sleep' === action.name ? 'Wait 800 ms for a transient loading state.' : action.description ?? action.name;
-    const response = await (0, index_js_namespaceObject.callAI)([], options.modelRuntime, {
-        abortSignal: options.abortSignal,
-        systemOneRequest: {
-            state,
-            images: {
-                screenshot: image.imageBase64,
-                ...Object.fromEntries(references.map(({ key, image })=>[
-                        key,
-                        image.imageBase64
-                    ]))
-            },
-            questions: {
-                action: {
-                    type: 'choice',
-                    instructions: 'Choose exactly the next step toward `goal` using the current `screenshot`, context, history and feedback. History records proposed actions; feedback and current screenshot establish what actually happened. Do not repeat completed steps. Reference images describe targets, not the current page.',
-                    criteria
-                },
-                ...criteria.Scroll ? {
-                    direction: {
-                        type: 'choice',
-                        instructions: 'If scrolling is the next step toward `goal`, which direction reveals the desired off-screen content? Evaluate from the screenshot and history independently of the action question.',
-                        criteria: {
-                            down: 'Reveal content below',
-                            up: 'Reveal content above',
-                            left: 'Reveal content to the left',
-                            right: 'Reveal content to the right'
-                        }
-                    }
-                } : {}
-            }
+    const context = await (0, external_planning_context_js_namespaceObject.prepareJevPlanningContext)(instruction, options);
+    const decision = await (0, external_planning_decision_js_namespaceObject.decideJevAction)(context, options);
+    const { selected, choice, response } = decision;
+    if ('delegate' === choice.kind) return {
+        actions: [],
+        log: `Jev selected ${selected}`,
+        rawResponse: response.content,
+        usage: response.usage,
+        shouldContinuePlanning: true,
+        delegation: {
+            reason: choice.action ? `The previous planner suggested ${choice.action.name}; delegating planning with this action as an advisory suggestion` : 'The previous planner delegated planning of the next step',
+            suggestedAction: choice.action?.name
         }
-    });
-    const { answers } = system_one_js_namespaceObject.systemOneResponseSchema.parse(JSON.parse(response.content));
-    (0, utils_namespaceObject.assert)('choice' === answers.action.type, 'Invalid Jev planning action');
-    const selected = answers.action.choice;
-    if ('Delegate' === selected) return delegate('The next step requires parameters or output that Jev cannot generate');
-    if (('Tap' === selected || 'Scroll' === selected) && options.locateModelRuntime?.config.modelFamily !== 'jev') return delegate("The configured Locate model requires a concrete target description from a general-purpose planner");
-    const locate = {
-        prompt: `Locate the single ${'Scroll' === selected ? 'scrollable region to scroll' : 'element to tap'} for the NEXT step toward this goal. Use the current screenshot as authoritative. Previous proposals may have failed; use feedback. Task context: ${JSON.stringify(state)}`,
-        images: instruction.referenceImages
     };
-    const actions = (()=>{
-        if ('Complete' === selected) return [];
-        if ('Sleep' === selected) return [
-            {
-                type: 'Sleep',
-                param: {
-                    timeMs: 800
-                }
-            }
-        ];
-        if ('Tap' === selected) return [
-            {
-                type: 'Tap',
-                param: {
-                    locate
-                }
-            }
-        ];
-        (0, utils_namespaceObject.assert)('Scroll' === selected && answers.direction?.type === 'choice', 'Invalid Jev scroll action');
-        return [
-            {
-                type: 'Scroll',
-                param: {
-                    locate,
-                    direction: answers.direction.choice,
-                    scrollType: 'singleAction'
-                }
-            }
-        ];
-    })();
+    const actions = (0, external_planning_action_js_namespaceObject.buildJevAction)(decision, context, instruction, options);
     const log = `Jev selected ${selected}`;
     const feedback = history.pendingFeedbackMessage;
     history.resetPendingFeedbackMessageIfExists();
@@ -184,8 +72,8 @@ const jevPlan = async (instruction, options)=>{
         log,
         rawResponse: response.content,
         usage: response.usage,
-        shouldContinuePlanning: 'Complete' !== selected,
-        ...'Complete' === selected ? {
+        shouldContinuePlanning: 'complete' !== choice.kind,
+        ...'complete' === choice.kind ? {
             finalizeSuccess: true
         } : {},
         yamlFlow: (0, external_common_js_namespaceObject.buildYamlFlowFromPlans)(actions, options.actionSpace)