@midscene/core 1.15.5 → 1.16.0

package/dist/es/ai-model/models/jev/adapter.mjs

@@ -0,0 +1,135 @@
+import { userPromptToString } from "../../../common.mjs";
+import { assert } from "@midscene/shared/utils";
+import { callAI } from "../../service-caller/index.mjs";
+import { systemOneResponseSchema } from "../../service-caller/system-one.mjs";
+import { prepareUserPrompt } from "../../shared/multimodal-prompt.mjs";
+import { mapJevBox, prepareJevImage } from "./image.mjs";
+import { jevPlan } from "./planning.mjs";
+async function prepareReferences(prompt) {
+    const { referenceImages } = await prepareUserPrompt(prompt);
+    const prepared = await Promise.all(referenceImages.map(async (image, index)=>({
+            key: `reference_${index}`,
+            name: image.name,
+            image: await prepareJevImage(image.url)
+        })));
+    return {
+        references: prepared.map(({ key, name })=>({
+                image: `\`${key}\``,
+                name
+            })),
+        images: Object.fromEntries(prepared.map(({ key, image })=>[
+                key,
+                image.imageBase64
+            ]))
+    };
+}
+const locate = async ({ targetElementDescription, locateImage, options })=>{
+    const image = await prepareJevImage(locateImage.imageBase64);
+    const references = await prepareReferences(targetElementDescription);
+    const target = userPromptToString(targetElementDescription);
+    const response = await callAI([], options.modelRuntime, {
+        abortSignal: options.abortSignal,
+        systemOneRequest: {
+            state: {
+                target,
+                references: references.references
+            },
+            images: {
+                screenshot: image.imageBase64,
+                ...references.images
+            },
+            questions: {
+                present: {
+                    type: 'noul',
+                    instructions: 'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.'
+                },
+                target: {
+                    type: 'box',
+                    instructions: 'The single target described in `target`, in `screenshot`.'
+                }
+            }
+        }
+    });
+    const { answers } = systemOneResponseSchema.parse(JSON.parse(response.content));
+    assert('noul' === answers.present.type && 'box' === answers.target.type, 'Invalid Jev locate answers');
+    return {
+        rawResponse: response.content,
+        usage: response.usage,
+        ...answers.present.noul >= 0.5 ? {
+            locatedPixelResult: mapJevBox(answers.target.box, image, locateImage)
+        } : {
+            errors: [
+                'The target is not present in the screenshot'
+            ]
+        }
+    };
+};
+const adapter_boolean = async (options)=>{
+    const references = await prepareReferences({
+        prompt: '',
+        ...options.multimodalPrompt
+    });
+    const frames = options.extractOption?.screenshotIncluded === false ? [] : options.context.screenshotSequence?.length ? options.context.screenshotSequence : [
+        options.context.screenshot
+    ];
+    const screenshots = await Promise.all(frames.map((frame)=>prepareJevImage(frame.base64)));
+    const result = await callAI([], options.modelRuntime, {
+        abortSignal: options.abortSignal,
+        systemOneRequest: {
+            state: {
+                statement: options.dataQuery,
+                context: options.extractOption?.context,
+                pageDescription: options.pageDescription,
+                frames: screenshots.map((_, index)=>`\`frame_${index}\``),
+                references: references.references
+            },
+            images: {
+                ...Object.fromEntries(screenshots.map((image, index)=>[
+                        `frame_${index}`,
+                        image.imageBase64
+                    ])),
+                ...references.images
+            },
+            questions: {
+                result: {
+                    type: 'noul',
+                    instructions: "Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present."
+                }
+            }
+        }
+    });
+    const answer = systemOneResponseSchema.parse(JSON.parse(result.content)).answers.result;
+    assert('noul' === answer.type, 'Invalid Jev boolean answer');
+    return {
+        value: answer.noul >= 0.5,
+        rawResponse: result.content,
+        usage: result.usage
+    };
+};
+const jevAdapters = {
+    jev: {
+        supportedTasks: [
+            'locate',
+            'boolean',
+            'planning'
+        ],
+        supportedProtocols: [],
+        locate: {
+            kind: 'custom',
+            locateFn: locate
+        },
+        insight: {
+            booleanFn: adapter_boolean
+        },
+        planning: {
+            kind: 'custom',
+            planFn: jevPlan,
+            cacheEnabled: false,
+            supportsActionDeepLocate: false,
+            defaultReplanningCycleLimit: 20
+        }
+    }
+};
+export { jevAdapters };
+
+//# sourceMappingURL=adapter.mjs.map

package/dist/es/ai-model/models/jev/adapter.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/adapter.mjs","sources":["../../../../../src/ai-model/models/jev/adapter.ts"],"sourcesContent":["import { userPromptToString } from '@/common';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { callAI } from '../../service-caller';\nimport { systemOneResponseSchema } from '../../service-caller/system-one';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\nimport type { LocateFn } from '../../workflows/grounding/types';\nimport type { BooleanFn } from '../../workflows/insight/boolean';\nimport { mapJevBox, prepareJevImage } from './image';\nimport { jevPlan } from './planning';\n\nasync function prepareReferences(\n  prompt: Parameters<typeof prepareUserPrompt>[0],\n) {\n  const { referenceImages } = await prepareUserPrompt(prompt);\n  const prepared = await Promise.all(\n    referenceImages.map(async (image, index) => ({\n      key: `reference_${index}`,\n      name: image.name,\n      image: await prepareJevImage(image.url),\n    })),\n  );\n  return {\n    references: prepared.map(({ key, name }) => ({\n      image: `\\`${key}\\``,\n      name,\n    })),\n    images: Object.fromEntries(\n      prepared.map(({ key, image }) => [key, image.imageBase64]),\n    ),\n  };\n}\n\nconst locate: LocateFn = async ({\n  targetElementDescription,\n  locateImage,\n  options,\n}) => {\n  const image = await prepareJevImage(locateImage.imageBase64);\n  const references = await prepareReferences(targetElementDescription);\n  const target = userPromptToString(targetElementDescription);\n  const response = await callAI([], options.modelRuntime, {\n    abortSignal: options.abortSignal,\n    systemOneRequest: {\n      state: { target, references: references.references },\n      images: { screenshot: image.imageBase64, ...references.images },\n      questions: {\n        present: {\n          type: 'noul',\n          instructions:\n            'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.',\n        },\n        target: {\n          type: 'box',\n          instructions:\n            'The single target described in `target`, in `screenshot`.',\n        },\n      },\n    },\n  });\n  const { answers } = systemOneResponseSchema.parse(\n    JSON.parse(response.content),\n  );\n  assert(\n    answers.present.type === 'noul' && answers.target.type === 'box',\n    'Invalid Jev locate answers',\n  );\n  return {\n    rawResponse: response.content,\n    usage: response.usage,\n    ...(answers.present.noul >= 0.5\n      ? {\n          locatedPixelResult: mapJevBox(answers.target.box, image, locateImage),\n        }\n      : { errors: ['The target is not present in the screenshot'] }),\n  };\n};\n\nconst boolean: BooleanFn = async (options) => {\n  const references = await prepareReferences({\n    prompt: '',\n    ...options.multimodalPrompt,\n  });\n  const frames =\n    options.extractOption?.screenshotIncluded === false\n      ? []\n      : options.context.screenshotSequence?.length\n        ? options.context.screenshotSequence\n        : [options.context.screenshot];\n  const screenshots = await Promise.all(\n    frames.map((frame) => prepareJevImage(frame.base64)),\n  );\n  const result = await callAI([], options.modelRuntime, {\n    abortSignal: options.abortSignal,\n    systemOneRequest: {\n      state: {\n        statement: options.dataQuery,\n        context: options.extractOption?.context,\n        pageDescription: options.pageDescription,\n        frames: screenshots.map((_, index) => `\\`frame_${index}\\``),\n        references: references.references,\n      },\n      images: {\n        ...Object.fromEntries(\n          screenshots.map((image, index) => [\n            `frame_${index}`,\n            image.imageBase64,\n          ]),\n        ),\n        ...references.images,\n      },\n      questions: {\n        result: {\n          type: 'noul',\n          instructions:\n            'Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present.',\n        },\n      },\n    },\n  });\n  const answer = systemOneResponseSchema.parse(JSON.parse(result.content))\n    .answers.result;\n  assert(answer.type === 'noul', 'Invalid Jev boolean answer');\n  return {\n    value: answer.noul >= 0.5,\n    rawResponse: result.content,\n    usage: result.usage,\n  };\n};\n\nexport const jevAdapters = {\n  jev: {\n    supportedTasks: ['locate', 'boolean', 'planning'],\n    supportedProtocols: [],\n    locate: { kind: 'custom', locateFn: locate },\n    insight: { booleanFn: boolean },\n    planning: {\n      kind: 'custom',\n      planFn: jevPlan,\n      cacheEnabled: false,\n      supportsActionDeepLocate: false,\n      defaultReplanningCycleLimit: 20,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'jev'>;\n"],"names":["prepareReferences","prompt","referenceImages","prepareUserPrompt","prepared","Promise","image","index","prepareJevImage","key","name","Object","locate","targetElementDescription","locateImage","options","references","target","userPromptToString","response","callAI","answers","systemOneResponseSchema","JSON","assert","mapJevBox","boolean","frames","screenshots","frame","result","_","answer","jevAdapters","jevPlan"],"mappings":";;;;;;;AAYA,eAAeA,kBACbC,MAA+C;IAE/C,MAAM,EAAEC,eAAe,EAAE,GAAG,MAAMC,kBAAkBF;IACpD,MAAMG,WAAW,MAAMC,QAAQ,GAAG,CAChCH,gBAAgB,GAAG,CAAC,OAAOI,OAAOC,QAAW;YAC3C,KAAK,CAAC,UAAU,EAAEA,OAAO;YACzB,MAAMD,MAAM,IAAI;YAChB,OAAO,MAAME,gBAAgBF,MAAM,GAAG;QACxC;IAEF,OAAO;QACL,YAAYF,SAAS,GAAG,CAAC,CAAC,EAAEK,GAAG,EAAEC,IAAI,EAAE,GAAM;gBAC3C,OAAO,CAAC,EAAE,EAAED,IAAI,EAAE,CAAC;gBACnBC;YACF;QACA,QAAQC,OAAO,WAAW,CACxBP,SAAS,GAAG,CAAC,CAAC,EAAEK,GAAG,EAAEH,KAAK,EAAE,GAAK;gBAACG;gBAAKH,MAAM,WAAW;aAAC;IAE7D;AACF;AAEA,MAAMM,SAAmB,OAAO,EAC9BC,wBAAwB,EACxBC,WAAW,EACXC,OAAO,EACR;IACC,MAAMT,QAAQ,MAAME,gBAAgBM,YAAY,WAAW;IAC3D,MAAME,aAAa,MAAMhB,kBAAkBa;IAC3C,MAAMI,SAASC,mBAAmBL;IAClC,MAAMM,WAAW,MAAMC,OAAO,EAAE,EAAEL,QAAQ,YAAY,EAAE;QACtD,aAAaA,QAAQ,WAAW;QAChC,kBAAkB;YAChB,OAAO;gBAAEE;gBAAQ,YAAYD,WAAW,UAAU;YAAC;YACnD,QAAQ;gBAAE,YAAYV,MAAM,WAAW;gBAAE,GAAGU,WAAW,MAAM;YAAC;YAC9D,WAAW;gBACT,SAAS;oBACP,MAAM;oBACN,cACE;gBACJ;gBACA,QAAQ;oBACN,MAAM;oBACN,cACE;gBACJ;YACF;QACF;IACF;IACA,MAAM,EAAEK,OAAO,EAAE,GAAGC,wBAAwB,KAAK,CAC/CC,KAAK,KAAK,CAACJ,SAAS,OAAO;IAE7BK,OACEH,AAAyB,WAAzBA,QAAQ,OAAO,CAAC,IAAI,IAAeA,AAAwB,UAAxBA,QAAQ,MAAM,CAAC,IAAI,EACtD;IAEF,OAAO;QACL,aAAaF,SAAS,OAAO;QAC7B,OAAOA,SAAS,KAAK;QACrB,GAAIE,QAAQ,OAAO,CAAC,IAAI,IAAI,MACxB;YACE,oBAAoBI,UAAUJ,QAAQ,MAAM,CAAC,GAAG,EAAEf,OAAOQ;QAC3D,IACA;YAAE,QAAQ;gBAAC;aAA8C;QAAC,CAAC;IACjE;AACF;AAEA,MAAMY,kBAAqB,OAAOX;IAChC,MAAMC,aAAa,MAAMhB,kBAAkB;QACzC,QAAQ;QACR,GAAGe,QAAQ,gBAAgB;IAC7B;IACA,MAAMY,SACJZ,QAAQ,aAAa,EAAE,uBAAuB,QAC1C,EAAE,GACFA,QAAQ,OAAO,CAAC,kBAAkB,EAAE,SAClCA,QAAQ,OAAO,CAAC,kBAAkB,GAClC;QAACA,QAAQ,OAAO,CAAC,UAAU;KAAC;IACpC,MAAMa,cAAc,MAAMvB,QAAQ,GAAG,CACnCsB,OAAO,GAAG,CAAC,CAACE,QAAUrB,gBAAgBqB,MAAM,MAAM;IAEpD,MAAMC,SAAS,MAAMV,OAAO,EAAE,EAAEL,QAAQ,YAAY,EAAE;QACpD,aAAaA,QAAQ,WAAW;QAChC,kBAAkB;YAChB,OAAO;gBACL,WAAWA,QAAQ,SAAS;gBAC5B,SAASA,QAAQ,aAAa,EAAE;gBAChC,iBAAiBA,QAAQ,eAAe;gBACxC,QAAQa,YAAY,GAAG,CAAC,CAACG,GAAGxB,QAAU,CAAC,QAAQ,EAAEA,MAAM,EAAE,CAAC;gBAC1D,YAAYS,WAAW,UAAU;YACnC;YACA,QAAQ;gBACN,GAAGL,OAAO,WAAW,CACnBiB,YAAY,GAAG,CAAC,CAACtB,OAAOC,QAAU;wBAChC,CAAC,MAAM,EAAEA,OAAO;wBAChBD,MAAM,WAAW;qBAClB,EACF;gBACD,GAAGU,WAAW,MAAM;YACtB;YACA,WAAW;gBACT,QAAQ;oBACN,MAAM;oBACN,cACE;gBACJ;YACF;QACF;IACF;IACA,MAAMgB,SAASV,wBAAwB,KAAK,CAACC,KAAK,KAAK,CAACO,OAAO,OAAO,GACnE,OAAO,CAAC,MAAM;IACjBN,OAAOQ,AAAgB,WAAhBA,OAAO,IAAI,EAAa;IAC/B,OAAO;QACL,OAAOA,OAAO,IAAI,IAAI;QACtB,aAAaF,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,MAAMG,cAAc;IACzB,KAAK;QACH,gBAAgB;YAAC;YAAU;YAAW;SAAW;QACjD,oBAAoB,EAAE;QACtB,QAAQ;YAAE,MAAM;YAAU,UAAUrB;QAAO;QAC3C,SAAS;YAAE,WAAWc;QAAQ;QAC9B,UAAU;YACR,MAAM;YACN,QAAQQ;YACR,cAAc;YACd,0BAA0B;YAC1B,6BAA6B;QAC/B;IACF;AACF"}

package/dist/es/ai-model/models/jev/image.mjs

@@ -0,0 +1,48 @@
+import { Buffer } from "node:buffer";
+import { imageInfoOfBase64, padBase64ImageToMinimumSize, parseBase64, preProcessImageUrl, resizeBase64ImageToJpeg } from "@midscene/shared/img";
+import { assert } from "@midscene/shared/utils";
+async function prepareJevImage(url) {
+    const base64 = await preProcessImageUrl(url, true);
+    const source = await imageInfoOfBase64(base64);
+    const scale = Math.min(1, 1568 / source.width, 1568 / source.height);
+    const contentSize = {
+        width: Math.max(1, Math.round(source.width * scale)),
+        height: Math.max(1, Math.round(source.height * scale))
+    };
+    const resized = await resizeBase64ImageToJpeg(base64, {
+        targetSize: contentSize
+    });
+    const prepared = await padBase64ImageToMinimumSize(resized, 280);
+    assert(Buffer.from(parseBase64(prepared.imageBase64).body, 'base64').length <= 16777216, 'Jev image exceeds 16 MiB');
+    return {
+        ...prepared,
+        contentSize,
+        sourceSize: {
+            width: source.width,
+            height: source.height
+        }
+    };
+}
+function mapJevBox(box, image, targetSize) {
+    const left = box.left * image.width * targetSize.width / image.contentSize.width;
+    const top = box.top * image.height * targetSize.height / image.contentSize.height;
+    const right = box.right * image.width * targetSize.width / image.contentSize.width;
+    const bottom = box.bottom * image.height * targetSize.height / image.contentSize.height;
+    const center = [
+        (left + right) / 2,
+        (top + bottom) / 2
+    ];
+    assert(center[0] >= 0 && center[0] < targetSize.width && center[1] >= 0 && center[1] < targetSize.height, 'Jev returned a target in image padding, outside the screenshot');
+    return {
+        center,
+        rect: {
+            left,
+            top,
+            width: right - left,
+            height: bottom - top
+        }
+    };
+}
+export { mapJevBox, prepareJevImage };
+
+//# sourceMappingURL=image.mjs.map

package/dist/es/ai-model/models/jev/image.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/image.mjs","sources":["../../../../../src/ai-model/models/jev/image.ts"],"sourcesContent":["import { Buffer } from 'node:buffer';\nimport {\n  imageInfoOfBase64,\n  padBase64ImageToMinimumSize,\n  parseBase64,\n  preProcessImageUrl,\n  resizeBase64ImageToJpeg,\n} from '@midscene/shared/img';\nimport { assert } from '@midscene/shared/utils';\nimport type { PixelLocateResult } from '../../shared/model-locate-result';\n\nexport async function prepareJevImage(url: string) {\n  const base64 = await preProcessImageUrl(url, true);\n  const source = await imageInfoOfBase64(base64);\n  const scale = Math.min(1, 1568 / source.width, 1568 / source.height);\n  const contentSize = {\n    width: Math.max(1, Math.round(source.width * scale)),\n    height: Math.max(1, Math.round(source.height * scale)),\n  };\n  const resized = await resizeBase64ImageToJpeg(base64, {\n    targetSize: contentSize,\n  });\n  const prepared = await padBase64ImageToMinimumSize(resized, 280);\n  assert(\n    Buffer.from(parseBase64(prepared.imageBase64).body, 'base64').length <=\n      16 * 1024 * 1024,\n    'Jev image exceeds 16 MiB',\n  );\n  return {\n    ...prepared,\n    contentSize,\n    sourceSize: { width: source.width, height: source.height },\n  };\n}\n\nexport function mapJevBox(\n  box: { left: number; top: number; right: number; bottom: number },\n  image: Awaited<ReturnType<typeof prepareJevImage>>,\n  targetSize: { width: number; height: number },\n): PixelLocateResult {\n  // Jev returns fractions of the padded image. Undo padding and scale before\n  // the grounding workflow maps a DeepLocate crop back to the full screenshot.\n  const left =\n    (box.left * image.width * targetSize.width) / image.contentSize.width;\n  const top =\n    (box.top * image.height * targetSize.height) / image.contentSize.height;\n  const right =\n    (box.right * image.width * targetSize.width) / image.contentSize.width;\n  const bottom =\n    (box.bottom * image.height * targetSize.height) / image.contentSize.height;\n  const center: [number, number] = [(left + right) / 2, (top + bottom) / 2];\n  assert(\n    center[0] >= 0 &&\n      center[0] < targetSize.width &&\n      center[1] >= 0 &&\n      center[1] < targetSize.height,\n    'Jev returned a target in image padding, outside the screenshot',\n  );\n  return {\n    center,\n    rect: { left, top, width: right - left, height: bottom - top },\n  };\n}\n"],"names":["prepareJevImage","url","base64","preProcessImageUrl","source","imageInfoOfBase64","scale","Math","contentSize","resized","resizeBase64ImageToJpeg","prepared","padBase64ImageToMinimumSize","assert","Buffer","parseBase64","mapJevBox","box","image","targetSize","left","top","right","bottom","center"],"mappings":";;;AAWO,eAAeA,gBAAgBC,GAAW;IAC/C,MAAMC,SAAS,MAAMC,mBAAmBF,KAAK;IAC7C,MAAMG,SAAS,MAAMC,kBAAkBH;IACvC,MAAMI,QAAQC,KAAK,GAAG,CAAC,GAAG,OAAOH,OAAO,KAAK,EAAE,OAAOA,OAAO,MAAM;IACnE,MAAMI,cAAc;QAClB,OAAOD,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,OAAO,KAAK,GAAGE;QAC7C,QAAQC,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,OAAO,MAAM,GAAGE;IACjD;IACA,MAAMG,UAAU,MAAMC,wBAAwBR,QAAQ;QACpD,YAAYM;IACd;IACA,MAAMG,WAAW,MAAMC,4BAA4BH,SAAS;IAC5DI,OACEC,OAAO,IAAI,CAACC,YAAYJ,SAAS,WAAW,EAAE,IAAI,EAAE,UAAU,MAAM,IAClE,UACF;IAEF,OAAO;QACL,GAAGA,QAAQ;QACXH;QACA,YAAY;YAAE,OAAOJ,OAAO,KAAK;YAAE,QAAQA,OAAO,MAAM;QAAC;IAC3D;AACF;AAEO,SAASY,UACdC,GAAiE,EACjEC,KAAkD,EAClDC,UAA6C;IAI7C,MAAMC,OACHH,IAAI,IAAI,GAAGC,MAAM,KAAK,GAAGC,WAAW,KAAK,GAAID,MAAM,WAAW,CAAC,KAAK;IACvE,MAAMG,MACHJ,IAAI,GAAG,GAAGC,MAAM,MAAM,GAAGC,WAAW,MAAM,GAAID,MAAM,WAAW,CAAC,MAAM;IACzE,MAAMI,QACHL,IAAI,KAAK,GAAGC,MAAM,KAAK,GAAGC,WAAW,KAAK,GAAID,MAAM,WAAW,CAAC,KAAK;IACxE,MAAMK,SACHN,IAAI,MAAM,GAAGC,MAAM,MAAM,GAAGC,WAAW,MAAM,GAAID,MAAM,WAAW,CAAC,MAAM;IAC5E,MAAMM,SAA2B;QAAEJ,CAAAA,OAAOE,KAAI,IAAK;QAAID,CAAAA,MAAME,MAAK,IAAK;KAAE;IACzEV,OACEW,MAAM,CAAC,EAAE,IAAI,KACXA,MAAM,CAAC,EAAE,GAAGL,WAAW,KAAK,IAC5BK,MAAM,CAAC,EAAE,IAAI,KACbA,MAAM,CAAC,EAAE,GAAGL,WAAW,MAAM,EAC/B;IAEF,OAAO;QACLK;QACA,MAAM;YAAEJ;YAAMC;YAAK,OAAOC,QAAQF;YAAM,QAAQG,SAASF;QAAI;IAC/D;AACF"}

package/dist/es/ai-model/models/jev/planning.mjs

@@ -0,0 +1,168 @@
+import { buildYamlFlowFromPlans } from "../../../common.mjs";
+import { assert } from "@midscene/shared/utils";
+import { callAI } from "../../service-caller/index.mjs";
+import { systemOneResponseSchema } from "../../service-caller/system-one.mjs";
+import { standardPlan } from "../../workflows/planning/standard-planning.mjs";
+import { prepareJevImage } from "./image.mjs";
+const jevPlan = async (instruction, options)=>{
+    const history = options.conversationHistory;
+    const delegate = async (reason)=>{
+        assert(options.getDefaultModelRuntime, 'Default model runtime is required for Jev planning delegation');
+        const base = options.getDefaultModelRuntime();
+        assert('jev' !== base.config.modelFamily, 'Jev cannot delegate planning to itself');
+        const runtime = {
+            ...base,
+            config: {
+                ...base.config,
+                intent: 'planning'
+            },
+            executionId: options.modelRuntime.executionId,
+            onUsage: base.onUsage ? (usage)=>base.onUsage?.({
+                    ...usage,
+                    intent: 'planning'
+                }) : void 0,
+            modelRouting: {
+                requestedModel: options.modelRuntime.config.modelName,
+                requestedSlot: options.modelRuntime.config.slot,
+                task: 'planning',
+                reason
+            }
+        };
+        const planner = 'custom' === runtime.adapter.planning.kind ? runtime.adapter.planning.planFn : standardPlan;
+        return planner(instruction, {
+            ...options,
+            modelRuntime: runtime,
+            includeLocateInPlanning: false
+        });
+    };
+    const image = await prepareJevImage(options.context.screenshot.base64);
+    const references = await Promise.all(instruction.referenceImages.map(async (reference, index)=>({
+            key: `reference_${index}`,
+            name: reference.name,
+            image: await prepareJevImage(reference.url)
+        })));
+    const state = {
+        goal: instruction.text,
+        context: options.actionContext,
+        history: history.historicalLogsToText(),
+        memory: history.memoriesToText(),
+        feedback: history.pendingFeedbackMessage,
+        references: references.map(({ key, name })=>({
+                image: `\`${key}\``,
+                name
+            })),
+        availableActions: options.actionSpace.map(({ name, description })=>({
+                name,
+                description
+            }))
+    };
+    const criteria = {
+        Complete: 'The entire user goal is already satisfied, confirmed by the current screenshot and execution feedback. No text answer is requested.',
+        Delegate: 'The next step requires text input, generated output, an action or parameters not covered by the other choices. Also use for impossible tasks needing explanation.'
+    };
+    for (const action of options.actionSpace)if ('Tap' === action.name || 'Scroll' === action.name || 'Sleep' === action.name) criteria[action.name] = 'Sleep' === action.name ? 'Wait 800 ms for a transient loading state.' : action.description ?? action.name;
+    const response = await callAI([], options.modelRuntime, {
+        abortSignal: options.abortSignal,
+        systemOneRequest: {
+            state,
+            images: {
+                screenshot: image.imageBase64,
+                ...Object.fromEntries(references.map(({ key, image })=>[
+                        key,
+                        image.imageBase64
+                    ]))
+            },
+            questions: {
+                action: {
+                    type: 'choice',
+                    instructions: 'Choose exactly the next step toward `goal` using the current `screenshot`, context, history and feedback. History records proposed actions; feedback and current screenshot establish what actually happened. Do not repeat completed steps. Reference images describe targets, not the current page.',
+                    criteria
+                },
+                ...criteria.Scroll ? {
+                    direction: {
+                        type: 'choice',
+                        instructions: 'If scrolling is the next step toward `goal`, which direction reveals the desired off-screen content? Evaluate from the screenshot and history independently of the action question.',
+                        criteria: {
+                            down: 'Reveal content below',
+                            up: 'Reveal content above',
+                            left: 'Reveal content to the left',
+                            right: 'Reveal content to the right'
+                        }
+                    }
+                } : {}
+            }
+        }
+    });
+    const { answers } = systemOneResponseSchema.parse(JSON.parse(response.content));
+    assert('choice' === answers.action.type, 'Invalid Jev planning action');
+    const selected = answers.action.choice;
+    if ('Delegate' === selected) return delegate('The next step requires parameters or output that Jev cannot generate');
+    if (('Tap' === selected || 'Scroll' === selected) && options.locateModelRuntime?.config.modelFamily !== 'jev') return delegate("The configured Locate model requires a concrete target description from a general-purpose planner");
+    const locate = {
+        prompt: `Locate the single ${'Scroll' === selected ? 'scrollable region to scroll' : 'element to tap'} for the NEXT step toward this goal. Use the current screenshot as authoritative. Previous proposals may have failed; use feedback. Task context: ${JSON.stringify(state)}`,
+        images: instruction.referenceImages
+    };
+    const actions = (()=>{
+        if ('Complete' === selected) return [];
+        if ('Sleep' === selected) return [
+            {
+                type: 'Sleep',
+                param: {
+                    timeMs: 800
+                }
+            }
+        ];
+        if ('Tap' === selected) return [
+            {
+                type: 'Tap',
+                param: {
+                    locate
+                }
+            }
+        ];
+        assert('Scroll' === selected && answers.direction?.type === 'choice', 'Invalid Jev scroll action');
+        return [
+            {
+                type: 'Scroll',
+                param: {
+                    locate,
+                    direction: answers.direction.choice,
+                    scrollType: 'singleAction'
+                }
+            }
+        ];
+    })();
+    const log = `Jev selected ${selected}`;
+    const feedback = history.pendingFeedbackMessage;
+    history.resetPendingFeedbackMessageIfExists();
+    history.appendMessage({
+        role: 'user',
+        content: `Goal: ${instruction.text}\nExecution feedback: ${feedback}`
+    });
+    history.appendMessage({
+        role: 'assistant',
+        content: JSON.stringify({
+            actions,
+            log
+        })
+    });
+    history.appendHistoricalLog(`${log}: ${JSON.stringify(actions.map(({ type, param })=>({
+            type,
+            direction: param?.direction
+        })))}`);
+    history.compressHistory(50, 20);
+    return {
+        actions,
+        log,
+        rawResponse: response.content,
+        usage: response.usage,
+        shouldContinuePlanning: 'Complete' !== selected,
+        ...'Complete' === selected ? {
+            finalizeSuccess: true
+        } : {},
+        yamlFlow: buildYamlFlowFromPlans(actions, options.actionSpace)
+    };
+};
+export { jevPlan };
+
+//# sourceMappingURL=planning.mjs.map

package/dist/es/ai-model/models/jev/planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/planning.mjs","sources":["../../../../../src/ai-model/models/jev/planning.ts"],"sourcesContent":["import { buildYamlFlowFromPlans } from '@/common';\nimport type { AIUsageInfo, PlanningAction } from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport { callAI } from '../../service-caller';\nimport { systemOneResponseSchema } from '../../service-caller/system-one';\nimport { standardPlan } from '../../workflows/planning/standard-planning';\nimport type { PlanFn } from '../../workflows/planning/types';\nimport { prepareJevImage } from './image';\n\nexport const jevPlan: PlanFn = async (instruction, options) => {\n  const history = options.conversationHistory;\n  const delegate = async (reason: string) => {\n    assert(\n      options.getDefaultModelRuntime,\n      'Default model runtime is required for Jev planning delegation',\n    );\n    const base = options.getDefaultModelRuntime();\n    assert(\n      base.config.modelFamily !== 'jev',\n      'Jev cannot delegate planning to itself',\n    );\n    const runtime = {\n      ...base,\n      config: { ...base.config, intent: 'planning' as const },\n      executionId: options.modelRuntime.executionId,\n      onUsage: base.onUsage\n        ? (usage: AIUsageInfo) =>\n            base.onUsage?.({ ...usage, intent: 'planning' })\n        : undefined,\n      modelRouting: {\n        requestedModel: options.modelRuntime.config.modelName,\n        requestedSlot: options.modelRuntime.config.slot,\n        task: 'planning' as const,\n        reason,\n      },\n    };\n    const planner =\n      runtime.adapter.planning.kind === 'custom'\n        ? runtime.adapter.planning.planFn\n        : standardPlan;\n    return planner(instruction, {\n      ...options,\n      modelRuntime: runtime,\n      includeLocateInPlanning: false,\n    });\n  };\n  const image = await prepareJevImage(options.context.screenshot.base64);\n  const references = await Promise.all(\n    instruction.referenceImages.map(async (reference, index) => ({\n      key: `reference_${index}`,\n      name: reference.name,\n      image: await prepareJevImage(reference.url),\n    })),\n  );\n  const state = {\n    goal: instruction.text,\n    context: options.actionContext,\n    history: history.historicalLogsToText(),\n    memory: history.memoriesToText(),\n    feedback: history.pendingFeedbackMessage,\n    references: references.map(({ key, name }) => ({\n      image: `\\`${key}\\``,\n      name,\n    })),\n    availableActions: options.actionSpace.map(({ name, description }) => ({\n      name,\n      description,\n    })),\n  };\n  const criteria: Record<string, string> = {\n    Complete:\n      'The entire user goal is already satisfied, confirmed by the current screenshot and execution feedback. No text answer is requested.',\n    Delegate:\n      'The next step requires text input, generated output, an action or parameters not covered by the other choices. Also use for impossible tasks needing explanation.',\n  };\n  for (const action of options.actionSpace) {\n    if (\n      action.name === 'Tap' ||\n      action.name === 'Scroll' ||\n      action.name === 'Sleep'\n    ) {\n      criteria[action.name] =\n        action.name === 'Sleep'\n          ? 'Wait 800 ms for a transient loading state.'\n          : (action.description ?? action.name);\n    }\n  }\n  const response = await callAI([], options.modelRuntime, {\n    abortSignal: options.abortSignal,\n    systemOneRequest: {\n      state,\n      images: {\n        screenshot: image.imageBase64,\n        ...Object.fromEntries(\n          references.map(({ key, image }) => [key, image.imageBase64]),\n        ),\n      },\n      questions: {\n        action: {\n          type: 'choice',\n          instructions:\n            'Choose exactly the next step toward `goal` using the current `screenshot`, context, history and feedback. History records proposed actions; feedback and current screenshot establish what actually happened. Do not repeat completed steps. Reference images describe targets, not the current page.',\n          criteria,\n        },\n        ...(criteria.Scroll\n          ? {\n              direction: {\n                type: 'choice' as const,\n                instructions:\n                  'If scrolling is the next step toward `goal`, which direction reveals the desired off-screen content? Evaluate from the screenshot and history independently of the action question.',\n                criteria: {\n                  down: 'Reveal content below',\n                  up: 'Reveal content above',\n                  left: 'Reveal content to the left',\n                  right: 'Reveal content to the right',\n                },\n              },\n            }\n          : {}),\n      },\n    },\n  });\n  const { answers } = systemOneResponseSchema.parse(\n    JSON.parse(response.content),\n  );\n  assert(answers.action.type === 'choice', 'Invalid Jev planning action');\n  const selected = answers.action.choice;\n  if (selected === 'Delegate') {\n    return delegate(\n      'The next step requires parameters or output that Jev cannot generate',\n    );\n  }\n  if (\n    (selected === 'Tap' || selected === 'Scroll') &&\n    options.locateModelRuntime?.config.modelFamily !== 'jev'\n  ) {\n    return delegate(\n      'The configured Locate model requires a concrete target description from a general-purpose planner',\n    );\n  }\n  // The normal Locate stage resolves this contextual target with its own runtime.\n  // Do not embed planning coordinates or bypass the configured Locate model.\n  const locate = {\n    prompt: `Locate the single ${selected === 'Scroll' ? 'scrollable region to scroll' : 'element to tap'} for the NEXT step toward this goal. Use the current screenshot as authoritative. Previous proposals may have failed; use feedback. Task context: ${JSON.stringify(state)}`,\n    images: instruction.referenceImages,\n  };\n  const actions: PlanningAction[] = (() => {\n    if (selected === 'Complete') {\n      return [];\n    }\n    if (selected === 'Sleep') {\n      return [{ type: 'Sleep', param: { timeMs: 800 } }];\n    }\n    if (selected === 'Tap') {\n      return [{ type: 'Tap', param: { locate } }];\n    }\n    assert(\n      selected === 'Scroll' && answers.direction?.type === 'choice',\n      'Invalid Jev scroll action',\n    );\n    return [\n      {\n        type: 'Scroll',\n        param: {\n          locate,\n          direction: answers.direction.choice,\n          scrollType: 'singleAction',\n        },\n      },\n    ];\n  })();\n  const log = `Jev selected ${selected}`;\n  const feedback = history.pendingFeedbackMessage;\n  history.resetPendingFeedbackMessageIfExists();\n  history.appendMessage({\n    role: 'user',\n    content: `Goal: ${instruction.text}\\nExecution feedback: ${feedback}`,\n  });\n  history.appendMessage({\n    role: 'assistant',\n    content: JSON.stringify({ actions, log }),\n  });\n  history.appendHistoricalLog(\n    `${log}: ${JSON.stringify(actions.map(({ type, param }) => ({ type, direction: param?.direction })))}`,\n  );\n  history.compressHistory(50, 20);\n  return {\n    actions,\n    log,\n    rawResponse: response.content,\n    usage: response.usage,\n    shouldContinuePlanning: selected !== 'Complete',\n    ...(selected === 'Complete' ? { finalizeSuccess: true } : {}),\n    yamlFlow: buildYamlFlowFromPlans(actions, options.actionSpace),\n  };\n};\n"],"names":["jevPlan","instruction","options","history","delegate","reason","assert","base","runtime","usage","undefined","planner","standardPlan","image","prepareJevImage","references","Promise","reference","index","state","key","name","description","criteria","action","response","callAI","Object","answers","systemOneResponseSchema","JSON","selected","locate","actions","log","feedback","type","param","buildYamlFlowFromPlans"],"mappings":";;;;;;AASO,MAAMA,UAAkB,OAAOC,aAAaC;IACjD,MAAMC,UAAUD,QAAQ,mBAAmB;IAC3C,MAAME,WAAW,OAAOC;QACtBC,OACEJ,QAAQ,sBAAsB,EAC9B;QAEF,MAAMK,OAAOL,QAAQ,sBAAsB;QAC3CI,OACEC,AAA4B,UAA5BA,KAAK,MAAM,CAAC,WAAW,EACvB;QAEF,MAAMC,UAAU;YACd,GAAGD,IAAI;YACP,QAAQ;gBAAE,GAAGA,KAAK,MAAM;gBAAE,QAAQ;YAAoB;YACtD,aAAaL,QAAQ,YAAY,CAAC,WAAW;YAC7C,SAASK,KAAK,OAAO,GACjB,CAACE,QACCF,KAAK,OAAO,GAAG;oBAAE,GAAGE,KAAK;oBAAE,QAAQ;gBAAW,KAChDC;YACJ,cAAc;gBACZ,gBAAgBR,QAAQ,YAAY,CAAC,MAAM,CAAC,SAAS;gBACrD,eAAeA,QAAQ,YAAY,CAAC,MAAM,CAAC,IAAI;gBAC/C,MAAM;gBACNG;YACF;QACF;QACA,MAAMM,UACJH,AAAkC,aAAlCA,QAAQ,OAAO,CAAC,QAAQ,CAAC,IAAI,GACzBA,QAAQ,OAAO,CAAC,QAAQ,CAAC,MAAM,GAC/BI;QACN,OAAOD,QAAQV,aAAa;YAC1B,GAAGC,OAAO;YACV,cAAcM;YACd,yBAAyB;QAC3B;IACF;IACA,MAAMK,QAAQ,MAAMC,gBAAgBZ,QAAQ,OAAO,CAAC,UAAU,CAAC,MAAM;IACrE,MAAMa,aAAa,MAAMC,QAAQ,GAAG,CAClCf,YAAY,eAAe,CAAC,GAAG,CAAC,OAAOgB,WAAWC,QAAW;YAC3D,KAAK,CAAC,UAAU,EAAEA,OAAO;YACzB,MAAMD,UAAU,IAAI;YACpB,OAAO,MAAMH,gBAAgBG,UAAU,GAAG;QAC5C;IAEF,MAAME,QAAQ;QACZ,MAAMlB,YAAY,IAAI;QACtB,SAASC,QAAQ,aAAa;QAC9B,SAASC,QAAQ,oBAAoB;QACrC,QAAQA,QAAQ,cAAc;QAC9B,UAAUA,QAAQ,sBAAsB;QACxC,YAAYY,WAAW,GAAG,CAAC,CAAC,EAAEK,GAAG,EAAEC,IAAI,EAAE,GAAM;gBAC7C,OAAO,CAAC,EAAE,EAAED,IAAI,EAAE,CAAC;gBACnBC;YACF;QACA,kBAAkBnB,QAAQ,WAAW,CAAC,GAAG,CAAC,CAAC,EAAEmB,IAAI,EAAEC,WAAW,EAAE,GAAM;gBACpED;gBACAC;YACF;IACF;IACA,MAAMC,WAAmC;QACvC,UACE;QACF,UACE;IACJ;IACA,KAAK,MAAMC,UAAUtB,QAAQ,WAAW,CACtC,IACEsB,AAAgB,UAAhBA,OAAO,IAAI,IACXA,AAAgB,aAAhBA,OAAO,IAAI,IACXA,AAAgB,YAAhBA,OAAO,IAAI,EAEXD,QAAQ,CAACC,OAAO,IAAI,CAAC,GACnBA,AAAgB,YAAhBA,OAAO,IAAI,GACP,+CACCA,OAAO,WAAW,IAAIA,OAAO,IAAI;IAG5C,MAAMC,WAAW,MAAMC,OAAO,EAAE,EAAExB,QAAQ,YAAY,EAAE;QACtD,aAAaA,QAAQ,WAAW;QAChC,kBAAkB;YAChBiB;YACA,QAAQ;gBACN,YAAYN,MAAM,WAAW;gBAC7B,GAAGc,OAAO,WAAW,CACnBZ,WAAW,GAAG,CAAC,CAAC,EAAEK,GAAG,EAAEP,KAAK,EAAE,GAAK;wBAACO;wBAAKP,MAAM,WAAW;qBAAC,EAC5D;YACH;YACA,WAAW;gBACT,QAAQ;oBACN,MAAM;oBACN,cACE;oBACFU;gBACF;gBACA,GAAIA,SAAS,MAAM,GACf;oBACE,WAAW;wBACT,MAAM;wBACN,cACE;wBACF,UAAU;4BACR,MAAM;4BACN,IAAI;4BACJ,MAAM;4BACN,OAAO;wBACT;oBACF;gBACF,IACA,CAAC,CAAC;YACR;QACF;IACF;IACA,MAAM,EAAEK,OAAO,EAAE,GAAGC,wBAAwB,KAAK,CAC/CC,KAAK,KAAK,CAACL,SAAS,OAAO;IAE7BnB,OAAOsB,AAAwB,aAAxBA,QAAQ,MAAM,CAAC,IAAI,EAAe;IACzC,MAAMG,WAAWH,QAAQ,MAAM,CAAC,MAAM;IACtC,IAAIG,AAAa,eAAbA,UACF,OAAO3B,SACL;IAGJ,IACG2B,AAAAA,CAAAA,AAAa,UAAbA,YAAsBA,AAAa,aAAbA,QAAoB,KAC3C7B,QAAQ,kBAAkB,EAAE,OAAO,gBAAgB,OAEnD,OAAOE,SACL;IAKJ,MAAM4B,SAAS;QACb,QAAQ,CAAC,kBAAkB,EAAED,AAAa,aAAbA,WAAwB,gCAAgC,iBAAiB,kJAAkJ,EAAED,KAAK,SAAS,CAACX,QAAQ;QACjR,QAAQlB,YAAY,eAAe;IACrC;IACA,MAAMgC,UAA6B,AAAC;QAClC,IAAIF,AAAa,eAAbA,UACF,OAAO,EAAE;QAEX,IAAIA,AAAa,YAAbA,UACF,OAAO;YAAC;gBAAE,MAAM;gBAAS,OAAO;oBAAE,QAAQ;gBAAI;YAAE;SAAE;QAEpD,IAAIA,AAAa,UAAbA,UACF,OAAO;YAAC;gBAAE,MAAM;gBAAO,OAAO;oBAAEC;gBAAO;YAAE;SAAE;QAE7C1B,OACEyB,AAAa,aAAbA,YAAyBH,QAAQ,SAAS,EAAE,SAAS,UACrD;QAEF,OAAO;YACL;gBACE,MAAM;gBACN,OAAO;oBACLI;oBACA,WAAWJ,QAAQ,SAAS,CAAC,MAAM;oBACnC,YAAY;gBACd;YACF;SACD;IACH;IACA,MAAMM,MAAM,CAAC,aAAa,EAAEH,UAAU;IACtC,MAAMI,WAAWhC,QAAQ,sBAAsB;IAC/CA,QAAQ,mCAAmC;IAC3CA,QAAQ,aAAa,CAAC;QACpB,MAAM;QACN,SAAS,CAAC,MAAM,EAAEF,YAAY,IAAI,CAAC,sBAAsB,EAAEkC,UAAU;IACvE;IACAhC,QAAQ,aAAa,CAAC;QACpB,MAAM;QACN,SAAS2B,KAAK,SAAS,CAAC;YAAEG;YAASC;QAAI;IACzC;IACA/B,QAAQ,mBAAmB,CACzB,GAAG+B,IAAI,EAAE,EAAEJ,KAAK,SAAS,CAACG,QAAQ,GAAG,CAAC,CAAC,EAAEG,IAAI,EAAEC,KAAK,EAAE,GAAM;YAAED;YAAM,WAAWC,OAAO;QAAU,MAAM;IAExGlC,QAAQ,eAAe,CAAC,IAAI;IAC5B,OAAO;QACL8B;QACAC;QACA,aAAaT,SAAS,OAAO;QAC7B,OAAOA,SAAS,KAAK;QACrB,wBAAwBM,AAAa,eAAbA;QACxB,GAAIA,AAAa,eAAbA,WAA0B;YAAE,iBAAiB;QAAK,IAAI,CAAC,CAAC;QAC5D,UAAUO,uBAAuBL,SAAS/B,QAAQ,WAAW;IAC/D;AACF"}

package/dist/es/ai-model/models/registry.mjs

@@ -7,11 +7,13 @@ import { doubaoAdapters } from "./doubao.mjs";
 import { geminiAdapters } from "./gemini.mjs";
 import { glmAdapters } from "./glm.mjs";
 import { gptAdapters } from "./gpt.mjs";
+import { jevAdapters } from "./jev/adapter.mjs";
 import { kimiAdapters } from "./kimi.mjs";
 import { mimoAdapters } from "./mimo.mjs";
 import { qwenAdapters } from "./qwen.mjs";
 import { uiTarsAdapters } from "./ui-tars/adapter.mjs";
 const MODEL_ADAPTER_CONFIGS = {
+    ...jevAdapters,
     ...qwenAdapters,
     ...deepSeekAdapters,
     ...doubaoAdapters,

package/dist/es/ai-model/models/registry.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/registry.mjs","sources":["../../../../src/ai-model/models/registry.ts"],"sourcesContent":["import type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ResolvedModelAdapter } from '../model-adapter/resolve';\nimport type {\n  ModelAdapter,\n  ModelAdapterDefinition,\n  ModelRuntime,\n} from '../model-adapter/types';\nimport { autoGlmAdapters } from './auto-glm/adapter';\nimport { deepSeekAdapters } from './deepseek/adapter';\nimport { defaultOpenAICompatibleAdapterConfig } from './default';\nimport { doubaoAdapters } from './doubao';\nimport { geminiAdapters } from './gemini';\nimport { glmAdapters } from './glm';\nimport { gptAdapters } from './gpt';\nimport { kimiAdapters } from './kimi';\nimport { mimoAdapters } from './mimo';\nimport { qwenAdapters } from './qwen';\nimport { uiTarsAdapters } from './ui-tars/adapter';\n\nexport const MODEL_ADAPTER_CONFIGS = {\n  ...qwenAdapters,\n  ...deepSeekAdapters,\n  ...doubaoAdapters,\n  ...geminiAdapters,\n  ...uiTarsAdapters,\n  ...glmAdapters,\n  ...autoGlmAdapters,\n  ...gptAdapters,\n  ...kimiAdapters,\n  ...mimoAdapters,\n} satisfies Record<TModelFamily, ModelAdapterDefinition>;\n\ntype ModelAdapterCacheKey = TModelFamily | 'default';\n\nconst modelAdapterCache = new Map<ModelAdapterCacheKey, ModelAdapter>();\nconst debugModelAdapter = getDebug('ai:model-adapter');\n\nfunction debugAdapterUnsupportedUserConfig(\n  modelFamily: ModelAdapterCacheKey,\n  adapter: ModelAdapter,\n): void {\n  if (adapter.chatCompletion.unsupportedUserConfig.length > 0) {\n    debugModelAdapter(\n      `model adapter \"${modelFamily}\" (chat-completion) unsupportedUserConfig: ${JSON.stringify(adapter.chatCompletion.unsupportedUserConfig)}`,\n    );\n  }\n  if (adapter.responses.unsupportedUserConfig.length > 0) {\n    debugModelAdapter(\n      `model adapter \"${modelFamily}\" (responses) unsupportedUserConfig: ${JSON.stringify(adapter.responses.unsupportedUserConfig)}`,\n    );\n  }\n}\n\nexport function getModelAdapter(modelFamily?: TModelFamily): ModelAdapter {\n  const cacheKey: ModelAdapterCacheKey = modelFamily ?? 'default';\n  let adapter = modelAdapterCache.get(cacheKey);\n  if (adapter) {\n    return adapter;\n  }\n\n  const config = modelFamily\n    ? MODEL_ADAPTER_CONFIGS[modelFamily]\n    : defaultOpenAICompatibleAdapterConfig;\n  if (!config) {\n    throw new Error(\n      `No model adapter registered for modelFamily: ${modelFamily}`,\n    );\n  }\n\n  adapter = new ResolvedModelAdapter(config, cacheKey);\n  modelAdapterCache.set(cacheKey, adapter);\n  debugAdapterUnsupportedUserConfig(cacheKey, adapter);\n\n  return adapter;\n}\n\nexport function getModelRuntime(config: IModelConfig): ModelRuntime {\n  return {\n    config,\n    adapter: getModelAdapter(config.modelFamily),\n  };\n}\n"],"names":["MODEL_ADAPTER_CONFIGS","qwenAdapters","deepSeekAdapters","doubaoAdapters","geminiAdapters","uiTarsAdapters","glmAdapters","autoGlmAdapters","gptAdapters","kimiAdapters","mimoAdapters","modelAdapterCache","Map","debugModelAdapter","getDebug","debugAdapterUnsupportedUserConfig","modelFamily","adapter","JSON","getModelAdapter","cacheKey","config","defaultOpenAICompatibleAdapterConfig","Error","ResolvedModelAdapter","getModelRuntime"],"mappings":";;;;;;;;;;;;;AAoBO,MAAMA,wBAAwB;IACnC,GAAGC,YAAY;IACf,GAAGC,gBAAgB;IACnB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,WAAW;IACd,GAAGC,eAAe;IAClB,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,YAAY;AACjB;AAIA,MAAMC,oBAAoB,IAAIC;AAC9B,MAAMC,oBAAoBC,SAAS;AAEnC,SAASC,kCACPC,WAAiC,EACjCC,OAAqB;IAErB,IAAIA,QAAQ,cAAc,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACxDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,2CAA2C,EAAEE,KAAK,SAAS,CAACD,QAAQ,cAAc,CAAC,qBAAqB,GAAG;IAG7I,IAAIA,QAAQ,SAAS,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACnDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,qCAAqC,EAAEE,KAAK,SAAS,CAACD,QAAQ,SAAS,CAAC,qBAAqB,GAAG;AAGpI;AAEO,SAASE,gBAAgBH,WAA0B;IACxD,MAAMI,WAAiCJ,eAAe;IACtD,IAAIC,UAAUN,kBAAkB,GAAG,CAACS;IACpC,IAAIH,SACF,OAAOA;IAGT,MAAMI,SAASL,cACXhB,qBAAqB,CAACgB,YAAY,GAClCM;IACJ,IAAI,CAACD,QACH,MAAM,IAAIE,MACR,CAAC,6CAA6C,EAAEP,aAAa;IAIjEC,UAAU,IAAIO,qBAAqBH,QAAQD;IAC3CT,kBAAkB,GAAG,CAACS,UAAUH;IAChCF,kCAAkCK,UAAUH;IAE5C,OAAOA;AACT;AAEO,SAASQ,gBAAgBJ,MAAoB;IAClD,OAAO;QACLA;QACA,SAASF,gBAAgBE,OAAO,WAAW;IAC7C;AACF"}
+{"version":3,"file":"ai-model/models/registry.mjs","sources":["../../../../src/ai-model/models/registry.ts"],"sourcesContent":["import type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ResolvedModelAdapter } from '../model-adapter/resolve';\nimport type {\n  ModelAdapter,\n  ModelAdapterDefinition,\n  ModelRuntime,\n} from '../model-adapter/types';\nimport { autoGlmAdapters } from './auto-glm/adapter';\nimport { deepSeekAdapters } from './deepseek/adapter';\nimport { defaultOpenAICompatibleAdapterConfig } from './default';\nimport { doubaoAdapters } from './doubao';\nimport { geminiAdapters } from './gemini';\nimport { glmAdapters } from './glm';\nimport { gptAdapters } from './gpt';\nimport { jevAdapters } from './jev/adapter';\nimport { kimiAdapters } from './kimi';\nimport { mimoAdapters } from './mimo';\nimport { qwenAdapters } from './qwen';\nimport { uiTarsAdapters } from './ui-tars/adapter';\n\nexport const MODEL_ADAPTER_CONFIGS = {\n  ...jevAdapters,\n  ...qwenAdapters,\n  ...deepSeekAdapters,\n  ...doubaoAdapters,\n  ...geminiAdapters,\n  ...uiTarsAdapters,\n  ...glmAdapters,\n  ...autoGlmAdapters,\n  ...gptAdapters,\n  ...kimiAdapters,\n  ...mimoAdapters,\n} satisfies Record<TModelFamily, ModelAdapterDefinition>;\n\ntype ModelAdapterCacheKey = TModelFamily | 'default';\n\nconst modelAdapterCache = new Map<ModelAdapterCacheKey, ModelAdapter>();\nconst debugModelAdapter = getDebug('ai:model-adapter');\n\nfunction debugAdapterUnsupportedUserConfig(\n  modelFamily: ModelAdapterCacheKey,\n  adapter: ModelAdapter,\n): void {\n  if (adapter.chatCompletion.unsupportedUserConfig.length > 0) {\n    debugModelAdapter(\n      `model adapter \"${modelFamily}\" (chat-completion) unsupportedUserConfig: ${JSON.stringify(adapter.chatCompletion.unsupportedUserConfig)}`,\n    );\n  }\n  if (adapter.responses.unsupportedUserConfig.length > 0) {\n    debugModelAdapter(\n      `model adapter \"${modelFamily}\" (responses) unsupportedUserConfig: ${JSON.stringify(adapter.responses.unsupportedUserConfig)}`,\n    );\n  }\n}\n\nexport function getModelAdapter(modelFamily?: TModelFamily): ModelAdapter {\n  const cacheKey: ModelAdapterCacheKey = modelFamily ?? 'default';\n  let adapter = modelAdapterCache.get(cacheKey);\n  if (adapter) {\n    return adapter;\n  }\n\n  const config = modelFamily\n    ? MODEL_ADAPTER_CONFIGS[modelFamily]\n    : defaultOpenAICompatibleAdapterConfig;\n  if (!config) {\n    throw new Error(\n      `No model adapter registered for modelFamily: ${modelFamily}`,\n    );\n  }\n\n  adapter = new ResolvedModelAdapter(config, cacheKey);\n  modelAdapterCache.set(cacheKey, adapter);\n  debugAdapterUnsupportedUserConfig(cacheKey, adapter);\n\n  return adapter;\n}\n\nexport function getModelRuntime(config: IModelConfig): ModelRuntime {\n  return {\n    config,\n    adapter: getModelAdapter(config.modelFamily),\n  };\n}\n"],"names":["MODEL_ADAPTER_CONFIGS","jevAdapters","qwenAdapters","deepSeekAdapters","doubaoAdapters","geminiAdapters","uiTarsAdapters","glmAdapters","autoGlmAdapters","gptAdapters","kimiAdapters","mimoAdapters","modelAdapterCache","Map","debugModelAdapter","getDebug","debugAdapterUnsupportedUserConfig","modelFamily","adapter","JSON","getModelAdapter","cacheKey","config","defaultOpenAICompatibleAdapterConfig","Error","ResolvedModelAdapter","getModelRuntime"],"mappings":";;;;;;;;;;;;;;AAqBO,MAAMA,wBAAwB;IACnC,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,gBAAgB;IACnB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,WAAW;IACd,GAAGC,eAAe;IAClB,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,YAAY;AACjB;AAIA,MAAMC,oBAAoB,IAAIC;AAC9B,MAAMC,oBAAoBC,SAAS;AAEnC,SAASC,kCACPC,WAAiC,EACjCC,OAAqB;IAErB,IAAIA,QAAQ,cAAc,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACxDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,2CAA2C,EAAEE,KAAK,SAAS,CAACD,QAAQ,cAAc,CAAC,qBAAqB,GAAG;IAG7I,IAAIA,QAAQ,SAAS,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACnDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,qCAAqC,EAAEE,KAAK,SAAS,CAACD,QAAQ,SAAS,CAAC,qBAAqB,GAAG;AAGpI;AAEO,SAASE,gBAAgBH,WAA0B;IACxD,MAAMI,WAAiCJ,eAAe;IACtD,IAAIC,UAAUN,kBAAkB,GAAG,CAACS;IACpC,IAAIH,SACF,OAAOA;IAGT,MAAMI,SAASL,cACXjB,qBAAqB,CAACiB,YAAY,GAClCM;IACJ,IAAI,CAACD,QACH,MAAM,IAAIE,MACR,CAAC,6CAA6C,EAAEP,aAAa;IAIjEC,UAAU,IAAIO,qBAAqBH,QAAQD;IAC3CT,kBAAkB,GAAG,CAACS,UAAUH;IAChCF,kCAAkCK,UAAUH;IAE5C,OAAOA;AACT;AAEO,SAASQ,gBAAgBJ,MAAoB;IAClD,OAAO;QACLA;QACA,SAASF,gBAAgBE,OAAO,WAAW;IAC7C;AACF"}

package/dist/es/ai-model/models/task-routing.mjs

@@ -0,0 +1,28 @@
+import { getDebug } from "@midscene/shared/logger";
+import { getModelRuntime } from "./registry.mjs";
+function resolveModelForTask(configs, intent, task) {
+    const requested = getModelRuntime(configs.getModelConfig(intent));
+    if (!requested.adapter.supportedTasks || requested.adapter.supportedTasks.includes(task)) return requested;
+    const config = configs.getModelConfig('default');
+    const runtime = getModelRuntime({
+        ...config,
+        intent
+    });
+    if (runtime.adapter.supportedTasks && !runtime.adapter.supportedTasks.includes(task)) throw new Error(`The default model cannot perform ${task}. Configure a general-purpose default model.`);
+    const reason = `${requested.config.modelFamily} does not support ${task}; using the default model`;
+    getDebug('ai:model-routing', {
+        console: true
+    })(reason);
+    return {
+        ...runtime,
+        modelRouting: {
+            requestedModel: requested.config.modelName,
+            requestedSlot: requested.config.slot,
+            task,
+            reason
+        }
+    };
+}
+export { resolveModelForTask };
+
+//# sourceMappingURL=task-routing.mjs.map

package/dist/es/ai-model/models/task-routing.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/task-routing.mjs","sources":["../../../../src/ai-model/models/task-routing.ts"],"sourcesContent":["import type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ModelRuntime, ModelTask } from '../model-adapter/types';\nimport { getModelRuntime } from './registry';\n\nexport function resolveModelForTask(\n  configs: { getModelConfig(intent: TIntent): IModelConfig },\n  intent: TIntent,\n  task: ModelTask,\n): ModelRuntime {\n  const requested = getModelRuntime(configs.getModelConfig(intent));\n  if (\n    !requested.adapter.supportedTasks ||\n    requested.adapter.supportedTasks.includes(task)\n  ) {\n    return requested;\n  }\n  const config = configs.getModelConfig('default');\n  const runtime = getModelRuntime({ ...config, intent });\n  if (\n    runtime.adapter.supportedTasks &&\n    !runtime.adapter.supportedTasks.includes(task)\n  ) {\n    throw new Error(\n      `The default model cannot perform ${task}. Configure a general-purpose default model.`,\n    );\n  }\n  const reason = `${requested.config.modelFamily} does not support ${task}; using the default model`;\n  getDebug('ai:model-routing', { console: true })(reason);\n  return {\n    ...runtime,\n    modelRouting: {\n      requestedModel: requested.config.modelName,\n      requestedSlot: requested.config.slot,\n      task,\n      reason,\n    },\n  };\n}\n"],"names":["resolveModelForTask","configs","intent","task","requested","getModelRuntime","config","runtime","Error","reason","getDebug"],"mappings":";;AAKO,SAASA,oBACdC,OAA0D,EAC1DC,MAAe,EACfC,IAAe;IAEf,MAAMC,YAAYC,gBAAgBJ,QAAQ,cAAc,CAACC;IACzD,IACE,CAACE,UAAU,OAAO,CAAC,cAAc,IACjCA,UAAU,OAAO,CAAC,cAAc,CAAC,QAAQ,CAACD,OAE1C,OAAOC;IAET,MAAME,SAASL,QAAQ,cAAc,CAAC;IACtC,MAAMM,UAAUF,gBAAgB;QAAE,GAAGC,MAAM;QAAEJ;IAAO;IACpD,IACEK,QAAQ,OAAO,CAAC,cAAc,IAC9B,CAACA,QAAQ,OAAO,CAAC,cAAc,CAAC,QAAQ,CAACJ,OAEzC,MAAM,IAAIK,MACR,CAAC,iCAAiC,EAAEL,KAAK,4CAA4C,CAAC;IAG1F,MAAMM,SAAS,GAAGL,UAAU,MAAM,CAAC,WAAW,CAAC,kBAAkB,EAAED,KAAK,yBAAyB,CAAC;IAClGO,SAAS,oBAAoB;QAAE,SAAS;IAAK,GAAGD;IAChD,OAAO;QACL,GAAGF,OAAO;QACV,cAAc;YACZ,gBAAgBH,UAAU,MAAM,CAAC,SAAS;YAC1C,eAAeA,UAAU,MAAM,CAAC,IAAI;YACpCD;YACAM;QACF;IACF;AACF"}

package/dist/es/ai-model/models/utils/intent.mjs

@@ -1,5 +1,5 @@
 function isLocateIntent(intent) {
-    return 'default' === intent;
+    return 'locate' === intent;
 }
 export { isLocateIntent };
 

package/dist/es/ai-model/models/utils/intent.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/utils/intent.mjs","sources":["../../../../../src/ai-model/models/utils/intent.ts"],"sourcesContent":["import type { TIntent } from '@midscene/shared/env';\n\nexport function isLocateIntent(intent?: TIntent): boolean {\n  return intent === 'default';\n}\n"],"names":["isLocateIntent","intent"],"mappings":"AAEO,SAASA,eAAeC,MAAgB;IAC7C,OAAOA,AAAW,cAAXA;AACT"}
+{"version":3,"file":"ai-model/models/utils/intent.mjs","sources":["../../../../../src/ai-model/models/utils/intent.ts"],"sourcesContent":["import type { TIntent } from '@midscene/shared/env';\n\nexport function isLocateIntent(intent?: TIntent): boolean {\n  return intent === 'locate';\n}\n"],"names":["isLocateIntent","intent"],"mappings":"AAEO,SAASA,eAAeC,MAAgB;IAC7C,OAAOA,AAAW,aAAXA;AACT"}

package/dist/es/ai-model/service-caller/call-ai.mjs

@@ -7,6 +7,7 @@ import { callOpenAI } from "./openai/call-openai.mjs";
 import { prepareChatCompletion } from "./openai/chat-completion/chat-completion.mjs";
 import { prepareResponses } from "./openai/responses/responses.mjs";
 import { buildRequestAbortSignal, resolveEffectiveTimeoutMs, runWithAbortSignal, waitForRetry } from "./request-timeout.mjs";
+import { callSystemOne, prepareSystemOne } from "./system-one.mjs";
 import { appendAIRequestFailureSummary, buildUsageInfo, nextInternalCallId, normalizeRetryCount, toError } from "./utils.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
@@ -27,6 +28,13 @@ async function callAI(messages, modelRuntime, options) {
     const internalCallId = nextInternalCallId();
     const { config: modelConfig } = modelRuntime;
     const prepare = async ()=>{
+        if (options?.systemOneRequest) {
+            assert('jev' === modelConfig.modelFamily, 'System One requests require the Jev model family');
+            return {
+                protocol: 'system-one',
+                input: await prepareSystemOne(modelConfig, options.systemOneRequest)
+            };
+        }
         const input = {
             messages,
             modelRuntime,
@@ -57,7 +65,7 @@ async function callAI(messages, modelRuntime, options) {
         prepared
     });
     const { rawUsage, timeCost, totalTimeCost, retryCount, requestId, responseModelName, ...response } = result;
-    const usage = buildUsageInfo({
+    const rawUsageInfo = buildUsageInfo({
         apiProtocol: prepared.protocol,
         stream: response.isStreamed,
         usageData: rawUsage,
@@ -71,6 +79,10 @@ async function callAI(messages, modelRuntime, options) {
         slot: modelConfig.slot,
         internalCallId
     });
+    const usage = rawUsageInfo ? {
+        ...rawUsageInfo,
+        modelRouting: modelRuntime.modelRouting
+    } : void 0;
     const debugProfileStats = getDebug('ai:profile:stats');
     const debugProfileDetail = getDebug('ai:profile:detail');
     debugProfileStats(`model, ${modelConfig.modelName}, mode, ${modelConfig.modelFamily || 'default'}, streaming, ${response.isStreamed}, prompt-tokens, ${usage?.prompt_tokens ?? ''}, completion-tokens, ${usage?.completion_tokens ?? ''}, total-tokens, ${usage?.total_tokens ?? ''}, cached-input, ${usage?.cached_input ?? ''}, cost-ms, ${timeCost ?? ''}, requestId, ${requestId ?? ''}, slot, ${modelConfig.slot}, configured-temperature, ${modelConfig.temperature ?? ''}`);
@@ -136,6 +148,7 @@ async function callModelOnce({ messages, modelRuntime, options, executionId, int
             slot: modelConfig.slot,
             intent: modelConfig.intent,
             modelFamily: modelConfig.modelFamily,
+            modelRouting: modelRuntime.modelRouting,
             ...event,
             attempt
         });
@@ -161,7 +174,7 @@ async function callModelOnce({ messages, modelRuntime, options, executionId, int
         } : options
     };
     try {
-        return await runWithAbortSignal(requestSignal, ()=>'codex' === prepared.protocol ? callCodex(context, prepared.input) : callOpenAI(context, prepared));
+        return await runWithAbortSignal(requestSignal, ()=>'system-one' === prepared.protocol ? callSystemOne(context, prepared.input) : 'codex' === prepared.protocol ? callCodex(context, prepared.input) : callOpenAI(context, prepared));
     } catch (error) {
         options?.abortSignal?.throwIfAborted();
         const lastError = toError(error);