@midscene/core 1.8.11 → 1.9.0

package/dist/es/ai-model/models/qwen.mjs

@@ -0,0 +1,113 @@
+import { unwrapCoordinateListLikeInput } from "../shared/model-locate-result/index.mjs";
+const defaultBboxSize = 20;
+function topLeftPointToPixelBbox(x, y) {
+    return [
+        Math.round(x),
+        Math.round(y),
+        Math.round(x + defaultBboxSize),
+        Math.round(y + defaultBboxSize)
+    ];
+}
+function parseQwen25RawLocateValue(input) {
+    const bbox = unwrapCoordinateListLikeInput(input);
+    if (bbox.length < 2) {
+        const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;
+        throw new Error(msg);
+    }
+    if ('number' == typeof bbox[2] && 'number' == typeof bbox[3]) return {
+        type: 'bbox',
+        coordinates: [
+            bbox[0],
+            bbox[1],
+            bbox[2],
+            bbox[3]
+        ]
+    };
+    return {
+        type: 'point',
+        coordinates: [
+            bbox[0],
+            bbox[1]
+        ]
+    };
+}
+function normalizeQwen25ResultToPixelBbox(result) {
+    if ('bbox' === result.type) return [
+        Math.round(result.coordinates[0]),
+        Math.round(result.coordinates[1]),
+        Math.round(result.coordinates[2]),
+        Math.round(result.coordinates[3])
+    ];
+    return topLeftPointToPixelBbox(result.coordinates[0], result.coordinates[1]);
+}
+const buildQwenChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const { reasoningEnabled, reasoningBudget } = userConfig;
+    const effectiveReasoningEnabled = reasoningEnabled ?? false;
+    const config = {
+        temperature: userConfig.temperature ?? midsceneDefaults.temperature,
+        enable_thinking: effectiveReasoningEnabled
+    };
+    if (void 0 !== reasoningBudget) config.thinking_budget = reasoningBudget;
+    return {
+        config
+    };
+};
+const buildQwen25ChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    return {
+        config: {
+            temperature: userConfig.temperature ?? midsceneDefaults.temperature,
+            vl_high_resolution_images: true
+        }
+    };
+};
+const qwen3Adapter = {
+    chatCompletion: {
+        unsupportedUserConfig: [
+            'reasoningEffort'
+        ],
+        buildChatCompletionParams: buildQwenChatCompletionParams
+    },
+    locate: {
+        resultAdapter: {
+            coordinates: {
+                shape: 'bbox',
+                order: 'xy',
+                normalizedBy: 1000
+            }
+        }
+    }
+};
+const qwenAdapters = {
+    'qwen2.5-vl': {
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEnabled',
+                'reasoningEffort',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: buildQwen25ChatCompletionParams
+        },
+        imagePreprocess: {
+            padBlockSize: 28
+        },
+        locate: {
+            resultAdapter: {
+                coordinates: {
+                    shape: 'bbox',
+                    order: 'xy'
+                },
+                parseRawLocateValue: parseQwen25RawLocateValue,
+                mapLocateResultToPixelBbox: normalizeQwen25ResultToPixelBbox
+            }
+        }
+    },
+    'qwen3-vl': qwen3Adapter,
+    qwen3: qwen3Adapter,
+    'qwen3.5': qwen3Adapter,
+    'qwen3.6': qwen3Adapter
+};
+export { qwenAdapters };
+
+//# sourceMappingURL=qwen.mjs.map

package/dist/es/ai-model/models/qwen.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/qwen.mjs","sources":["../../../../src/ai-model/models/qwen.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport {\n  type LocateResultValue,\n  type PixelBbox,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from './types';\n\nconst defaultBboxSize = 20;\n\nfunction topLeftPointToPixelBbox(x: number, y: number): PixelBbox {\n  return [\n    Math.round(x),\n    Math.round(y),\n    Math.round(x + defaultBboxSize),\n    Math.round(y + defaultBboxSize),\n  ];\n}\n\nfunction parseQwen25RawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any) as number[];\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  if (typeof bbox[2] === 'number' && typeof bbox[3] === 'number') {\n    return {\n      type: 'bbox',\n      coordinates: [bbox[0], bbox[1], bbox[2], bbox[3]],\n    };\n  }\n\n  return { type: 'point', coordinates: [bbox[0], bbox[1]] };\n}\n\nfunction normalizeQwen25ResultToPixelBbox(\n  result: LocateResultValue,\n): PixelBbox {\n  if (result.type === 'bbox') {\n    return [\n      Math.round(result.coordinates[0]),\n      Math.round(result.coordinates[1]),\n      Math.round(result.coordinates[2]),\n      Math.round(result.coordinates[3]),\n    ];\n  }\n\n  return topLeftPointToPixelBbox(result.coordinates[0], result.coordinates[1]);\n}\n\nconst buildQwenChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningBudget } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const config: Record<string, unknown> = {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n    enable_thinking: effectiveReasoningEnabled,\n  };\n\n  if (reasoningBudget !== undefined) {\n    config.thinking_budget = reasoningBudget;\n  }\n\n  return { config };\n};\n\nconst buildQwen25ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  return {\n    config: {\n      temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n      vl_high_resolution_images: true,\n    },\n  };\n};\n\nconst qwen3Adapter: ModelAdapterDefinition = {\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningEffort'],\n    buildChatCompletionParams: buildQwenChatCompletionParams,\n  },\n  locate: {\n    resultAdapter: {\n      coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n    },\n  },\n};\n\nexport const qwenAdapters = {\n  'qwen2.5-vl': {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: buildQwen25ChatCompletionParams,\n    },\n    imagePreprocess: {\n      padBlockSize: 28,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy' },\n        parseRawLocateValue: parseQwen25RawLocateValue,\n        mapLocateResultToPixelBbox: normalizeQwen25ResultToPixelBbox,\n      },\n    },\n  },\n  'qwen3-vl': qwen3Adapter,\n  qwen3: qwen3Adapter,\n  'qwen3.5': qwen3Adapter,\n  'qwen3.6': qwen3Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6'\n>;\n"],"names":["defaultBboxSize","topLeftPointToPixelBbox","x","y","Math","parseQwen25RawLocateValue","input","bbox","unwrapCoordinateListLikeInput","msg","JSON","Error","normalizeQwen25ResultToPixelBbox","result","buildQwenChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningBudget","effectiveReasoningEnabled","config","undefined","buildQwen25ChatCompletionParams","qwen3Adapter","qwenAdapters"],"mappings":";AAYA,MAAMA,kBAAkB;AAExB,SAASC,wBAAwBC,CAAS,EAAEC,CAAS;IACnD,OAAO;QACLC,KAAK,KAAK,CAACF;QACXE,KAAK,KAAK,CAACD;QACXC,KAAK,KAAK,CAACF,IAAIF;QACfI,KAAK,KAAK,CAACD,IAAIH;KAChB;AACH;AAEA,SAASK,0BAA0BC,KAAc;IAC/C,MAAMC,OAAOC,8BAA8BF;IAC3C,IAAIC,KAAK,MAAM,GAAG,GAAG;QACnB,MAAME,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACH,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAII,MAAMF;IAClB;IAEA,IAAI,AAAmB,YAAnB,OAAOF,IAAI,CAAC,EAAE,IAAiB,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EAC/C,OAAO;QACL,MAAM;QACN,aAAa;YAACA,IAAI,CAAC,EAAE;YAAEA,IAAI,CAAC,EAAE;YAAEA,IAAI,CAAC,EAAE;YAAEA,IAAI,CAAC,EAAE;SAAC;IACnD;IAGF,OAAO;QAAE,MAAM;QAAS,aAAa;YAACA,IAAI,CAAC,EAAE;YAAEA,IAAI,CAAC,EAAE;SAAC;IAAC;AAC1D;AAEA,SAASK,iCACPC,MAAyB;IAEzB,IAAIA,AAAgB,WAAhBA,OAAO,IAAI,EACb,OAAO;QACLT,KAAK,KAAK,CAACS,OAAO,WAAW,CAAC,EAAE;QAChCT,KAAK,KAAK,CAACS,OAAO,WAAW,CAAC,EAAE;QAChCT,KAAK,KAAK,CAACS,OAAO,WAAW,CAAC,EAAE;QAChCT,KAAK,KAAK,CAACS,OAAO,WAAW,CAAC,EAAE;KACjC;IAGH,OAAOZ,wBAAwBY,OAAO,WAAW,CAAC,EAAE,EAAEA,OAAO,WAAW,CAAC,EAAE;AAC7E;AAEA,MAAMC,gCAAgC,CACpCR;IAEA,MAAM,EAAES,gBAAgB,EAAEC,UAAU,EAAE,GAAGV;IACzC,MAAM,EAAEW,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,4BAA4BF,oBAAoB;IACtD,MAAMG,SAAkC;QACtC,aAAaJ,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACnE,iBAAiBI;IACnB;IAEA,IAAID,AAAoBG,WAApBH,iBACFE,OAAO,eAAe,GAAGF;IAG3B,OAAO;QAAEE;IAAO;AAClB;AAEA,MAAME,kCAAkC,CACtChB;IAEA,MAAM,EAAES,gBAAgB,EAAEC,UAAU,EAAE,GAAGV;IACzC,OAAO;QACL,QAAQ;YACN,aAAaU,WAAW,WAAW,IAAID,iBAAiB,WAAW;YACnE,2BAA2B;QAC7B;IACF;AACF;AAEA,MAAMQ,eAAuC;IAC3C,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BT;IAC7B;IACA,QAAQ;QACN,eAAe;YACb,aAAa;gBAAE,OAAO;gBAAQ,OAAO;gBAAM,cAAc;YAAK;QAChE;IACF;AACF;AAEO,MAAMU,eAAe;IAC1B,cAAc;QACZ,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2BF;QAC7B;QACA,iBAAiB;YACf,cAAc;QAChB;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;gBAAK;gBAC1C,qBAAqBjB;gBACrB,4BAA4BO;YAC9B;QACF;IACF;IACA,YAAYW;IACZ,OAAOA;IACP,WAAWA;IACX,WAAWA;AACb"}

package/dist/es/ai-model/models/registry.mjs

@@ -0,0 +1,45 @@
+import { getDebug } from "@midscene/shared/logger";
+import { autoGlmAdapters } from "./auto-glm/adapter.mjs";
+import { defaultOpenAICompatibleAdapterConfig } from "./default.mjs";
+import { doubaoAdapters } from "./doubao.mjs";
+import { geminiAdapters } from "./gemini.mjs";
+import { glmAdapters } from "./glm.mjs";
+import { gptAdapters } from "./gpt.mjs";
+import { qwenAdapters } from "./qwen.mjs";
+import { ResolvedModelAdapter } from "./resolved.mjs";
+import { uiTarsAdapters } from "./ui-tars/adapter.mjs";
+const MODEL_ADAPTER_CONFIGS = {
+    ...qwenAdapters,
+    ...doubaoAdapters,
+    ...geminiAdapters,
+    ...uiTarsAdapters,
+    ...glmAdapters,
+    ...autoGlmAdapters,
+    ...gptAdapters
+};
+const modelAdapterCache = new Map();
+const debugModelAdapter = getDebug('ai:model-adapter');
+function debugAdapterUnsupportedUserConfig(modelFamily, adapter) {
+    if (0 === adapter.chatCompletion.unsupportedUserConfig.length) return;
+    debugModelAdapter(`model adapter "${modelFamily}" unsupportedUserConfig: ${JSON.stringify(adapter.chatCompletion.unsupportedUserConfig)}`);
+}
+function getModelAdapter(modelFamily) {
+    const cacheKey = modelFamily ?? 'default';
+    let adapter = modelAdapterCache.get(cacheKey);
+    if (adapter) return adapter;
+    const config = modelFamily ? MODEL_ADAPTER_CONFIGS[modelFamily] : defaultOpenAICompatibleAdapterConfig;
+    if (!config) throw new Error(`No model adapter registered for modelFamily: ${modelFamily}`);
+    adapter = new ResolvedModelAdapter(config, cacheKey);
+    modelAdapterCache.set(cacheKey, adapter);
+    debugAdapterUnsupportedUserConfig(cacheKey, adapter);
+    return adapter;
+}
+function getModelRuntime(config) {
+    return {
+        config,
+        adapter: getModelAdapter(config.modelFamily)
+    };
+}
+export { MODEL_ADAPTER_CONFIGS, getModelAdapter, getModelRuntime };
+
+//# sourceMappingURL=registry.mjs.map

package/dist/es/ai-model/models/registry.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/registry.mjs","sources":["../../../../src/ai-model/models/registry.ts"],"sourcesContent":["import type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { autoGlmAdapters } from './auto-glm/adapter';\nimport { defaultOpenAICompatibleAdapterConfig } from './default';\nimport { doubaoAdapters } from './doubao';\nimport { geminiAdapters } from './gemini';\nimport { glmAdapters } from './glm';\nimport { gptAdapters } from './gpt';\nimport { qwenAdapters } from './qwen';\nimport { ResolvedModelAdapter } from './resolved';\nimport type {\n  ModelAdapter,\n  ModelAdapterDefinition,\n  ModelRuntime,\n} from './types';\nimport { uiTarsAdapters } from './ui-tars/adapter';\n\nexport const MODEL_ADAPTER_CONFIGS = {\n  ...qwenAdapters,\n  ...doubaoAdapters,\n  ...geminiAdapters,\n  ...uiTarsAdapters,\n  ...glmAdapters,\n  ...autoGlmAdapters,\n  ...gptAdapters,\n} satisfies Record<TModelFamily, ModelAdapterDefinition>;\n\ntype ModelAdapterCacheKey = TModelFamily | 'default';\n\nconst modelAdapterCache = new Map<ModelAdapterCacheKey, ModelAdapter>();\nconst debugModelAdapter = getDebug('ai:model-adapter');\n\nfunction debugAdapterUnsupportedUserConfig(\n  modelFamily: ModelAdapterCacheKey,\n  adapter: ModelAdapter,\n): void {\n  if (adapter.chatCompletion.unsupportedUserConfig.length === 0) {\n    return;\n  }\n\n  debugModelAdapter(\n    `model adapter \"${modelFamily}\" unsupportedUserConfig: ${JSON.stringify(\n      adapter.chatCompletion.unsupportedUserConfig,\n    )}`,\n  );\n}\n\nexport function getModelAdapter(modelFamily?: TModelFamily): ModelAdapter {\n  const cacheKey: ModelAdapterCacheKey = modelFamily ?? 'default';\n  let adapter = modelAdapterCache.get(cacheKey);\n  if (adapter) {\n    return adapter;\n  }\n\n  const config = modelFamily\n    ? MODEL_ADAPTER_CONFIGS[modelFamily]\n    : defaultOpenAICompatibleAdapterConfig;\n  if (!config) {\n    throw new Error(\n      `No model adapter registered for modelFamily: ${modelFamily}`,\n    );\n  }\n\n  adapter = new ResolvedModelAdapter(config, cacheKey);\n  modelAdapterCache.set(cacheKey, adapter);\n  debugAdapterUnsupportedUserConfig(cacheKey, adapter);\n\n  return adapter;\n}\n\nexport function getModelRuntime(config: IModelConfig): ModelRuntime {\n  return {\n    config,\n    adapter: getModelAdapter(config.modelFamily),\n  };\n}\n"],"names":["MODEL_ADAPTER_CONFIGS","qwenAdapters","doubaoAdapters","geminiAdapters","uiTarsAdapters","glmAdapters","autoGlmAdapters","gptAdapters","modelAdapterCache","Map","debugModelAdapter","getDebug","debugAdapterUnsupportedUserConfig","modelFamily","adapter","JSON","getModelAdapter","cacheKey","config","defaultOpenAICompatibleAdapterConfig","Error","ResolvedModelAdapter","getModelRuntime"],"mappings":";;;;;;;;;;AAiBO,MAAMA,wBAAwB;IACnC,GAAGC,YAAY;IACf,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,WAAW;IACd,GAAGC,eAAe;IAClB,GAAGC,WAAW;AAChB;AAIA,MAAMC,oBAAoB,IAAIC;AAC9B,MAAMC,oBAAoBC,SAAS;AAEnC,SAASC,kCACPC,WAAiC,EACjCC,OAAqB;IAErB,IAAIA,AAAwD,MAAxDA,QAAQ,cAAc,CAAC,qBAAqB,CAAC,MAAM,EACrD;IAGFJ,kBACE,CAAC,eAAe,EAAEG,YAAY,yBAAyB,EAAEE,KAAK,SAAS,CACrED,QAAQ,cAAc,CAAC,qBAAqB,GAC3C;AAEP;AAEO,SAASE,gBAAgBH,WAA0B;IACxD,MAAMI,WAAiCJ,eAAe;IACtD,IAAIC,UAAUN,kBAAkB,GAAG,CAACS;IACpC,IAAIH,SACF,OAAOA;IAGT,MAAMI,SAASL,cACXb,qBAAqB,CAACa,YAAY,GAClCM;IACJ,IAAI,CAACD,QACH,MAAM,IAAIE,MACR,CAAC,6CAA6C,EAAEP,aAAa;IAIjEC,UAAU,IAAIO,qBAAqBH,QAAQD;IAC3CT,kBAAkB,GAAG,CAACS,UAAUH;IAChCF,kCAAkCK,UAAUH;IAE5C,OAAOA;AACT;AAEO,SAASQ,gBAAgBJ,MAAoB;IAClD,OAAO;QACLA;QACA,SAASF,gBAAgBE,OAAO,WAAW;IAC7C;AACF"}

package/dist/es/ai-model/models/resolved.mjs

@@ -0,0 +1,104 @@
+import { normalJsonParser } from "../service-caller/json.mjs";
+import { createLocateResultAdapter } from "../shared/model-locate-result/factory.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const defaultReplanningCycleLimit = 20;
+const defaultImageDetail = (_input)=>void 0;
+const defaultChatCompletionParams = ({ midsceneDefaults, userConfig })=>({
+        config: {
+            temperature: userConfig.temperature ?? midsceneDefaults.temperature
+        }
+    });
+const midsceneChatCompletionDefaults = {
+    temperature: 0
+};
+const defaultLocateResultAdapterDefinition = {
+    coordinates: {
+        shape: 'bbox',
+        order: 'xy',
+        normalizedBy: 1000
+    }
+};
+function resolveJsonParser(jsonParser) {
+    if (!jsonParser || 'lenient-json' === jsonParser) return normalJsonParser;
+    if ('function' == typeof jsonParser) return jsonParser;
+    throw new Error(`Unknown json parser preset: ${jsonParser}`);
+}
+function resolveChatCompletion(chatCompletion) {
+    const buildChatCompletionParams = chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;
+    const resolveImageDetail = chatCompletion?.resolveImageDetail ?? defaultImageDetail;
+    const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];
+    return {
+        unsupportedUserConfig,
+        buildChatCompletionParams: (input)=>{
+            const context = {
+                ...input,
+                userConfig: input.userConfig ?? {},
+                midsceneDefaults: midsceneChatCompletionDefaults
+            };
+            return buildChatCompletionParams(context);
+        },
+        resolveImageDetail: (input)=>resolveImageDetail({
+                ...input,
+                userConfig: input.userConfig ?? {},
+                midsceneDefaults: midsceneChatCompletionDefaults
+            })
+    };
+}
+function resolveImagePreprocess(imagePreprocess) {
+    return {
+        padBlockSize: imagePreprocess?.padBlockSize
+    };
+}
+function resolvePlanning(planning) {
+    if (planning?.kind === 'custom') return {
+        kind: 'custom',
+        cacheEnabled: planning.cacheEnabled ?? true,
+        defaultReplanningCycleLimit: planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
+        supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,
+        planFn: planning.planFn
+    };
+    return {
+        kind: 'standard',
+        cacheEnabled: planning?.cacheEnabled ?? true,
+        defaultReplanningCycleLimit: planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
+        supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true
+    };
+}
+function resolveLocate(locate) {
+    if (locate?.kind === 'custom') return {
+        kind: 'custom',
+        supportsSearchArea: locate.supportsSearchArea ?? false,
+        locateFn: locate.locateFn
+    };
+    return {
+        kind: 'standard',
+        supportsSearchArea: locate?.supportsSearchArea ?? true,
+        resultAdapter: createLocateResultAdapter(locate?.resultAdapter ?? defaultLocateResultAdapterDefinition)
+    };
+}
+class ResolvedModelAdapter {
+    constructor(config, modelFamily){
+        _define_property(this, "jsonParser", void 0);
+        _define_property(this, "chatCompletion", void 0);
+        _define_property(this, "imagePreprocess", void 0);
+        _define_property(this, "planning", void 0);
+        _define_property(this, "locate", void 0);
+        this.jsonParser = resolveJsonParser(config.jsonParser);
+        this.chatCompletion = resolveChatCompletion(config.chatCompletion);
+        this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);
+        this.planning = resolvePlanning(config.planning);
+        this.locate = resolveLocate(config.locate);
+    }
+}
+export { ResolvedModelAdapter };
+
+//# sourceMappingURL=resolved.mjs.map

package/dist/es/ai-model/models/resolved.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/resolved.mjs","sources":["../../../../src/ai-model/models/resolved.ts"],"sourcesContent":["import { normalJsonParser } from '../service-caller/json';\nimport { createLocateResultAdapter } from '../shared/model-locate-result/factory';\nimport type { LocateResultAdapterDefinition } from '../shared/model-locate-result/types';\nimport type {\n  ChatCompletionAdapter,\n  ChatCompletionCallContext,\n  ChatCompletionCallInput,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  MidsceneChatCompletionDefaults,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n} from './types';\n\nconst defaultReplanningCycleLimit = 20;\n\nconst defaultImageDetail = (_input: unknown) => undefined;\n\nconst defaultChatCompletionParams = ({\n  midsceneDefaults,\n  userConfig,\n}: ChatCompletionCallContext) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n  temperature: 0,\n};\n\nconst defaultLocateResultAdapterDefinition: LocateResultAdapterDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return normalJsonParser;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveChatCompletion(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n  const buildChatCompletionParams =\n    chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n  const resolveImageDetail =\n    chatCompletion?.resolveImageDetail ?? defaultImageDetail;\n  const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n\n  return {\n    unsupportedUserConfig,\n    buildChatCompletionParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      };\n      return buildChatCompletionParams(context);\n    },\n    resolveImageDetail: (input) =>\n      resolveImageDetail({\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      }),\n  };\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nfunction resolvePlanning(\n  planning: ModelAdapterDefinition['planning'],\n): PlanningAdapter {\n  if (planning?.kind === 'custom') {\n    return {\n      kind: 'custom',\n      cacheEnabled: planning.cacheEnabled ?? true,\n      defaultReplanningCycleLimit:\n        planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n      supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n      planFn: planning.planFn,\n    };\n  }\n\n  return {\n    kind: 'standard',\n    cacheEnabled: planning?.cacheEnabled ?? true,\n    defaultReplanningCycleLimit:\n      planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n    supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true,\n  };\n}\n\nfunction resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    return {\n      kind: 'custom',\n      supportsSearchArea: locate.supportsSearchArea ?? false,\n      locateFn: locate.locateFn,\n    };\n  }\n\n  return {\n    kind: 'standard',\n    supportsSearchArea: locate?.supportsSearchArea ?? true,\n    resultAdapter: createLocateResultAdapter(\n      locate?.resultAdapter ?? defaultLocateResultAdapterDefinition,\n    ),\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    this.planning = resolvePlanning(config.planning);\n    this.locate = resolveLocate(config.locate);\n  }\n}\n"],"names":["defaultReplanningCycleLimit","defaultImageDetail","_input","undefined","defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","defaultLocateResultAdapterDefinition","resolveJsonParser","jsonParser","normalJsonParser","Error","resolveChatCompletion","chatCompletion","buildChatCompletionParams","resolveImageDetail","unsupportedUserConfig","input","context","resolveImagePreprocess","imagePreprocess","resolvePlanning","planning","resolveLocate","locate","createLocateResultAdapter","ResolvedModelAdapter","config","modelFamily"],"mappings":";;;;;;;;;;;;AAgBA,MAAMA,8BAA8B;AAEpC,MAAMC,qBAAqB,CAACC,SAAoBC;AAEhD,MAAMC,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEA,MAAMC,uCAAsE;IAC1E,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEA,SAASC,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,sBACPC,cAAwD;IAExD,MAAMC,4BACJD,gBAAgB,6BAA6BV;IAC/C,MAAMY,qBACJF,gBAAgB,sBAAsBb;IACxC,MAAMgB,wBAAwBH,gBAAgB,yBAAyB,EAAE;IAEzE,OAAO;QACLG;QACA,2BAA2B,CAACC;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBX;YACpB;YACA,OAAOQ,0BAA0BI;QACnC;QACA,oBAAoB,CAACD,QACnBF,mBAAmB;gBACjB,GAAGE,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBX;YACpB;IACJ;AACF;AAEA,SAASa,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEA,SAASC,gBACPC,QAA4C;IAE5C,IAAIA,UAAU,SAAS,UACrB,OAAO;QACL,MAAM;QACN,cAAcA,SAAS,YAAY,IAAI;QACvC,6BACEA,SAAS,2BAA2B,IAAIvB;QAC1C,0BAA0BuB,SAAS,wBAAwB,IAAI;QAC/D,QAAQA,SAAS,MAAM;IACzB;IAGF,OAAO;QACL,MAAM;QACN,cAAcA,UAAU,gBAAgB;QACxC,6BACEA,UAAU,+BAA+BvB;QAC3C,0BAA0BuB,UAAU,4BAA4B;IAClE;AACF;AAEA,SAASC,cACPC,MAAwC;IAExC,IAAIA,QAAQ,SAAS,UACnB,OAAO;QACL,MAAM;QACN,oBAAoBA,OAAO,kBAAkB,IAAI;QACjD,UAAUA,OAAO,QAAQ;IAC3B;IAGF,OAAO;QACL,MAAM;QACN,oBAAoBA,QAAQ,sBAAsB;QAClD,eAAeC,0BACbD,QAAQ,iBAAiBjB;IAE7B;AACF;AAEO,MAAMmB;IAOX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QANjE,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,UAAU,GAAGpB,kBAAkBmB,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGf,sBAAsBe,OAAO,cAAc;QACjE,IAAI,CAAC,eAAe,GAAGR,uBAAuBQ,OAAO,eAAe;QACpE,IAAI,CAAC,QAAQ,GAAGN,gBAAgBM,OAAO,QAAQ;QAC/C,IAAI,CAAC,MAAM,GAAGJ,cAAcI,OAAO,MAAM;IAC3C;AACF"}

package/dist/es/ai-model/models/ui-tars/adapter.mjs

@@ -0,0 +1,142 @@
+import { UITarsModelVersion } from "@midscene/shared/env";
+import { assert } from "@midscene/shared/utils";
+import { jsonrepair } from "jsonrepair";
+import { extractJSONFromCodeBlock, safeParseJson } from "../../service-caller/json.mjs";
+import { unwrapCoordinateListLikeInput } from "../../shared/model-locate-result/index.mjs";
+import { uiTarsPlanning } from "./planning.mjs";
+const defaultVlmUiTarsReplanningCycleLimit = 40;
+function normalizeJsonObject(obj, context = {}) {
+    if (null == obj) return obj;
+    if (Array.isArray(obj)) return obj.map((item)=>normalizeJsonObject(item, context));
+    if ('object' == typeof obj) {
+        const normalized = {};
+        for (const [key, value] of Object.entries(obj)){
+            const trimmedKey = key.trim();
+            const preserveStringValue = context.preserveStringValueKeys?.includes(trimmedKey) ?? false;
+            const normalizedValue = 'string' == typeof value ? preserveStringValue ? value : value.trim() : normalizeJsonObject(value, context);
+            normalized[trimmedKey] = normalizedValue;
+        }
+        return normalized;
+    }
+    return 'string' == typeof obj ? obj.trim() : obj;
+}
+function shouldRepairUiTarsLocateJson(source) {
+    return 'locate' === source || 'section-locator' === source || 'planning-action-param' === source;
+}
+function preprocessUiTarsLocateJson(input) {
+    if (input.includes('bbox')) while(/\d+\s+\d+/.test(input))input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2');
+    return input;
+}
+const uiTarsJsonParser = (raw, context = {
+    source: 'generic-object'
+})=>{
+    const { source } = context;
+    try {
+        return safeParseJson(raw, context);
+    } catch (firstError) {
+        if (!shouldRepairUiTarsLocateJson(source)) throw firstError;
+        const jsonString = preprocessUiTarsLocateJson(extractJSONFromCodeBlock(raw));
+        try {
+            return normalizeJsonObject(JSON.parse(jsonrepair(jsonString)), context);
+        } catch (error) {
+            throw Error(`failed to parse LLM response into JSON. Error - ${String(error ?? firstError ?? 'unknown error')}. Response - \n ${raw}`);
+        }
+    }
+};
+function parseUiTarsRawLocateValue(input) {
+    const bbox = unwrapCoordinateListLikeInput(input);
+    if ('string' == typeof bbox) {
+        assert(/^(\d+)\s(\d+)\s(\d+)\s(\d+)$/.test(bbox.trim()), `invalid bbox data string for ui-tars mode: ${bbox}`);
+        const splitted = bbox.split(' ');
+        if (4 === splitted.length) return {
+            type: 'bbox',
+            coordinates: [
+                Number(splitted[0]),
+                Number(splitted[1]),
+                Number(splitted[2]),
+                Number(splitted[3])
+            ]
+        };
+        throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);
+    }
+    let bboxList = [];
+    if (Array.isArray(bbox) && 'string' == typeof bbox[0]) bbox.forEach((item)=>{
+        if ('string' == typeof item && item.includes(',')) {
+            const [x, y] = item.split(',');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else if ('string' == typeof item && item.includes(' ')) {
+            const [x, y] = item.split(' ');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else bboxList.push(Number(item));
+    });
+    else bboxList = bbox;
+    if (4 === bboxList.length || 5 === bboxList.length) return {
+        type: 'bbox',
+        coordinates: [
+            bboxList[0],
+            bboxList[1],
+            bboxList[2],
+            bboxList[3]
+        ]
+    };
+    if (6 === bboxList.length || 2 === bboxList.length || 3 === bboxList.length || 7 === bboxList.length) return {
+        type: 'point',
+        coordinates: [
+            bboxList[0],
+            bboxList[1]
+        ]
+    };
+    if (8 === bbox.length) return {
+        type: 'bbox',
+        coordinates: [
+            bboxList[0],
+            bboxList[1],
+            bboxList[4],
+            bboxList[5]
+        ]
+    };
+    const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;
+    throw new Error(msg);
+}
+function createUiTarsAdapter(uiTarsModelVersion) {
+    return {
+        jsonParser: uiTarsJsonParser,
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEnabled',
+                'reasoningEffort',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: ({ midsceneDefaults, userConfig })=>({
+                    config: {
+                        temperature: userConfig.temperature ?? midsceneDefaults.temperature
+                    }
+                })
+        },
+        planning: {
+            kind: 'custom',
+            cacheEnabled: false,
+            defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,
+            planFn: (userInstruction, options)=>uiTarsPlanning(userInstruction, options, uiTarsModelVersion)
+        },
+        locate: {
+            resultAdapter: {
+                coordinates: {
+                    shape: 'bbox',
+                    order: 'xy',
+                    normalizedBy: 1000
+                },
+                parseRawLocateValue: parseUiTarsRawLocateValue
+            }
+        }
+    };
+}
+const uiTarsDoubao15Adapter = createUiTarsAdapter(UITarsModelVersion.DOUBAO_1_5_20B);
+const uiTarsAdapters = {
+    'vlm-ui-tars': createUiTarsAdapter(UITarsModelVersion.V1_0),
+    'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,
+    'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter
+};
+export { uiTarsAdapters };
+
+//# sourceMappingURL=adapter.mjs.map

package/dist/es/ai-model/models/ui-tars/adapter.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/adapter.mjs","sources":["../../../../../src/ai-model/models/ui-tars/adapter.ts"],"sourcesContent":["import { type TModelFamily, UITarsModelVersion } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport { jsonrepair } from 'jsonrepair';\nimport {\n  extractJSONFromCodeBlock,\n  safeParseJson,\n} from '../../service-caller/json';\nimport {\n  type LocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../../shared/model-locate-result';\nimport type {\n  JsonParserContext,\n  JsonParserSource,\n  ModelAdapterDefinition,\n} from '../types';\nimport { uiTarsPlanning } from './planning';\n\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nfunction normalizeJsonObject(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item, context));\n  }\n\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n    for (const [key, value] of Object.entries(obj)) {\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : normalizeJsonObject(value, context);\n      normalized[trimmedKey] = normalizedValue;\n    }\n    return normalized;\n  }\n\n  return typeof obj === 'string' ? obj.trim() : obj;\n}\n\nfunction shouldRepairUiTarsLocateJson(source: JsonParserSource) {\n  return (\n    source === 'locate' ||\n    source === 'section-locator' ||\n    source === 'planning-action-param'\n  );\n}\n\nfunction preprocessUiTarsLocateJson(input: string) {\n  if (input.includes('bbox')) {\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nconst uiTarsJsonParser: ModelAdapterDefinition['jsonParser'] = (\n  raw,\n  context = { source: 'generic-object' },\n) => {\n  const { source } = context;\n  try {\n    return safeParseJson(raw, context);\n  } catch (firstError) {\n    if (!shouldRepairUiTarsLocateJson(source)) {\n      throw firstError;\n    }\n\n    const jsonString = preprocessUiTarsLocateJson(\n      extractJSONFromCodeBlock(raw),\n    );\n    try {\n      return normalizeJsonObject(JSON.parse(jsonrepair(jsonString)), context);\n    } catch (error) {\n      throw Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          error ?? firstError ?? 'unknown error',\n        )}. Response - \\n ${raw}`,\n      );\n    }\n  }\n};\n\n// UI-TARS has not received active updates for a long time, so this parser is\n// intentionally kept separate from Doubao even though the current logic is the\n// same. This avoids coupling UI-TARS behavior to future Doubao adapter changes.\nfunction parseUiTarsRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for ui-tars mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return {\n        type: 'bbox',\n        coordinates: [\n          Number(splitted[0]),\n          Number(splitted[1]),\n          Number(splitted[2]),\n          Number(splitted[3]),\n        ],\n      };\n    }\n    throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return {\n      type: 'bbox',\n      coordinates: [bboxList[0], bboxList[1], bboxList[2], bboxList[3]],\n    };\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return { type: 'point', coordinates: [bboxList[0], bboxList[1]] };\n  }\n\n  if (bbox.length === 8) {\n    return {\n      type: 'bbox',\n      coordinates: [bboxList[0], bboxList[1], bboxList[4], bboxList[5]],\n    };\n  }\n\n  const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction createUiTarsAdapter(\n  uiTarsModelVersion: UITarsModelVersion,\n): ModelAdapterDefinition {\n  return {\n    jsonParser: uiTarsJsonParser,\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => ({\n        config: {\n          temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n        },\n      }),\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,\n      planFn: (userInstruction, options) =>\n        uiTarsPlanning(userInstruction, options, uiTarsModelVersion),\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n        parseRawLocateValue: parseUiTarsRawLocateValue,\n      },\n    },\n  };\n}\n\nconst uiTarsDoubao15Adapter = createUiTarsAdapter(\n  UITarsModelVersion.DOUBAO_1_5_20B,\n);\n\nexport const uiTarsAdapters = {\n  'vlm-ui-tars': createUiTarsAdapter(UITarsModelVersion.V1_0),\n  'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,\n  'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5'\n>;\n"],"names":["defaultVlmUiTarsReplanningCycleLimit","normalizeJsonObject","obj","context","Array","item","normalized","key","value","Object","trimmedKey","preserveStringValue","normalizedValue","shouldRepairUiTarsLocateJson","source","preprocessUiTarsLocateJson","input","uiTarsJsonParser","raw","safeParseJson","firstError","jsonString","extractJSONFromCodeBlock","JSON","jsonrepair","error","Error","String","parseUiTarsRawLocateValue","bbox","unwrapCoordinateListLikeInput","assert","splitted","Number","bboxList","x","y","msg","createUiTarsAdapter","uiTarsModelVersion","midsceneDefaults","userConfig","userInstruction","options","uiTarsPlanning","uiTarsDoubao15Adapter","UITarsModelVersion","uiTarsAdapters"],"mappings":";;;;;;AAkBA,MAAMA,uCAAuC;AAE7C,SAASC,oBACPC,GAAQ,EACRC,UAA8D,CAAC,CAAC;IAEhE,IAAID,QAAAA,KACF,OAAOA;IAGT,IAAIE,MAAM,OAAO,CAACF,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACG,OAASJ,oBAAoBI,MAAMF;IAGrD,IAAI,AAAe,YAAf,OAAOD,KAAkB;QAC3B,MAAMI,aAAkB,CAAC;QACzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACP,KAAM;YAC9C,MAAMQ,aAAaH,IAAI,IAAI;YAC3B,MAAMI,sBACJR,QAAQ,uBAAuB,EAAE,SAASO,eAAe;YAC3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOJ,QACHG,sBACEH,QACAA,MAAM,IAAI,KACZP,oBAAoBO,OAAOL;YACjCG,UAAU,CAACI,WAAW,GAAGE;QAC3B;QACA,OAAON;IACT;IAEA,OAAO,AAAe,YAAf,OAAOJ,MAAmBA,IAAI,IAAI,KAAKA;AAChD;AAEA,SAASW,6BAA6BC,MAAwB;IAC5D,OACEA,AAAW,aAAXA,UACAA,AAAW,sBAAXA,UACAA,AAAW,4BAAXA;AAEJ;AAEA,SAASC,2BAA2BC,KAAa;IAC/C,IAAIA,MAAM,QAAQ,CAAC,SACjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEA,MAAMC,mBAAyD,CAC7DC,KACAf,UAAU;IAAE,QAAQ;AAAiB,CAAC;IAEtC,MAAM,EAAEW,MAAM,EAAE,GAAGX;IACnB,IAAI;QACF,OAAOgB,cAAcD,KAAKf;IAC5B,EAAE,OAAOiB,YAAY;QACnB,IAAI,CAACP,6BAA6BC,SAChC,MAAMM;QAGR,MAAMC,aAAaN,2BACjBO,yBAAyBJ;QAE3B,IAAI;YACF,OAAOjB,oBAAoBsB,KAAK,KAAK,CAACC,WAAWH,cAAclB;QACjE,EAAE,OAAOsB,OAAO;YACd,MAAMC,MACJ,CAAC,gDAAgD,EAAEC,OACjDF,SAASL,cAAc,iBACvB,gBAAgB,EAAEF,KAAK;QAE7B;IACF;AACF;AAKA,SAASU,0BAA0BZ,KAAc;IAC/C,MAAMa,OAAOC,8BAA8Bd;IAC3C,IAAI,AAAgB,YAAhB,OAAOa,MAAmB;QAC5BE,OACE,+BAA+B,IAAI,CAACF,KAAK,IAAI,KAC7C,CAAC,2CAA2C,EAAEA,MAAM;QAEtD,MAAMG,WAAWH,KAAK,KAAK,CAAC;QAC5B,IAAIG,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,MAAM;YACN,aAAa;gBACXC,OAAOD,QAAQ,CAAC,EAAE;gBAClBC,OAAOD,QAAQ,CAAC,EAAE;gBAClBC,OAAOD,QAAQ,CAAC,EAAE;gBAClBC,OAAOD,QAAQ,CAAC,EAAE;aACnB;QACH;QAEF,MAAM,IAAIN,MAAM,CAAC,2CAA2C,EAAEG,MAAM;IACtE;IAEA,IAAIK,WAAqB,EAAE;IAC3B,IAAI9B,MAAM,OAAO,CAACyB,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACxB;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAAC8B,GAAGC,EAAE,GAAG/B,KAAK,KAAK,CAAC;YAC1B6B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAO/B,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAAC8B,GAAGC,EAAE,GAAG/B,KAAK,KAAK,CAAC;YAC1B6B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OACEF,SAAS,IAAI,CAACD,OAAO5B;IAEzB;SAEA6B,WAAWL;IAGb,IAAIK,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACL,MAAM;QACN,aAAa;YAACA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;SAAC;IACnE;IAGF,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QAAE,MAAM;QAAS,aAAa;YAACA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;SAAC;IAAC;IAGlE,IAAIL,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACL,MAAM;QACN,aAAa;YAACK,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;SAAC;IACnE;IAGF,MAAMG,MAAM,CAAC,oCAAoC,EAAEd,KAAK,SAAS,CAACM,MAAM,CAAC,CAAC;IAC1E,MAAM,IAAIH,MAAMW;AAClB;AAEA,SAASC,oBACPC,kBAAsC;IAEtC,OAAO;QACL,YAAYtB;QACZ,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEuB,gBAAgB,EAAEC,UAAU,EAAE,GAAM;oBAChE,QAAQ;wBACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;oBACrE;gBACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6BxC;YAC7B,QAAQ,CAAC0C,iBAAiBC,UACxBC,eAAeF,iBAAiBC,SAASJ;QAC7C;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;oBAAM,cAAc;gBAAK;gBAC9D,qBAAqBX;YACvB;QACF;IACF;AACF;AAEA,MAAMiB,wBAAwBP,oBAC5BQ,mBAAmB,cAAc;AAG5B,MAAMC,iBAAiB;IAC5B,eAAeT,oBAAoBQ,mBAAmB,IAAI;IAC1D,sBAAsBD;IACtB,0BAA0BA;AAC5B"}

package/dist/es/ai-model/{ui-tars-planning.mjs → models/ui-tars/planning.mjs}

@@ -2,22 +2,33 @@ import { getDebug } from "@midscene/shared/logger";
 import { transformHotkeyInput } from "@midscene/shared/us-keyboard-layout";
 import { assert } from "@midscene/shared/utils";
 import { actionParser } from "@ui-tars/action-parser";
-import { getSummary, getUiTarsPlanningPrompt } from "./prompt/ui-tars-planning.mjs";
-import { AIResponseParseError, callAIWithStringResponse } from "./service-caller/index.mjs";
+import { getSummary, getUiTarsPlanningPrompt } from "../../prompt/ui-tars-planning.mjs";
+import { AIResponseParseError, callAIWithStringResponse } from "../../service-caller/index.mjs";
+import { finalizePixelBbox } from "../../shared/model-locate-result/bbox.mjs";
+import { mapLocateResultToPixelBboxByCoordinates } from "../../shared/model-locate-result/pixel-bbox-mapper.mjs";
 const debug = getDebug('ui-tars-planning');
 const warnLog = getDebug('ui-tars-planning', {
     console: true
 });
-const bboxSize = 10;
-const pointToBbox = (point, width, height)=>[
-        Math.round(Math.max(point.x - bboxSize / 2, 0)),
-        Math.round(Math.max(point.y - bboxSize / 2, 0)),
-        Math.round(Math.min(point.x + bboxSize / 2, width)),
-        Math.round(Math.min(point.y + bboxSize / 2, height))
-    ];
-async function uiTarsPlanning(userInstruction, options) {
-    const { conversationHistory, context, modelConfig, actionContext } = options;
-    const { uiTarsModelVersion } = modelConfig;
+function pointToLocateParam(point, thought, size) {
+    const ctx = {
+        preparedSize: size
+    };
+    const pixelBbox = mapLocateResultToPixelBboxByCoordinates({
+        type: 'point',
+        coordinates: point
+    }, ctx, {
+        shape: 'point',
+        order: 'xy',
+        normalizedBy: 1
+    });
+    return {
+        prompt: thought || '',
+        locatedPixelBbox: finalizePixelBbox(pixelBbox, point, ctx)
+    };
+}
+async function uiTarsPlanning(userInstruction, options, uiTarsModelVersion) {
+    const { conversationHistory, context, modelRuntime, actionContext } = options;
     let instruction = userInstruction;
     if (actionContext) instruction = `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n<user_instruction>${userInstruction}</user_instruction>`;
     const systemPrompt = getUiTarsPlanningPrompt() + instruction;
@@ -39,7 +50,7 @@ async function uiTarsPlanning(userInstruction, options) {
             content: systemPrompt
         },
         ...conversationHistory.snapshot()
-    ], modelConfig, {
+    ], modelRuntime, {
         abortSignal: options.abortSignal
     });
     let convertedText;
@@ -73,76 +84,46 @@ async function uiTarsPlanning(userInstruction, options) {
         const actionType = (action.action_type || '').toLowerCase();
         if ('click' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
-            const point = getPoint(action.action_inputs.start_box, shotSize);
-            const locate = {
-                prompt: action.thought || '',
-                bbox: pointToBbox({
-                    x: point[0],
-                    y: point[1]
-                }, shotSize.width, shotSize.height)
-            };
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = pointToLocateParam(point, action.thought, shotSize);
             transformActions.push({
                 type: 'Tap',
                 param: {
-                    locate: locate
+                    locate
                 }
             });
         } else if ('left_double' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
-            const point = getPoint(action.action_inputs.start_box, shotSize);
-            const locate = {
-                prompt: action.thought || '',
-                bbox: pointToBbox({
-                    x: point[0],
-                    y: point[1]
-                }, shotSize.width, shotSize.height)
-            };
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = pointToLocateParam(point, action.thought, shotSize);
             transformActions.push({
                 type: 'DoubleClick',
                 param: {
-                    locate: locate
+                    locate
                 },
                 thought: action.thought || ''
             });
         } else if ('right_single' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
-            const point = getPoint(action.action_inputs.start_box, shotSize);
-            const locate = {
-                prompt: action.thought || '',
-                bbox: pointToBbox({
-                    x: point[0],
-                    y: point[1]
-                }, shotSize.width, shotSize.height)
-            };
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = pointToLocateParam(point, action.thought, shotSize);
             transformActions.push({
                 type: 'RightClick',
                 param: {
-                    locate: locate
+                    locate
                 },
                 thought: action.thought || ''
             });
         } else if ('drag' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
             assert(action.action_inputs.end_box, 'end_box is required');
-            const startPoint = getPoint(action.action_inputs.start_box, shotSize);
-            const endPoint = getPoint(action.action_inputs.end_box, shotSize);
+            const startPoint = getPoint(action.action_inputs.start_box);
+            const endPoint = getPoint(action.action_inputs.end_box);
             transformActions.push({
                 type: 'DragAndDrop',
                 param: {
-                    from: {
-                        prompt: action.thought || '',
-                        bbox: pointToBbox({
-                            x: startPoint[0],
-                            y: startPoint[1]
-                        }, shotSize.width, shotSize.height)
-                    },
-                    to: {
-                        prompt: action.thought || '',
-                        bbox: pointToBbox({
-                            x: endPoint[0],
-                            y: endPoint[1]
-                        }, shotSize.width, shotSize.height)
-                    }
+                    from: pointToLocateParam(startPoint, action.thought, shotSize),
+                    to: pointToLocateParam(endPoint, action.thought, shotSize)
                 },
                 thought: action.thought || ''
             });
@@ -165,7 +146,7 @@ async function uiTarsPlanning(userInstruction, options) {
             transformActions.push({
                 type: 'Finished',
                 param: {},
-                thought: action.thought || ''
+                thought: action.action_inputs.content || action.thought || ''
             });
         } else if ('hotkey' === actionType) if (action.action_inputs.key) {
             const keys = transformHotkeyInput(action.action_inputs.key);
@@ -234,16 +215,17 @@ function convertBboxToCoordinates(text) {
         const y = Math.floor((y1Num + y2Num) / 2);
         return `(${x},${y})`;
     }
-    const cleanedText = text.replace(/\[EOS\]/g, '');
+    const cleanedText = text.replace(/\[EOS\]/g, '').replace(/```(?:[a-zA-Z0-9_-]+)?/g, '');
     return cleanedText.replace(pattern, replaceMatch).trim();
 }
-function getPoint(startBox, size) {
+function getPoint(startBox) {
     const [x, y] = JSON.parse(startBox);
+    assert('number' == typeof x && Number.isFinite(x) && 'number' == typeof y && Number.isFinite(y), `invalid point data for ui-tars planning: ${startBox}`);
     return [
-        x * size.width,
-        y * size.height
+        x,
+        y
     ];
 }
 export { uiTarsPlanning };
 
-//# sourceMappingURL=ui-tars-planning.mjs.map
+//# sourceMappingURL=planning.mjs.map