@aiscene/core 9.0.6 → 9.0.8

package/dist/es/ai-model/models/kimi.mjs

@@ -0,0 +1,102 @@
+import { createLocateResultValue, parseCoordinateList } from "../shared/model-locate-result/index.mjs";
+const kimiNormalizedPointCoordinatesMeta = {
+    shape: 'point',
+    order: 'xy',
+    normalizedBy: 1
+};
+const kimiPixelPointCoordinatesMeta = {
+    shape: 'point',
+    order: 'xy'
+};
+function parseKimiRawLocateValue(input) {
+    const point = parseCoordinateList(input, 'point');
+    if (point.length < 2) throw new Error(`invalid point data: ${JSON.stringify(input)} `);
+    const [x, y] = point;
+    const coordinatesMeta = x <= 1 && y <= 1 ? kimiNormalizedPointCoordinatesMeta : kimiPixelPointCoordinatesMeta;
+    return createLocateResultValue(coordinatesMeta, [
+        x,
+        y
+    ]);
+}
+const buildKimiChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const { reasoningEnabled } = userConfig;
+    const effectiveReasoningEnabled = reasoningEnabled ?? false;
+    const commonOverrideConfig = {};
+    commonOverrideConfig.temperature = void 0;
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
+    const modelSpecificConfig = {
+        thinking: {
+            type: effectiveReasoningEnabled ? 'enabled' : 'disabled'
+        }
+    };
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            ...modelSpecificConfig
+        }
+    };
+};
+const buildKimi3ChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const commonOverrideConfig = {};
+    commonOverrideConfig.temperature = void 0;
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
+    const reasoningEffort = userConfig.reasoningEffort;
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            ...reasoningEffort ? {
+                reasoning_effort: reasoningEffort
+            } : {}
+        }
+    };
+};
+const kimiAdapters = {
+    kimi: {
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEffort',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: buildKimiChatCompletionParams,
+            useReasoningAsContentFallback: true
+        },
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: kimiNormalizedPointCoordinatesMeta,
+                    parseRawLocateValue: parseKimiRawLocateValue
+                }
+            }
+        }
+    },
+    kimi3: {
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEnabled',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: buildKimi3ChatCompletionParams,
+            useReasoningAsContentFallback: true,
+            replayRawAssistantMessage: true
+        },
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: kimiNormalizedPointCoordinatesMeta,
+                    parseRawLocateValue: parseKimiRawLocateValue
+                }
+            }
+        }
+    }
+};
+export { kimiAdapters };
+
+//# sourceMappingURL=kimi.mjs.map

package/dist/es/ai-model/models/kimi.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/kimi.mjs","sources":["../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["import type { TModelFamily } from '@aiscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  parseCoordinateList,\n} from '../shared/model-locate-result';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1,\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n  const point = parseCoordinateList(input, 'point');\n  if (point.length < 2) {\n    throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n  }\n  const [x, y] = point;\n  // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n  // normalized coordinates, otherwise they are treated as screenshot pixels.\n  const coordinatesMeta =\n    x <= 1 && y <= 1\n      ? kimiNormalizedPointCoordinatesMeta\n      : kimiPixelPointCoordinatesMeta;\n  return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // kimi disallow custom temperature\n  commonOverrideConfig.temperature = undefined;\n\n  // Kimi Chat Completions response_format:\n  // https://platform.kimi.com/docs/api/chat\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst buildKimi3ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // Kimi disallows custom temperature.\n  commonOverrideConfig.temperature = undefined;\n\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  // Kimi K3 currently only supports reasoning_effort=\"max\"; its docs say\n  // additional effort levels will be available later.\n  const reasoningEffort = userConfig.reasoningEffort;\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}),\n    },\n  };\n};\n\nexport const kimiAdapters = {\n  kimi: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimiChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: kimiNormalizedPointCoordinatesMeta,\n          parseRawLocateValue: parseKimiRawLocateValue,\n        },\n      },\n    },\n  },\n  kimi3: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEnabled', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimi3ChatCompletionParams,\n      useReasoningAsContentFallback: true,\n      // Kimi K3 multi-turn and tool calls must replay the complete assistant\n      // message, including reasoning_content and tool_calls.\n      // https://platform.kimi.com/docs/guide/use-thinking-effort\n      replayRawAssistantMessage: true,\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: kimiNormalizedPointCoordinatesMeta,\n          parseRawLocateValue: parseKimiRawLocateValue,\n        },\n      },\n    },\n  },\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'kimi' | 'kimi3'\n>;\n"],"names":["kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","buildKimi3ChatCompletionParams","reasoningEffort","kimiAdapters"],"mappings":";AAYA,MAAMA,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;AACT;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,oBAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,wBAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C;QACnD,UAAU;YACR,MAAMH,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,iCAAiC,CACrCjB;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAMc,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAEnC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAK/D,MAAMI,kBAAkBP,WAAW,eAAe;IAElD,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAII,kBAAkB;gBAAE,kBAAkBA;YAAgB,IAAI,CAAC,CAAC;QAClE;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BV;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAaZ;oBACb,qBAAqBE;gBACvB;YACF;QACF;IACF;IACA,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBAAC;gBAAoB;aAAkB;YAC9D,2BAA2BkB;YAC3B,+BAA+B;YAI/B,2BAA2B;QAC7B;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAapB;oBACb,qBAAqBE;gBACvB;YACF;QACF;IACF;AACF"}

package/dist/es/ai-model/models/mimo.mjs

@@ -0,0 +1,36 @@
+const buildMimoChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const { reasoningEnabled } = userConfig;
+    const commonOverrideConfig = {};
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
+    if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+    const modelSpecificConfig = {
+        thinking: {
+            type: reasoningEnabled ?? false ? 'enabled' : 'disabled'
+        }
+    };
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            ...modelSpecificConfig
+        }
+    };
+};
+const mimoAdapters = {
+    'xiaomi-mimo': {
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEffort',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: buildMimoChatCompletionParams,
+            useReasoningAsContentFallback: true
+        }
+    }
+};
+export { mimoAdapters };
+
+//# sourceMappingURL=mimo.mjs.map

package/dist/es/ai-model/models/mimo.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/mimo.mjs","sources":["../../../../src/ai-model/models/mimo.ts"],"sourcesContent":["import type { TModelFamily } from '@aiscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\n\nconst buildMimoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // https://platform.xiaomimimo.com/docs/zh-CN/api/chat/openai-api\n  // Observed with thinking disabled: Mimo needs json_object to return JSON.\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const mimoAdapters = {\n  'xiaomi-mimo': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildMimoChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'xiaomi-mimo'>;\n"],"names":["buildMimoChatCompletionParams","input","midsceneDefaults","userConfig","reasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","mimoAdapters"],"mappings":"AAOA,MAAMA,gCAAgC,CACpCC;IAEA,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGF;IACzC,MAAM,EAAEG,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,uBAAgD,CAAC;IAIvD,IACEF,AAA8B,WAA9BA,WAAW,cAAc,IACzBF,MAAM,0BAA0B,EAEhCI,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,IAAIF,AAA2BG,WAA3BH,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAG3D,MAAMI,sBAA+C;QACnD,UAAU;YACR,MAAOH,oBAAoB,QAAS,YAAY;QAClD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGF,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,eAAe;QACb,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BR;YAC3B,+BAA+B;QACjC;IACF;AACF"}

package/dist/es/ai-model/models/qwen.mjs

@@ -0,0 +1,139 @@
+import { createLocateResultValue, isBboxLocateResultValue, unwrapCoordinateListLikeInput } from "../shared/model-locate-result/index.mjs";
+const defaultBboxSize = 20;
+const qwen25BboxCoordinatesMeta = {
+    shape: 'bbox',
+    order: 'xy'
+};
+const qwen25PointCoordinatesMeta = {
+    shape: 'point',
+    order: 'xy'
+};
+const qwen3BboxCoordinatesMeta = {
+    shape: 'bbox',
+    order: 'xy',
+    normalizedBy: 1000
+};
+function topLeftPointToPixelBbox(x, y) {
+    return [
+        Math.round(x),
+        Math.round(y),
+        Math.round(x + defaultBboxSize),
+        Math.round(y + defaultBboxSize)
+    ];
+}
+function parseQwen25RawLocateValue(input) {
+    const bbox = unwrapCoordinateListLikeInput(input);
+    if (bbox.length < 2) {
+        const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;
+        throw new Error(msg);
+    }
+    if ('number' == typeof bbox[2] && 'number' == typeof bbox[3]) return createLocateResultValue(qwen25BboxCoordinatesMeta, [
+        bbox[0],
+        bbox[1],
+        bbox[2],
+        bbox[3]
+    ]);
+    return createLocateResultValue(qwen25PointCoordinatesMeta, [
+        bbox[0],
+        bbox[1]
+    ]);
+}
+function normalizeQwen25ResultToPixelBbox(result) {
+    if (isBboxLocateResultValue(result)) {
+        const { coordinates } = result;
+        return [
+            Math.round(coordinates[0]),
+            Math.round(coordinates[1]),
+            Math.round(coordinates[2]),
+            Math.round(coordinates[3])
+        ];
+    }
+    const { coordinates } = result;
+    return topLeftPointToPixelBbox(coordinates[0], coordinates[1]);
+}
+const buildQwenChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const { reasoningEnabled, reasoningBudget } = userConfig;
+    const commonOverrideConfig = {};
+    if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+    const modelSpecificConfig = {};
+    if ('default' !== reasoningEnabled) {
+        modelSpecificConfig.enable_thinking = reasoningEnabled ?? false;
+        if (void 0 !== reasoningBudget) modelSpecificConfig.thinking_budget = reasoningBudget;
+    }
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            ...modelSpecificConfig
+        }
+    };
+};
+const buildQwen25ChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const commonOverrideConfig = {};
+    if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            vl_high_resolution_images: true
+        }
+    };
+};
+const qwen3Adapter = {
+    acceptBbox2dAlias: true,
+    chatCompletion: {
+        unsupportedUserConfig: [
+            'reasoningEffort'
+        ],
+        buildChatCompletionParams: buildQwenChatCompletionParams,
+        messageExtraction: {
+            kind: 'default',
+            reasoningContentKeys: [
+                'reasoning_content',
+                'reasoning'
+            ]
+        },
+        useReasoningAsContentFallback: true
+    },
+    locate: {
+        element: {
+            resultFormat: {
+                coordinates: qwen3BboxCoordinatesMeta
+            }
+        }
+    }
+};
+const qwenAdapters = {
+    'qwen2.5-vl': {
+        acceptBbox2dAlias: true,
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEnabled',
+                'reasoningEffort',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: buildQwen25ChatCompletionParams
+        },
+        imagePreprocess: {
+            padBlockSize: 28
+        },
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: qwen25BboxCoordinatesMeta,
+                    parseRawLocateValue: parseQwen25RawLocateValue,
+                    mapLocateResultToPixelBbox: normalizeQwen25ResultToPixelBbox
+                }
+            }
+        }
+    },
+    'qwen3-vl': qwen3Adapter,
+    qwen3: qwen3Adapter,
+    'qwen3.5': qwen3Adapter,
+    'qwen3.6': qwen3Adapter
+};
+export { qwenAdapters };
+
+//# sourceMappingURL=qwen.mjs.map

package/dist/es/ai-model/models/qwen.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/qwen.mjs","sources":["../../../../src/ai-model/models/qwen.ts"],"sourcesContent":["import type { TModelFamily } from '@aiscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  type PixelBbox,\n  createLocateResultValue,\n  isBboxLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst defaultBboxSize = 20;\nconst qwen25BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n} as const;\nconst qwen25PointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\nconst qwen3BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\nfunction topLeftPointToPixelBbox(x: number, y: number): PixelBbox {\n  return [\n    Math.round(x),\n    Math.round(y),\n    Math.round(x + defaultBboxSize),\n    Math.round(y + defaultBboxSize),\n  ];\n}\n\nfunction parseQwen25RawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any) as number[];\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  if (typeof bbox[2] === 'number' && typeof bbox[3] === 'number') {\n    return createLocateResultValue(qwen25BboxCoordinatesMeta, [\n      bbox[0],\n      bbox[1],\n      bbox[2],\n      bbox[3],\n    ]);\n  }\n\n  return createLocateResultValue(qwen25PointCoordinatesMeta, [\n    bbox[0],\n    bbox[1],\n  ]);\n}\n\nfunction normalizeQwen25ResultToPixelBbox(\n  result: LocateResultValue,\n): PixelBbox {\n  if (isBboxLocateResultValue(result)) {\n    const { coordinates } = result;\n    return [\n      Math.round(coordinates[0]),\n      Math.round(coordinates[1]),\n      Math.round(coordinates[2]),\n      Math.round(coordinates[3]),\n    ];\n  }\n\n  const { coordinates } = result;\n  return topLeftPointToPixelBbox(coordinates[0], coordinates[1]);\n}\n\nconst buildQwenChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningBudget } = userConfig;\n\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // Alibaba Cloud Model Studio JSON mode:\n  // https://help.aliyun.com/zh/model-studio/json-mode\n  // Observed in qwen3.6 grounding runs: enabling this can make the model\n  // return only [\"bbox_2d\"] without coordinates.\n  // if (isLocateIntent(input.intent)) {\n  //   commonOverrideConfig.response_format = { type: 'json_object' };\n  // }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.enable_thinking = reasoningEnabled ?? false;\n    if (reasoningBudget !== undefined) {\n      modelSpecificConfig.thinking_budget = reasoningBudget;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst buildQwen25ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      vl_high_resolution_images: true,\n    },\n  };\n};\n\nconst qwen3Adapter: ModelAdapterDefinition = {\n  acceptBbox2dAlias: true,\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningEffort'],\n    buildChatCompletionParams: buildQwenChatCompletionParams,\n    messageExtraction: {\n      kind: 'default',\n      // DashScope returns Qwen thinking as `reasoning_content`. vLLM's\n      // OpenAI-compatible server is migrating the recommended field to\n      // `reasoning`, so Qwen adapters accept both serving conventions:\n      // https://github.com/vllm-project/vllm/issues/27755\n      reasoningContentKeys: ['reasoning_content', 'reasoning'],\n    },\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    element: {\n      resultFormat: {\n        coordinates: qwen3BboxCoordinatesMeta,\n      },\n    },\n  },\n};\n\nexport const qwenAdapters = {\n  'qwen2.5-vl': {\n    acceptBbox2dAlias: true,\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: buildQwen25ChatCompletionParams,\n    },\n    imagePreprocess: {\n      padBlockSize: 28,\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: qwen25BboxCoordinatesMeta,\n          parseRawLocateValue: parseQwen25RawLocateValue,\n          mapLocateResultToPixelBbox: normalizeQwen25ResultToPixelBbox,\n        },\n      },\n    },\n  },\n  'qwen3-vl': qwen3Adapter,\n  qwen3: qwen3Adapter,\n  'qwen3.5': qwen3Adapter,\n  'qwen3.6': qwen3Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6'\n>;\n"],"names":["defaultBboxSize","qwen25BboxCoordinatesMeta","qwen25PointCoordinatesMeta","qwen3BboxCoordinatesMeta","topLeftPointToPixelBbox","x","y","Math","parseQwen25RawLocateValue","input","bbox","unwrapCoordinateListLikeInput","msg","JSON","Error","createLocateResultValue","normalizeQwen25ResultToPixelBbox","result","isBboxLocateResultValue","coordinates","buildQwenChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningBudget","commonOverrideConfig","undefined","modelSpecificConfig","buildQwen25ChatCompletionParams","qwen3Adapter","qwenAdapters"],"mappings":";AAcA,MAAMA,kBAAkB;AACxB,MAAMC,4BAA4B;IAChC,OAAO;IACP,OAAO;AACT;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;AACT;AACA,MAAMC,2BAA2B;IAC/B,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAEA,SAASC,wBAAwBC,CAAS,EAAEC,CAAS;IACnD,OAAO;QACLC,KAAK,KAAK,CAACF;QACXE,KAAK,KAAK,CAACD;QACXC,KAAK,KAAK,CAACF,IAAIL;QACfO,KAAK,KAAK,CAACD,IAAIN;KAChB;AACH;AAEA,SAASQ,0BAA0BC,KAAc;IAC/C,MAAMC,OAAOC,8BAA8BF;IAC3C,IAAIC,KAAK,MAAM,GAAG,GAAG;QACnB,MAAME,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACH,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAII,MAAMF;IAClB;IAEA,IAAI,AAAmB,YAAnB,OAAOF,IAAI,CAAC,EAAE,IAAiB,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EAC/C,OAAOK,wBAAwBd,2BAA2B;QACxDS,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;IAGH,OAAOK,wBAAwBb,4BAA4B;QACzDQ,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;AACH;AAEA,SAASM,iCACPC,MAAyB;IAEzB,IAAIC,wBAAwBD,SAAS;QACnC,MAAM,EAAEE,WAAW,EAAE,GAAGF;QACxB,OAAO;YACLV,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;YACzBZ,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;YACzBZ,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;YACzBZ,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;SAC1B;IACH;IAEA,MAAM,EAAEA,WAAW,EAAE,GAAGF;IACxB,OAAOb,wBAAwBe,WAAW,CAAC,EAAE,EAAEA,WAAW,CAAC,EAAE;AAC/D;AAEA,MAAMC,gCAAgC,CACpCX;IAEA,MAAM,EAAEY,gBAAgB,EAAEC,UAAU,EAAE,GAAGb;IACzC,MAAM,EAAEc,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAE9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAW3D,MAAMK,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,eAAe,GAAGJ,oBAAoB;QAC1D,IAAIC,AAAoBE,WAApBF,iBACFG,oBAAoB,eAAe,GAAGH;IAE1C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,kCAAkC,CACtCnB;IAEA,MAAM,EAAEY,gBAAgB,EAAEC,UAAU,EAAE,GAAGb;IACzC,MAAMgB,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,2BAA2B;QAC7B;IACF;AACF;AAEA,MAAMI,eAAuC;IAC3C,mBAAmB;IACnB,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BT;QAC3B,mBAAmB;YACjB,MAAM;YAKN,sBAAsB;gBAAC;gBAAqB;aAAY;QAC1D;QACA,+BAA+B;IACjC;IACA,QAAQ;QACN,SAAS;YACP,cAAc;gBACZ,aAAajB;YACf;QACF;IACF;AACF;AAEO,MAAM2B,eAAe;IAC1B,cAAc;QACZ,mBAAmB;QACnB,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2BF;QAC7B;QACA,iBAAiB;YACf,cAAc;QAChB;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAa3B;oBACb,qBAAqBO;oBACrB,4BAA4BQ;gBAC9B;YACF;QACF;IACF;IACA,YAAYa;IACZ,OAAOA;IACP,WAAWA;IACX,WAAWA;AACb"}

package/dist/es/ai-model/models/registry.mjs

@@ -0,0 +1,51 @@
+import { getDebug } from "@aiscene/shared/logger";
+import { ResolvedModelAdapter } from "../model-adapter/resolve.mjs";
+import { autoGlmAdapters } from "./auto-glm/adapter.mjs";
+import { deepSeekAdapters } from "./deepseek/adapter.mjs";
+import { defaultOpenAICompatibleAdapterConfig } from "./default.mjs";
+import { doubaoAdapters } from "./doubao.mjs";
+import { geminiAdapters } from "./gemini.mjs";
+import { glmAdapters } from "./glm.mjs";
+import { gptAdapters } from "./gpt.mjs";
+import { kimiAdapters } from "./kimi.mjs";
+import { mimoAdapters } from "./mimo.mjs";
+import { qwenAdapters } from "./qwen.mjs";
+import { uiTarsAdapters } from "./ui-tars/adapter.mjs";
+const MODEL_ADAPTER_CONFIGS = {
+    ...qwenAdapters,
+    ...deepSeekAdapters,
+    ...doubaoAdapters,
+    ...geminiAdapters,
+    ...uiTarsAdapters,
+    ...glmAdapters,
+    ...autoGlmAdapters,
+    ...gptAdapters,
+    ...kimiAdapters,
+    ...mimoAdapters
+};
+const modelAdapterCache = new Map();
+const debugModelAdapter = getDebug('ai:model-adapter');
+function debugAdapterUnsupportedUserConfig(modelFamily, adapter) {
+    if (0 === adapter.chatCompletion.unsupportedUserConfig.length) return;
+    debugModelAdapter(`model adapter "${modelFamily}" unsupportedUserConfig: ${JSON.stringify(adapter.chatCompletion.unsupportedUserConfig)}`);
+}
+function getModelAdapter(modelFamily) {
+    const cacheKey = modelFamily ?? 'default';
+    let adapter = modelAdapterCache.get(cacheKey);
+    if (adapter) return adapter;
+    const config = modelFamily ? MODEL_ADAPTER_CONFIGS[modelFamily] : defaultOpenAICompatibleAdapterConfig;
+    if (!config) throw new Error(`No model adapter registered for modelFamily: ${modelFamily}`);
+    adapter = new ResolvedModelAdapter(config, cacheKey);
+    modelAdapterCache.set(cacheKey, adapter);
+    debugAdapterUnsupportedUserConfig(cacheKey, adapter);
+    return adapter;
+}
+function getModelRuntime(config) {
+    return {
+        config,
+        adapter: getModelAdapter(config.modelFamily)
+    };
+}
+export { MODEL_ADAPTER_CONFIGS, getModelAdapter, getModelRuntime };
+
+//# sourceMappingURL=registry.mjs.map

package/dist/es/ai-model/models/registry.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/registry.mjs","sources":["../../../../src/ai-model/models/registry.ts"],"sourcesContent":["import type { IModelConfig, TModelFamily } from '@aiscene/shared/env';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { ResolvedModelAdapter } from '../model-adapter/resolve';\nimport type {\n  ModelAdapter,\n  ModelAdapterDefinition,\n  ModelRuntime,\n} from '../model-adapter/types';\nimport { autoGlmAdapters } from './auto-glm/adapter';\nimport { deepSeekAdapters } from './deepseek/adapter';\nimport { defaultOpenAICompatibleAdapterConfig } from './default';\nimport { doubaoAdapters } from './doubao';\nimport { geminiAdapters } from './gemini';\nimport { glmAdapters } from './glm';\nimport { gptAdapters } from './gpt';\nimport { kimiAdapters } from './kimi';\nimport { mimoAdapters } from './mimo';\nimport { qwenAdapters } from './qwen';\nimport { uiTarsAdapters } from './ui-tars/adapter';\n\nexport const MODEL_ADAPTER_CONFIGS = {\n  ...qwenAdapters,\n  ...deepSeekAdapters,\n  ...doubaoAdapters,\n  ...geminiAdapters,\n  ...uiTarsAdapters,\n  ...glmAdapters,\n  ...autoGlmAdapters,\n  ...gptAdapters,\n  ...kimiAdapters,\n  ...mimoAdapters,\n} satisfies Record<TModelFamily, ModelAdapterDefinition>;\n\ntype ModelAdapterCacheKey = TModelFamily | 'default';\n\nconst modelAdapterCache = new Map<ModelAdapterCacheKey, ModelAdapter>();\nconst debugModelAdapter = getDebug('ai:model-adapter');\n\nfunction debugAdapterUnsupportedUserConfig(\n  modelFamily: ModelAdapterCacheKey,\n  adapter: ModelAdapter,\n): void {\n  if (adapter.chatCompletion.unsupportedUserConfig.length === 0) {\n    return;\n  }\n\n  debugModelAdapter(\n    `model adapter \"${modelFamily}\" unsupportedUserConfig: ${JSON.stringify(\n      adapter.chatCompletion.unsupportedUserConfig,\n    )}`,\n  );\n}\n\nexport function getModelAdapter(modelFamily?: TModelFamily): ModelAdapter {\n  const cacheKey: ModelAdapterCacheKey = modelFamily ?? 'default';\n  let adapter = modelAdapterCache.get(cacheKey);\n  if (adapter) {\n    return adapter;\n  }\n\n  const config = modelFamily\n    ? MODEL_ADAPTER_CONFIGS[modelFamily]\n    : defaultOpenAICompatibleAdapterConfig;\n  if (!config) {\n    throw new Error(\n      `No model adapter registered for modelFamily: ${modelFamily}`,\n    );\n  }\n\n  adapter = new ResolvedModelAdapter(config, cacheKey);\n  modelAdapterCache.set(cacheKey, adapter);\n  debugAdapterUnsupportedUserConfig(cacheKey, adapter);\n\n  return adapter;\n}\n\nexport function getModelRuntime(config: IModelConfig): ModelRuntime {\n  return {\n    config,\n    adapter: getModelAdapter(config.modelFamily),\n  };\n}\n"],"names":["MODEL_ADAPTER_CONFIGS","qwenAdapters","deepSeekAdapters","doubaoAdapters","geminiAdapters","uiTarsAdapters","glmAdapters","autoGlmAdapters","gptAdapters","kimiAdapters","mimoAdapters","modelAdapterCache","Map","debugModelAdapter","getDebug","debugAdapterUnsupportedUserConfig","modelFamily","adapter","JSON","getModelAdapter","cacheKey","config","defaultOpenAICompatibleAdapterConfig","Error","ResolvedModelAdapter","getModelRuntime"],"mappings":";;;;;;;;;;;;;AAoBO,MAAMA,wBAAwB;IACnC,GAAGC,YAAY;IACf,GAAGC,gBAAgB;IACnB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,WAAW;IACd,GAAGC,eAAe;IAClB,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,YAAY;AACjB;AAIA,MAAMC,oBAAoB,IAAIC;AAC9B,MAAMC,oBAAoBC,SAAS;AAEnC,SAASC,kCACPC,WAAiC,EACjCC,OAAqB;IAErB,IAAIA,AAAwD,MAAxDA,QAAQ,cAAc,CAAC,qBAAqB,CAAC,MAAM,EACrD;IAGFJ,kBACE,CAAC,eAAe,EAAEG,YAAY,yBAAyB,EAAEE,KAAK,SAAS,CACrED,QAAQ,cAAc,CAAC,qBAAqB,GAC3C;AAEP;AAEO,SAASE,gBAAgBH,WAA0B;IACxD,MAAMI,WAAiCJ,eAAe;IACtD,IAAIC,UAAUN,kBAAkB,GAAG,CAACS;IACpC,IAAIH,SACF,OAAOA;IAGT,MAAMI,SAASL,cACXhB,qBAAqB,CAACgB,YAAY,GAClCM;IACJ,IAAI,CAACD,QACH,MAAM,IAAIE,MACR,CAAC,6CAA6C,EAAEP,aAAa;IAIjEC,UAAU,IAAIO,qBAAqBH,QAAQD;IAC3CT,kBAAkB,GAAG,CAACS,UAAUH;IAChCF,kCAAkCK,UAAUH;IAE5C,OAAOA;AACT;AAEO,SAASQ,gBAAgBJ,MAAoB;IAClD,OAAO;QACLA;QACA,SAASF,gBAAgBE,OAAO,WAAW;IAC7C;AACF"}

package/dist/es/ai-model/models/ui-tars/actions.mjs

@@ -0,0 +1,147 @@
+import { getDebug } from "@aiscene/shared/logger";
+import { transformHotkeyInput } from "@aiscene/shared/us-keyboard-layout";
+import { assert } from "@aiscene/shared/utils";
+const debug = getDebug('ui-tars-planning');
+const warnLog = getDebug('ui-tars-planning', {
+    console: true
+});
+function transformUiTarsActions(parsedPlanningResponse) {
+    const transformActions = [];
+    const unhandledActions = [];
+    parsedPlanningResponse.actions.forEach((action)=>{
+        const actionType = (action.action_type || '').toLowerCase();
+        if ('click' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = {
+                point,
+                prompt: action.thought || ''
+            };
+            transformActions.push({
+                type: 'Tap',
+                param: {
+                    locate
+                }
+            });
+        } else if ('left_double' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = {
+                point,
+                prompt: action.thought || ''
+            };
+            transformActions.push({
+                type: 'DoubleClick',
+                param: {
+                    locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('right_single' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = {
+                point,
+                prompt: action.thought || ''
+            };
+            transformActions.push({
+                type: 'RightClick',
+                param: {
+                    locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('drag' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            assert(action.action_inputs.end_box, 'end_box is required');
+            const startPoint = getPoint(action.action_inputs.start_box);
+            const endPoint = getPoint(action.action_inputs.end_box);
+            transformActions.push({
+                type: 'DragAndDrop',
+                param: {
+                    from: {
+                        point: startPoint,
+                        prompt: action.thought || ''
+                    },
+                    to: {
+                        point: endPoint,
+                        prompt: action.thought || ''
+                    }
+                },
+                thought: action.thought || ''
+            });
+        } else if ('type' === actionType) transformActions.push({
+            type: 'Input',
+            param: {
+                value: action.action_inputs.content
+            },
+            thought: action.thought || ''
+        });
+        else if ('scroll' === actionType) transformActions.push({
+            type: 'Scroll',
+            param: {
+                direction: action.action_inputs.direction
+            },
+            thought: action.thought || ''
+        });
+        else if ('finished' === actionType) transformActions.push({
+            type: 'Finished',
+            param: {},
+            thought: action.action_inputs.content || action.thought || ''
+        });
+        else if ('hotkey' === actionType) if (action.action_inputs.key) {
+            const keys = transformHotkeyInput(action.action_inputs.key);
+            transformActions.push({
+                type: 'KeyboardPress',
+                param: {
+                    keyName: keys.join('+')
+                },
+                thought: action.thought || ''
+            });
+        } else warnLog('No key found in action: hotkey. Will not perform action.');
+        else if ('wait' === actionType) transformActions.push({
+            type: 'Sleep',
+            param: {
+                timeMs: 1000
+            },
+            thought: action.thought || ''
+        });
+        else if (actionType) {
+            unhandledActions.push({
+                type: actionType,
+                thought: action.thought || ''
+            });
+            debug('Unhandled action type:', actionType, 'thought:', action.thought);
+        }
+    });
+    if (0 === transformActions.length) throw new Error(buildNoUiTarsActionsError(parsedPlanningResponse.rawResponse, parsedPlanningResponse.actions, unhandledActions));
+    debug('transformActions', JSON.stringify(transformActions, null, 2));
+    return transformActions;
+}
+function getPoint(startBox) {
+    const [x, y] = JSON.parse(startBox);
+    assert('number' == typeof x && Number.isFinite(x) && 'number' == typeof y && Number.isFinite(y), `invalid point data for ui-tars planning: ${startBox}`);
+    return [
+        x,
+        y
+    ];
+}
+function buildNoUiTarsActionsError(rawResponse, actions, unhandledActions) {
+    const errorDetails = [];
+    if (0 === actions.length) {
+        errorDetails.push('Action parser returned no actions');
+        if (rawResponse.includes('Thought:') && !rawResponse.includes('Action:')) errorDetails.push('Response contains "Thought:" but missing "Action:" line');
+        else errorDetails.push('Response may be malformed or empty');
+    }
+    if (unhandledActions.length > 0) {
+        const types = unhandledActions.map((a)=>a.type).join(', ');
+        errorDetails.push(`Unhandled action types: ${types}`);
+    }
+    return [
+        'No actions found in UI-TARS response.',
+        ...errorDetails
+    ].join('\n');
+}
+export { transformUiTarsActions };
+
+//# sourceMappingURL=actions.mjs.map

package/dist/es/ai-model/models/ui-tars/actions.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/actions.mjs","sources":["../../../../../src/ai-model/models/ui-tars/actions.ts"],"sourcesContent":["import type { PlanningAction } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { transformHotkeyInput } from '@aiscene/shared/us-keyboard-layout';\nimport { assert } from '@aiscene/shared/utils';\nimport type {\n  DragAndDropPlanningAction,\n  LocatePlanningAction,\n} from '../../model-adapter/custom-planning-action';\nimport type { UiTarsParsedPlanningResponse } from './parser';\n\nconst debug = getDebug('ui-tars-planning');\nconst warnLog = getDebug('ui-tars-planning', { console: true });\n\nexport function transformUiTarsActions(\n  parsedPlanningResponse: UiTarsParsedPlanningResponse,\n): PlanningAction[] {\n  const transformActions: PlanningAction[] = [];\n  const unhandledActions: Array<{ type: string; thought: string }> = [];\n\n  parsedPlanningResponse.actions.forEach((action) => {\n    const actionType = (action.action_type || '').toLowerCase();\n    if (actionType === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate,\n        },\n      } satisfies LocatePlanningAction<'Tap'>);\n    } else if (actionType === 'left_double') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'DoubleClick',\n        param: {\n          locate,\n        },\n        thought: action.thought || '',\n      } satisfies LocatePlanningAction<'DoubleClick'>);\n    } else if (actionType === 'right_single') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'RightClick',\n        param: {\n          locate,\n        },\n        thought: action.thought || '',\n      } satisfies LocatePlanningAction<'RightClick'>);\n    } else if (actionType === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box);\n      const endPoint = getPoint(action.action_inputs.end_box);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            point: startPoint,\n            prompt: action.thought || '',\n          },\n          to: {\n            point: endPoint,\n            prompt: action.thought || '',\n          },\n        },\n        thought: action.thought || '',\n      } satisfies DragAndDropPlanningAction);\n    } else if (actionType === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'finished') {\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.action_inputs.content || action.thought || '',\n      });\n    } else if (actionType === 'hotkey') {\n      if (!action.action_inputs.key) {\n        warnLog('No key found in action: hotkey. Will not perform action.');\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys.join('+'),\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (actionType === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType) {\n      unhandledActions.push({\n        type: actionType,\n        thought: action.thought || '',\n      });\n      debug('Unhandled action type:', actionType, 'thought:', action.thought);\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(\n      buildNoUiTarsActionsError(\n        parsedPlanningResponse.rawResponse,\n        parsedPlanningResponse.actions,\n        unhandledActions,\n      ),\n    );\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n  return transformActions;\n}\n\nfunction getPoint(startBox: string): [number, number] {\n  const [x, y] = JSON.parse(startBox);\n  assert(\n    typeof x === 'number' &&\n      Number.isFinite(x) &&\n      typeof y === 'number' &&\n      Number.isFinite(y),\n    `invalid point data for ui-tars planning: ${startBox}`,\n  );\n  return [x, y];\n}\n\nfunction buildNoUiTarsActionsError(\n  rawResponse: string,\n  actions: UiTarsParsedPlanningResponse['actions'],\n  unhandledActions: Array<{ type: string; thought: string }>,\n): string {\n  const errorDetails: string[] = [];\n\n  if (actions.length === 0) {\n    errorDetails.push('Action parser returned no actions');\n\n    if (rawResponse.includes('Thought:') && !rawResponse.includes('Action:')) {\n      errorDetails.push(\n        'Response contains \"Thought:\" but missing \"Action:\" line',\n      );\n    } else {\n      errorDetails.push('Response may be malformed or empty');\n    }\n  }\n\n  if (unhandledActions.length > 0) {\n    const types = unhandledActions.map((a) => a.type).join(', ');\n    errorDetails.push(`Unhandled action types: ${types}`);\n  }\n\n  return ['No actions found in UI-TARS response.', ...errorDetails].join('\\n');\n}\n"],"names":["debug","getDebug","warnLog","transformUiTarsActions","parsedPlanningResponse","transformActions","unhandledActions","action","actionType","assert","point","getPoint","locate","startPoint","endPoint","keys","transformHotkeyInput","Error","buildNoUiTarsActionsError","JSON","startBox","x","y","Number","rawResponse","actions","errorDetails","types","a"],"mappings":";;;AAUA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,oBAAoB;IAAE,SAAS;AAAK;AAEtD,SAASE,uBACdC,sBAAoD;IAEpD,MAAMC,mBAAqC,EAAE;IAC7C,MAAMC,mBAA6D,EAAE;IAErEF,uBAAuB,OAAO,CAAC,OAAO,CAAC,CAACG;QACtC,MAAMC,aAAcD,AAAAA,CAAAA,OAAO,WAAW,IAAI,EAAC,EAAG,WAAW;QACzD,IAAIC,AAAe,YAAfA,YAAwB;YAC1BC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;YACF;QACF,OAAO,IAAIJ,AAAe,kBAAfA,YAA8B;YACvCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,mBAAfA,YAA+B;YACxCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YAAuB;YAChCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCE,OAAOF,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMM,aAAaF,SAASJ,OAAO,aAAa,CAAC,SAAS;YAC1D,MAAMO,WAAWH,SAASJ,OAAO,aAAa,CAAC,OAAO;YACtDF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,OAAOQ;wBACP,QAAQN,OAAO,OAAO,IAAI;oBAC5B;oBACA,IAAI;wBACF,OAAOO;wBACP,QAAQP,OAAO,OAAO,IAAI;oBAC5B;gBACF;gBACA,SAASA,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOE,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,aAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWE,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,eAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO,CAAC;YACR,SAASE,OAAO,aAAa,CAAC,OAAO,IAAIA,OAAO,OAAO,IAAI;QAC7D;aACK,IAAIC,AAAe,aAAfA,YACT,IAAKD,OAAO,aAAa,CAAC,GAAG,EAEtB;YACL,MAAMQ,OAAOC,qBAAqBT,OAAO,aAAa,CAAC,GAAG;YAE1DF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASU,KAAK,IAAI,CAAC;gBACrB;gBACA,SAASR,OAAO,OAAO,IAAI;YAC7B;QACF,OAXEL,QAAQ;aAYL,IAAIM,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASE,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,YAAY;YACrBF,iBAAiB,IAAI,CAAC;gBACpB,MAAME;gBACN,SAASD,OAAO,OAAO,IAAI;YAC7B;YACAP,MAAM,0BAA0BQ,YAAY,YAAYD,OAAO,OAAO;QACxE;IACF;IAEA,IAAIF,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIY,MACRC,0BACEd,uBAAuB,WAAW,EAClCA,uBAAuB,OAAO,EAC9BE;IAKNN,MAAM,oBAAoBmB,KAAK,SAAS,CAACd,kBAAkB,MAAM;IACjE,OAAOA;AACT;AAEA,SAASM,SAASS,QAAgB;IAChC,MAAM,CAACC,GAAGC,EAAE,GAAGH,KAAK,KAAK,CAACC;IAC1BX,OACE,AAAa,YAAb,OAAOY,KACLE,OAAO,QAAQ,CAACF,MAChB,AAAa,YAAb,OAAOC,KACPC,OAAO,QAAQ,CAACD,IAClB,CAAC,yCAAyC,EAAEF,UAAU;IAExD,OAAO;QAACC;QAAGC;KAAE;AACf;AAEA,SAASJ,0BACPM,WAAmB,EACnBC,OAAgD,EAChDnB,gBAA0D;IAE1D,MAAMoB,eAAyB,EAAE;IAEjC,IAAID,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;QACxBC,aAAa,IAAI,CAAC;QAElB,IAAIF,YAAY,QAAQ,CAAC,eAAe,CAACA,YAAY,QAAQ,CAAC,YAC5DE,aAAa,IAAI,CACf;aAGFA,aAAa,IAAI,CAAC;IAEtB;IAEA,IAAIpB,iBAAiB,MAAM,GAAG,GAAG;QAC/B,MAAMqB,QAAQrB,iBAAiB,GAAG,CAAC,CAACsB,IAAMA,EAAE,IAAI,EAAE,IAAI,CAAC;QACvDF,aAAa,IAAI,CAAC,CAAC,wBAAwB,EAAEC,OAAO;IACtD;IAEA,OAAO;QAAC;WAA4CD;KAAa,CAAC,IAAI,CAAC;AACzE"}

package/dist/es/ai-model/models/ui-tars/adapter.mjs

@@ -0,0 +1,102 @@
+import { UITarsModelVersion } from "@aiscene/shared/env";
+import { assert } from "@aiscene/shared/utils";
+import { createLocateResultValue, unwrapCoordinateListLikeInput } from "../../shared/model-locate-result/index.mjs";
+import { createUiTarsPlanner } from "./planning.mjs";
+const defaultVlmUiTarsReplanningCycleLimit = 40;
+const uiTarsBboxCoordinatesMeta = {
+    shape: 'bbox',
+    order: 'xy',
+    normalizedBy: 1000
+};
+const uiTarsPointCoordinatesMeta = {
+    shape: 'point',
+    order: 'xy',
+    normalizedBy: 1000
+};
+function parseUiTarsRawLocateValue(input) {
+    const bbox = unwrapCoordinateListLikeInput(input);
+    if ('string' == typeof bbox) {
+        assert(/^(\d+)\s(\d+)\s(\d+)\s(\d+)$/.test(bbox.trim()), `invalid bbox data string for ui-tars mode: ${bbox}`);
+        const splitted = bbox.split(' ');
+        if (4 === splitted.length) return createLocateResultValue(uiTarsBboxCoordinatesMeta, [
+            Number(splitted[0]),
+            Number(splitted[1]),
+            Number(splitted[2]),
+            Number(splitted[3])
+        ]);
+        throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);
+    }
+    let bboxList = [];
+    if (Array.isArray(bbox) && 'string' == typeof bbox[0]) bbox.forEach((item)=>{
+        if ('string' == typeof item && item.includes(',')) {
+            const [x, y] = item.split(',');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else if ('string' == typeof item && item.includes(' ')) {
+            const [x, y] = item.split(' ');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else bboxList.push(Number(item));
+    });
+    else bboxList = bbox;
+    if (4 === bboxList.length || 5 === bboxList.length) return createLocateResultValue(uiTarsBboxCoordinatesMeta, [
+        bboxList[0],
+        bboxList[1],
+        bboxList[2],
+        bboxList[3]
+    ]);
+    if (6 === bboxList.length || 2 === bboxList.length || 3 === bboxList.length || 7 === bboxList.length) return createLocateResultValue(uiTarsPointCoordinatesMeta, [
+        bboxList[0],
+        bboxList[1]
+    ]);
+    if (8 === bbox.length) return createLocateResultValue(uiTarsBboxCoordinatesMeta, [
+        bboxList[0],
+        bboxList[1],
+        bboxList[4],
+        bboxList[5]
+    ]);
+    const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;
+    throw new Error(msg);
+}
+function createUiTarsAdapter(uiTarsModelVersion) {
+    return {
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEnabled',
+                'reasoningEffort',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: ({ midsceneDefaults, userConfig })=>{
+                const commonOverrideConfig = {};
+                if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+                return {
+                    config: {
+                        ...midsceneDefaults,
+                        ...commonOverrideConfig
+                    }
+                };
+            }
+        },
+        planning: {
+            kind: 'custom',
+            cacheEnabled: false,
+            defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,
+            planner: createUiTarsPlanner(uiTarsModelVersion)
+        },
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: uiTarsBboxCoordinatesMeta,
+                    parseRawLocateValue: parseUiTarsRawLocateValue
+                }
+            }
+        }
+    };
+}
+const uiTarsDoubao15Adapter = createUiTarsAdapter(UITarsModelVersion.DOUBAO_1_5_20B);
+const uiTarsAdapters = {
+    'vlm-ui-tars': createUiTarsAdapter(UITarsModelVersion.V1_0),
+    'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,
+    'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter
+};
+export { uiTarsAdapters };
+
+//# sourceMappingURL=adapter.mjs.map