@midscene/core 0.26.7-beta-20250818081955.0 → 0.26.7-beta-20250820105545.0

package/dist/es/ai-model/service-caller/index.mjs

@@ -1,9 +1,9 @@
 import { AIResponseFormat } from "../../types.mjs";
 import { Anthropic } from "@anthropic-ai/sdk";
 import { DefaultAzureCredential, getBearerTokenProvider } from "@azure/identity";
-import { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, uiTarsModelVersion, vlLocateMode } from "@midscene/shared/env";
+import { MIDSCENE_API_TYPE, MIDSCENE_LANGSMITH_DEBUG, OPENAI_MAX_TOKENS, decideModelConfig, getAIConfig, getAIConfigInBoolean, uiTarsModelVersion, vlLocateMode } from "@midscene/shared/env";
 import { parseBase64 } from "@midscene/shared/img";
-import { enableDebug, getDebug } from "@midscene/shared/logger";
+import { getDebug } from "@midscene/shared/logger";
 import { assert, ifInBrowser } from "@midscene/shared/utils";
 import { HttpsProxyAgent } from "https-proxy-agent";
 import { jsonrepair } from "jsonrepair";
@@ -13,46 +13,9 @@ import { AIActionType } from "../common.mjs";
 import { assertSchema } from "../prompt/assertion.mjs";
 import { locatorSchema } from "../prompt/llm-locator.mjs";
 import { planSchema } from "../prompt/llm-planning.mjs";
-function checkAIConfig() {
-    const openaiKey = getAIConfig(OPENAI_API_KEY);
-    const azureConfig = getAIConfig(MIDSCENE_USE_AZURE_OPENAI);
-    const anthropicKey = getAIConfig(ANTHROPIC_API_KEY);
-    const initConfigJson = getAIConfig(MIDSCENE_OPENAI_INIT_CONFIG_JSON);
-    if (openaiKey) return true;
-    if (azureConfig) return true;
-    if (anthropicKey) return true;
-    return Boolean(initConfigJson);
-}
-let debugConfigInitialized = false;
-function initDebugConfig() {
-    if (debugConfigInitialized) return;
-    const shouldPrintTiming = getAIConfigInBoolean(MIDSCENE_DEBUG_AI_PROFILE);
-    let debugConfig = '';
-    if (shouldPrintTiming) {
-        console.warn('MIDSCENE_DEBUG_AI_PROFILE is deprecated, use DEBUG=midscene:ai:profile instead');
-        debugConfig = 'ai:profile';
-    }
-    const shouldPrintAIResponse = getAIConfigInBoolean(MIDSCENE_DEBUG_AI_RESPONSE);
-    if (shouldPrintAIResponse) {
-        console.warn('MIDSCENE_DEBUG_AI_RESPONSE is deprecated, use DEBUG=midscene:ai:response instead');
-        debugConfig = debugConfig ? 'ai:*' : 'ai:call';
-    }
-    if (debugConfig) enableDebug(debugConfig);
-    debugConfigInitialized = true;
-}
-const defaultModel = 'gpt-4o';
-function getModelName() {
-    let modelName = defaultModel;
-    const nameInConfig = getAIConfig(MIDSCENE_MODEL_NAME);
-    if (nameInConfig) modelName = nameInConfig;
-    return modelName;
-}
-async function createChatClient({ AIActionTypeValue }) {
-    initDebugConfig();
+async function createChatClient({ AIActionTypeValue, modelPreferences }) {
+    const { socksProxy, httpProxy, modelName, openaiBaseURL, openaiApiKey, openaiExtraConfig, openaiUseAzureDeprecated, useAzureOpenai, azureOpenaiScope, azureOpenaiKey, azureOpenaiEndpoint, azureOpenaiApiVersion, azureOpenaiDeployment, azureExtraConfig, useAnthropicSdk, anthropicApiKey } = decideModelConfig(modelPreferences, true);
     let openai;
-    const extraConfig = getAIConfigInJson(MIDSCENE_OPENAI_INIT_CONFIG_JSON);
-    const socksProxy = getAIConfig(MIDSCENE_OPENAI_SOCKS_PROXY);
-    const httpProxy = getAIConfig(MIDSCENE_OPENAI_HTTP_PROXY);
     let proxyAgent;
     const debugProxy = getDebug('ai:call:proxy');
     if (httpProxy) {
@@ -62,56 +25,47 @@ async function createChatClient({ AIActionTypeValue }) {
         debugProxy('using socks proxy', socksProxy);
         proxyAgent = new SocksProxyAgent(socksProxy);
     }
-    if (getAIConfig(OPENAI_USE_AZURE)) openai = new AzureOpenAI({
-        baseURL: getAIConfig(OPENAI_BASE_URL),
-        apiKey: getAIConfig(OPENAI_API_KEY),
+    if (openaiUseAzureDeprecated) openai = new AzureOpenAI({
+        baseURL: openaiBaseURL,
+        apiKey: openaiApiKey,
         httpAgent: proxyAgent,
-        ...extraConfig,
+        ...openaiExtraConfig,
         dangerouslyAllowBrowser: true
     });
-    else if (getAIConfig(MIDSCENE_USE_AZURE_OPENAI)) {
-        const extraAzureConfig = getAIConfigInJson(MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON);
-        const scope = getAIConfig(MIDSCENE_AZURE_OPENAI_SCOPE);
+    else if (useAzureOpenai) {
         let tokenProvider;
-        if (scope) {
+        if (azureOpenaiScope) {
             assert(!ifInBrowser, 'Azure OpenAI is not supported in browser with Midscene.');
             const credential = new DefaultAzureCredential();
-            assert(scope, 'MIDSCENE_AZURE_OPENAI_SCOPE is required');
-            tokenProvider = getBearerTokenProvider(credential, scope);
+            tokenProvider = getBearerTokenProvider(credential, azureOpenaiScope);
             openai = new AzureOpenAI({
                 azureADTokenProvider: tokenProvider,
-                endpoint: getAIConfig(AZURE_OPENAI_ENDPOINT),
-                apiVersion: getAIConfig(AZURE_OPENAI_API_VERSION),
-                deployment: getAIConfig(AZURE_OPENAI_DEPLOYMENT),
-                ...extraConfig,
-                ...extraAzureConfig
+                endpoint: azureOpenaiEndpoint,
+                apiVersion: azureOpenaiApiVersion,
+                deployment: azureOpenaiDeployment,
+                ...openaiExtraConfig,
+                ...azureExtraConfig
             });
         } else openai = new AzureOpenAI({
-            apiKey: getAIConfig(AZURE_OPENAI_KEY),
-            endpoint: getAIConfig(AZURE_OPENAI_ENDPOINT),
-            apiVersion: getAIConfig(AZURE_OPENAI_API_VERSION),
-            deployment: getAIConfig(AZURE_OPENAI_DEPLOYMENT),
+            apiKey: azureOpenaiKey,
+            endpoint: azureOpenaiEndpoint,
+            apiVersion: azureOpenaiApiVersion,
+            deployment: azureOpenaiDeployment,
             dangerouslyAllowBrowser: true,
-            ...extraConfig,
-            ...extraAzureConfig
-        });
-    } else if (!getAIConfig(MIDSCENE_USE_ANTHROPIC_SDK)) {
-        const baseURL = getAIConfig(OPENAI_BASE_URL);
-        if ('string' == typeof baseURL) {
-            if (!/^https?:\/\//.test(baseURL)) throw new Error(`OPENAI_BASE_URL must be a valid URL starting with http:// or https://, but got: ${baseURL}\nPlease check your config.`);
-        }
-        openai = new openai_0({
-            baseURL: getAIConfig(OPENAI_BASE_URL),
-            apiKey: getAIConfig(OPENAI_API_KEY),
-            httpAgent: proxyAgent,
-            ...extraConfig,
-            defaultHeaders: {
-                ...(null == extraConfig ? void 0 : extraConfig.defaultHeaders) || {},
-                [MIDSCENE_API_TYPE]: AIActionTypeValue.toString()
-            },
-            dangerouslyAllowBrowser: true
+            ...openaiExtraConfig,
+            ...azureExtraConfig
         });
-    }
+    } else if (!useAnthropicSdk) openai = new openai_0({
+        baseURL: openaiBaseURL,
+        apiKey: openaiApiKey,
+        httpAgent: proxyAgent,
+        ...openaiExtraConfig,
+        defaultHeaders: {
+            ...(null == openaiExtraConfig ? void 0 : openaiExtraConfig.defaultHeaders) || {},
+            [MIDSCENE_API_TYPE]: AIActionTypeValue.toString()
+        },
+        dangerouslyAllowBrowser: true
+    });
     if (openai && getAIConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)) {
         if (ifInBrowser) throw new Error('langsmith is not supported in browser');
         console.log('DEBUGGING MODE: langsmith wrapper enabled');
@@ -120,53 +74,51 @@ async function createChatClient({ AIActionTypeValue }) {
     }
     if (void 0 !== openai) return {
         completion: openai.chat.completions,
-        style: 'openai'
+        style: 'openai',
+        modelName
     };
-    if (getAIConfig(MIDSCENE_USE_ANTHROPIC_SDK)) {
-        const apiKey = getAIConfig(ANTHROPIC_API_KEY);
-        assert(apiKey, 'ANTHROPIC_API_KEY is required');
-        openai = new Anthropic({
-            apiKey,
-            httpAgent: proxyAgent,
-            dangerouslyAllowBrowser: true
-        });
-    }
+    if (useAnthropicSdk) openai = new Anthropic({
+        apiKey: anthropicApiKey,
+        httpAgent: proxyAgent,
+        dangerouslyAllowBrowser: true
+    });
     if (void 0 !== openai && openai.messages) return {
         completion: openai.messages,
-        style: 'anthropic'
+        style: 'anthropic',
+        modelName
     };
     throw new Error('Openai SDK or Anthropic SDK is not initialized');
 }
-async function call(messages, AIActionTypeValue, responseFormat, options) {
-    assert(checkAIConfig(), 'Cannot find config for AI model service. If you are using a self-hosted model without validating the API key, please set `OPENAI_API_KEY` to any non-null value. https://midscenejs.com/model-provider.html');
-    const { completion, style } = await createChatClient({
-        AIActionTypeValue
+async function call(messages, AIActionTypeValue, modelPreferences, options) {
+    const { completion, style, modelName } = await createChatClient({
+        AIActionTypeValue,
+        modelPreferences
     });
+    const responseFormat = getResponseFormat(modelName, AIActionTypeValue);
     const maxTokens = getAIConfig(OPENAI_MAX_TOKENS);
     const debugCall = getDebug('ai:call');
     const debugProfileStats = getDebug('ai:profile:stats');
     const debugProfileDetail = getDebug('ai:profile:detail');
     const startTime = Date.now();
-    const model = getModelName();
     const isStreaming = (null == options ? void 0 : options.stream) && (null == options ? void 0 : options.onChunk);
     let content;
     let accumulated = '';
     let usage;
     let timeCost;
     const commonConfig = {
-        temperature: 'vlm-ui-tars' === vlLocateMode() ? 0.0 : 0.1,
+        temperature: 'vlm-ui-tars' === vlLocateMode(modelPreferences) ? 0.0 : 0.1,
         stream: !!isStreaming,
         max_tokens: 'number' == typeof maxTokens ? maxTokens : Number.parseInt(maxTokens || '2048', 10),
-        ...'qwen-vl' === vlLocateMode() ? {
+        ...'qwen-vl' === vlLocateMode(modelPreferences) ? {
             vl_high_resolution_images: true
         } : {}
     };
     try {
         if ('openai' === style) {
-            debugCall(`sending ${isStreaming ? 'streaming ' : ''}request to ${model}`);
+            debugCall(`sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`);
             if (isStreaming) {
                 const stream = await completion.create({
-                    model,
+                    model: modelName,
                     messages,
                     response_format: responseFormat,
                     ...commonConfig
@@ -209,7 +161,7 @@ async function call(messages, AIActionTypeValue, responseFormat, options) {
                                 completion_tokens: usage.completion_tokens ?? 0,
                                 total_tokens: usage.total_tokens ?? 0,
                                 time_cost: timeCost ?? 0,
-                                model_name: model
+                                model_name: modelName
                             }
                         };
                         options.onChunk(finalChunk);
@@ -217,17 +169,17 @@ async function call(messages, AIActionTypeValue, responseFormat, options) {
                     }
                 }
                 content = accumulated;
-                debugProfileStats(`streaming model, ${model}, mode, ${vlLocateMode() || 'default'}, cost-ms, ${timeCost}`);
+                debugProfileStats(`streaming model, ${modelName}, mode, ${vlLocateMode(modelPreferences) || 'default'}, cost-ms, ${timeCost}`);
             } else {
                 var _result_usage, _result_usage1, _result_usage2;
                 const result = await completion.create({
-                    model,
+                    model: modelName,
                     messages,
                     response_format: responseFormat,
                     ...commonConfig
                 });
                 timeCost = Date.now() - startTime;
-                debugProfileStats(`model, ${model}, mode, ${vlLocateMode() || 'default'}, ui-tars-version, ${uiTarsModelVersion()}, prompt-tokens, ${(null == (_result_usage = result.usage) ? void 0 : _result_usage.prompt_tokens) || ''}, completion-tokens, ${(null == (_result_usage1 = result.usage) ? void 0 : _result_usage1.completion_tokens) || ''}, total-tokens, ${(null == (_result_usage2 = result.usage) ? void 0 : _result_usage2.total_tokens) || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`);
+                debugProfileStats(`model, ${modelName}, mode, ${vlLocateMode(modelPreferences) || 'default'}, ui-tars-version, ${uiTarsModelVersion(modelPreferences)}, prompt-tokens, ${(null == (_result_usage = result.usage) ? void 0 : _result_usage.prompt_tokens) || ''}, completion-tokens, ${(null == (_result_usage1 = result.usage) ? void 0 : _result_usage1.completion_tokens) || ''}, total-tokens, ${(null == (_result_usage2 = result.usage) ? void 0 : _result_usage2.total_tokens) || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`);
                 debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);
                 assert(result.choices, `invalid response from LLM service: ${JSON.stringify(result)}`);
                 content = result.choices[0].message.content;
@@ -254,7 +206,7 @@ async function call(messages, AIActionTypeValue, responseFormat, options) {
             };
             if (isStreaming) {
                 const stream = await completion.create({
-                    model,
+                    model: modelName,
                     system: 'You are a versatile professional in software UI automation',
                     messages: messages.map((m)=>({
                             role: 'user',
@@ -290,7 +242,7 @@ async function call(messages, AIActionTypeValue, responseFormat, options) {
                                 completion_tokens: anthropicUsage.output_tokens ?? 0,
                                 total_tokens: (anthropicUsage.input_tokens ?? 0) + (anthropicUsage.output_tokens ?? 0),
                                 time_cost: timeCost ?? 0,
-                                model_name: model
+                                model_name: modelName
                             } : void 0
                         };
                         options.onChunk(finalChunk);
@@ -300,7 +252,7 @@ async function call(messages, AIActionTypeValue, responseFormat, options) {
                 content = accumulated;
             } else {
                 const result = await completion.create({
-                    model,
+                    model: modelName,
                     system: 'You are a versatile professional in software UI automation',
                     messages: messages.map((m)=>({
                             role: 'user',
@@ -330,7 +282,7 @@ async function call(messages, AIActionTypeValue, responseFormat, options) {
                 completion_tokens: usage.completion_tokens ?? 0,
                 total_tokens: usage.total_tokens ?? 0,
                 time_cost: timeCost ?? 0,
-                model_name: model
+                model_name: modelName
             } : void 0,
             isStreamed: !!isStreaming
         };
@@ -342,10 +294,9 @@ async function call(messages, AIActionTypeValue, responseFormat, options) {
         throw newError;
     }
 }
-async function callToGetJSONObject(messages, AIActionTypeValue) {
+const getResponseFormat = (modelName, AIActionTypeValue)=>{
     let responseFormat;
-    const model = getModelName();
-    if (model.includes('gpt-4')) switch(AIActionTypeValue){
+    if (modelName.includes('gpt-4')) switch(AIActionTypeValue){
         case AIActionType.ASSERT:
             responseFormat = assertSchema;
             break;
@@ -362,19 +313,22 @@ async function callToGetJSONObject(messages, AIActionTypeValue) {
             };
             break;
     }
-    if ('gpt-4o-2024-05-13' === model) responseFormat = {
+    if ('gpt-4o-2024-05-13' === modelName) responseFormat = {
         type: AIResponseFormat.JSON
     };
-    const response = await call(messages, AIActionTypeValue, responseFormat);
+    return responseFormat;
+};
+async function callToGetJSONObject(messages, AIActionTypeValue, modelPreferences) {
+    const response = await call(messages, AIActionTypeValue, modelPreferences);
     assert(response, 'empty response');
-    const jsonContent = safeParseJson(response.content);
+    const jsonContent = safeParseJson(response.content, modelPreferences);
     return {
         content: jsonContent,
         usage: response.usage
     };
 }
-async function callAiFnWithStringResponse(msgs, AIActionTypeValue) {
-    const { content, usage } = await call(msgs, AIActionTypeValue);
+async function callAiFnWithStringResponse(msgs, AIActionTypeValue, modelPreferences) {
+    const { content, usage } = await call(msgs, AIActionTypeValue, modelPreferences);
     return {
         content,
         usage
@@ -395,7 +349,7 @@ function preprocessDoubaoBboxJson(input) {
     if (input.includes('bbox')) while(/\d+\s+\d+/.test(input))input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2');
     return input;
 }
-function safeParseJson(input) {
+function safeParseJson(input, modelPreferences) {
     const cleanJsonString = extractJSONFromCodeBlock(input);
     if (null == cleanJsonString ? void 0 : cleanJsonString.match(/\((\d+),(\d+)\)/)) {
         var _cleanJsonString_match;
@@ -407,12 +361,12 @@ function safeParseJson(input) {
     try {
         return JSON.parse(jsonrepair(cleanJsonString));
     } catch (e) {}
-    if ('doubao-vision' === vlLocateMode() || 'vlm-ui-tars' === vlLocateMode()) {
+    if ('doubao-vision' === vlLocateMode(modelPreferences) || 'vlm-ui-tars' === vlLocateMode(modelPreferences)) {
         const jsonString = preprocessDoubaoBboxJson(cleanJsonString);
         return JSON.parse(jsonrepair(jsonString));
     }
     throw Error(`failed to parse json response: ${input}`);
 }
-export { call, callAiFnWithStringResponse, callToGetJSONObject, checkAIConfig, extractJSONFromCodeBlock, getModelName, preprocessDoubaoBboxJson, safeParseJson };
+export { call, callAiFnWithStringResponse, callToGetJSONObject, extractJSONFromCodeBlock, getResponseFormat, preprocessDoubaoBboxJson, safeParseJson };
 
 //# sourceMappingURL=index.mjs.map

package/dist/es/ai-model/service-caller/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport { Anthropic } from '@anthropic-ai/sdk';\nimport {\n  DefaultAzureCredential,\n  getBearerTokenProvider,\n} from '@azure/identity';\nimport {\n  ANTHROPIC_API_KEY,\n  AZURE_OPENAI_API_VERSION,\n  AZURE_OPENAI_DEPLOYMENT,\n  AZURE_OPENAI_ENDPOINT,\n  AZURE_OPENAI_KEY,\n  MIDSCENE_API_TYPE,\n  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,\n  MIDSCENE_AZURE_OPENAI_SCOPE,\n  MIDSCENE_DEBUG_AI_PROFILE,\n  MIDSCENE_DEBUG_AI_RESPONSE,\n  MIDSCENE_LANGSMITH_DEBUG,\n  MIDSCENE_MODEL_NAME,\n  MIDSCENE_OPENAI_HTTP_PROXY,\n  MIDSCENE_OPENAI_INIT_CONFIG_JSON,\n  MIDSCENE_OPENAI_SOCKS_PROXY,\n  MIDSCENE_USE_ANTHROPIC_SDK,\n  MIDSCENE_USE_AZURE_OPENAI,\n  OPENAI_API_KEY,\n  OPENAI_BASE_URL,\n  OPENAI_MAX_TOKENS,\n  OPENAI_USE_AZURE,\n  getAIConfig,\n  getAIConfigInBoolean,\n  getAIConfigInJson,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { parseBase64 } from '@midscene/shared/img';\nimport { enableDebug, getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ifInBrowser } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI, { AzureOpenAI } from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { locatorSchema } from '../prompt/llm-locator';\nimport { planSchema } from '../prompt/llm-planning';\n\nexport function checkAIConfig() {\n  const openaiKey = getAIConfig(OPENAI_API_KEY);\n  const azureConfig = getAIConfig(MIDSCENE_USE_AZURE_OPENAI);\n  const anthropicKey = getAIConfig(ANTHROPIC_API_KEY);\n  const initConfigJson = getAIConfig(MIDSCENE_OPENAI_INIT_CONFIG_JSON);\n\n  if (openaiKey) return true;\n  if (azureConfig) return true;\n  if (anthropicKey) return true;\n\n  return Boolean(initConfigJson);\n}\n\n// if debug config is initialized\nlet debugConfigInitialized = false;\n\nfunction initDebugConfig() {\n  // if debug config is initialized, return\n  if (debugConfigInitialized) return;\n\n  const shouldPrintTiming = getAIConfigInBoolean(MIDSCENE_DEBUG_AI_PROFILE);\n  let debugConfig = '';\n  if (shouldPrintTiming) {\n    console.warn(\n      'MIDSCENE_DEBUG_AI_PROFILE is deprecated, use DEBUG=midscene:ai:profile instead',\n    );\n    debugConfig = 'ai:profile';\n  }\n  const shouldPrintAIResponse = getAIConfigInBoolean(\n    MIDSCENE_DEBUG_AI_RESPONSE,\n  );\n  if (shouldPrintAIResponse) {\n    console.warn(\n      'MIDSCENE_DEBUG_AI_RESPONSE is deprecated, use DEBUG=midscene:ai:response instead',\n    );\n    if (debugConfig) {\n      debugConfig = 'ai:*';\n    } else {\n      debugConfig = 'ai:call';\n    }\n  }\n  if (debugConfig) {\n    enableDebug(debugConfig);\n  }\n\n  // mark as initialized\n  debugConfigInitialized = true;\n}\n\n// default model\nconst defaultModel = 'gpt-4o';\nexport function getModelName() {\n  let modelName = defaultModel;\n  const nameInConfig = getAIConfig(MIDSCENE_MODEL_NAME);\n  if (nameInConfig) {\n    modelName = nameInConfig;\n  }\n  return modelName;\n}\n\nasync function createChatClient({\n  AIActionTypeValue,\n}: {\n  AIActionTypeValue: AIActionType;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  style: 'openai' | 'anthropic';\n}> {\n  initDebugConfig();\n  let openai: OpenAI | AzureOpenAI | undefined;\n  const extraConfig = getAIConfigInJson(MIDSCENE_OPENAI_INIT_CONFIG_JSON);\n\n  const socksProxy = getAIConfig(MIDSCENE_OPENAI_SOCKS_PROXY);\n  const httpProxy = getAIConfig(MIDSCENE_OPENAI_HTTP_PROXY);\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  if (getAIConfig(OPENAI_USE_AZURE)) {\n    // this is deprecated\n    openai = new AzureOpenAI({\n      baseURL: getAIConfig(OPENAI_BASE_URL),\n      apiKey: getAIConfig(OPENAI_API_KEY),\n      httpAgent: proxyAgent,\n      ...extraConfig,\n      dangerouslyAllowBrowser: true,\n    }) as OpenAI;\n  } else if (getAIConfig(MIDSCENE_USE_AZURE_OPENAI)) {\n    const extraAzureConfig = getAIConfigInJson(\n      MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,\n    );\n\n    // https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart?tabs=bash%2Cjavascript-key%2Ctypescript-keyless%2Cpython&pivots=programming-language-javascript#rest-api\n    // keyless authentication\n    const scope = getAIConfig(MIDSCENE_AZURE_OPENAI_SCOPE);\n    let tokenProvider: any = undefined;\n    if (scope) {\n      assert(\n        !ifInBrowser,\n        'Azure OpenAI is not supported in browser with Midscene.',\n      );\n      const credential = new DefaultAzureCredential();\n\n      assert(scope, 'MIDSCENE_AZURE_OPENAI_SCOPE is required');\n      tokenProvider = getBearerTokenProvider(credential, scope);\n\n      openai = new AzureOpenAI({\n        azureADTokenProvider: tokenProvider,\n        endpoint: getAIConfig(AZURE_OPENAI_ENDPOINT),\n        apiVersion: getAIConfig(AZURE_OPENAI_API_VERSION),\n        deployment: getAIConfig(AZURE_OPENAI_DEPLOYMENT),\n        ...extraConfig,\n        ...extraAzureConfig,\n      });\n    } else {\n      // endpoint, apiKey, apiVersion, deployment\n      openai = new AzureOpenAI({\n        apiKey: getAIConfig(AZURE_OPENAI_KEY),\n        endpoint: getAIConfig(AZURE_OPENAI_ENDPOINT),\n        apiVersion: getAIConfig(AZURE_OPENAI_API_VERSION),\n        deployment: getAIConfig(AZURE_OPENAI_DEPLOYMENT),\n        dangerouslyAllowBrowser: true,\n        ...extraConfig,\n        ...extraAzureConfig,\n      });\n    }\n  } else if (!getAIConfig(MIDSCENE_USE_ANTHROPIC_SDK)) {\n    const baseURL = getAIConfig(OPENAI_BASE_URL);\n    if (typeof baseURL === 'string') {\n      if (!/^https?:\\/\\//.test(baseURL)) {\n        throw new Error(\n          `OPENAI_BASE_URL must be a valid URL starting with http:// or https://, but got: ${baseURL}\\nPlease check your config.`,\n        );\n      }\n    }\n\n    openai = new OpenAI({\n      baseURL: getAIConfig(OPENAI_BASE_URL),\n      apiKey: getAIConfig(OPENAI_API_KEY),\n      httpAgent: proxyAgent,\n      ...extraConfig,\n      defaultHeaders: {\n        ...(extraConfig?.defaultHeaders || {}),\n        [MIDSCENE_API_TYPE]: AIActionTypeValue.toString(),\n      },\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  if (openai && getAIConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    const { wrapOpenAI } = await import('langsmith/wrappers');\n    openai = wrapOpenAI(openai);\n  }\n\n  if (typeof openai !== 'undefined') {\n    return {\n      completion: openai.chat.completions,\n      style: 'openai',\n    };\n  }\n\n  // Anthropic\n  if (getAIConfig(MIDSCENE_USE_ANTHROPIC_SDK)) {\n    const apiKey = getAIConfig(ANTHROPIC_API_KEY);\n    assert(apiKey, 'ANTHROPIC_API_KEY is required');\n    openai = new Anthropic({\n      apiKey,\n      httpAgent: proxyAgent,\n      dangerouslyAllowBrowser: true,\n    }) as any;\n  }\n\n  if (typeof openai !== 'undefined' && (openai as any).messages) {\n    return {\n      completion: (openai as any).messages,\n      style: 'anthropic',\n    };\n  }\n\n  throw new Error('Openai SDK or Anthropic SDK is not initialized');\n}\n\nexport async function call(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  responseFormat?:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  assert(\n    checkAIConfig(),\n    'Cannot find config for AI model service. If you are using a self-hosted model without validating the API key, please set `OPENAI_API_KEY` to any non-null value. https://midscenejs.com/model-provider.html',\n  );\n\n  const { completion, style } = await createChatClient({\n    AIActionTypeValue,\n  });\n\n  const maxTokens = getAIConfig(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n  const model = getModelName();\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlLocateMode() === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlLocateMode() === 'qwen-vl' // qwen specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    if (style === 'openai') {\n      debugCall(\n        `sending ${isStreaming ? 'streaming ' : ''}request to ${model}`,\n      );\n\n      if (isStreaming) {\n        const stream = (await completion.create(\n          {\n            model,\n            messages,\n            response_format: responseFormat,\n            ...commonConfig,\n          },\n          {\n            stream: true,\n          },\n        )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n          _request_id?: string | null;\n        };\n\n        for await (const chunk of stream) {\n          const content = chunk.choices?.[0]?.delta?.content || '';\n          const reasoning_content =\n            (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n          // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n          if (chunk.usage) {\n            usage = chunk.usage;\n          }\n\n          if (content || reasoning_content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              reasoning_content,\n              accumulated,\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.choices?.[0]?.finish_reason) {\n            timeCost = Date.now() - startTime;\n\n            // If usage is not available from the stream, provide a basic usage info\n            if (!usage) {\n              // Estimate token counts based on content length (rough approximation)\n              const estimatedTokens = Math.max(\n                1,\n                Math.floor(accumulated.length / 4),\n              );\n              usage = {\n                prompt_tokens: estimatedTokens,\n                completion_tokens: estimatedTokens,\n                total_tokens: estimatedTokens * 2,\n              };\n            }\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: {\n                prompt_tokens: usage.prompt_tokens ?? 0,\n                completion_tokens: usage.completion_tokens ?? 0,\n                total_tokens: usage.total_tokens ?? 0,\n                time_cost: timeCost ?? 0,\n                model_name: model,\n              },\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n        debugProfileStats(\n          `streaming model, ${model}, mode, ${vlLocateMode() || 'default'}, cost-ms, ${timeCost}`,\n        );\n      } else {\n        const result = await completion.create({\n          model,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n\n        debugProfileStats(\n          `model, ${model}, mode, ${vlLocateMode() || 'default'}, ui-tars-version, ${uiTarsModelVersion()}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n        );\n\n        debugProfileDetail(\n          `model usage detail: ${JSON.stringify(result.usage)}`,\n        );\n\n        assert(\n          result.choices,\n          `invalid response from LLM service: ${JSON.stringify(result)}`,\n        );\n        content = result.choices[0].message.content!;\n        usage = result.usage;\n      }\n\n      debugCall(`response: ${content}`);\n      assert(content, 'empty content');\n    } else if (style === 'anthropic') {\n      const convertImageContent = (content: any) => {\n        if (content.type === 'image_url') {\n          const imgBase64 = content.image_url.url;\n          assert(imgBase64, 'image_url is required');\n          const { mimeType, body } = parseBase64(content.image_url.url);\n          return {\n            source: {\n              type: 'base64',\n              media_type: mimeType,\n              data: body,\n            },\n            type: 'image',\n          };\n        }\n        return content;\n      };\n\n      if (isStreaming) {\n        const stream = (await completion.create({\n          model,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any)) as any;\n\n        for await (const chunk of stream) {\n          const content = chunk.delta?.text || '';\n          if (content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              accumulated,\n              reasoning_content: '',\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.type === 'message_stop') {\n            timeCost = Date.now() - startTime;\n            const anthropicUsage = chunk.usage;\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: anthropicUsage\n                ? {\n                    prompt_tokens: anthropicUsage.input_tokens ?? 0,\n                    completion_tokens: anthropicUsage.output_tokens ?? 0,\n                    total_tokens:\n                      (anthropicUsage.input_tokens ?? 0) +\n                      (anthropicUsage.output_tokens ?? 0),\n                    time_cost: timeCost ?? 0,\n                    model_name: model,\n                  }\n                : undefined,\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n      } else {\n        const result = await completion.create({\n          model,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n        content = (result as any).content[0].text as string;\n        usage = result.usage;\n      }\n\n      assert(content, 'empty content');\n    }\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: model,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport async function callToGetJSONObject<T>(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  const model = getModelName();\n\n  if (model.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.INSPECT_ELEMENT:\n        responseFormat = locatorSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  if (model === 'gpt-4o-2024-05-13') {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  const response = await call(messages, AIActionTypeValue, responseFormat);\n  assert(response, 'empty response');\n  const jsonContent = safeParseJson(response.content);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAiFnWithStringResponse<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await call(msgs, AIActionTypeValue);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(input: string) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (vlLocateMode() === 'doubao-vision' || vlLocateMode() === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["checkAIConfig","openaiKey","getAIConfig","OPENAI_API_KEY","azureConfig","MIDSCENE_USE_AZURE_OPENAI","anthropicKey","ANTHROPIC_API_KEY","initConfigJson","MIDSCENE_OPENAI_INIT_CONFIG_JSON","Boolean","debugConfigInitialized","initDebugConfig","shouldPrintTiming","getAIConfigInBoolean","MIDSCENE_DEBUG_AI_PROFILE","debugConfig","console","shouldPrintAIResponse","MIDSCENE_DEBUG_AI_RESPONSE","enableDebug","defaultModel","getModelName","modelName","nameInConfig","MIDSCENE_MODEL_NAME","createChatClient","AIActionTypeValue","openai","extraConfig","getAIConfigInJson","socksProxy","MIDSCENE_OPENAI_SOCKS_PROXY","httpProxy","MIDSCENE_OPENAI_HTTP_PROXY","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","OPENAI_USE_AZURE","AzureOpenAI","OPENAI_BASE_URL","extraAzureConfig","MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON","scope","MIDSCENE_AZURE_OPENAI_SCOPE","tokenProvider","assert","ifInBrowser","credential","DefaultAzureCredential","getBearerTokenProvider","AZURE_OPENAI_ENDPOINT","AZURE_OPENAI_API_VERSION","AZURE_OPENAI_DEPLOYMENT","AZURE_OPENAI_KEY","MIDSCENE_USE_ANTHROPIC_SDK","baseURL","Error","OpenAI","MIDSCENE_API_TYPE","MIDSCENE_LANGSMITH_DEBUG","wrapOpenAI","apiKey","Anthropic","call","messages","responseFormat","options","completion","style","maxTokens","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","model","isStreaming","content","accumulated","usage","timeCost","commonConfig","vlLocateMode","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","uiTarsModelVersion","JSON","convertImageContent","imgBase64","mimeType","body","parseBase64","m","Array","_chunk_delta","anthropicUsage","e","newError","callToGetJSONObject","AIActionType","assertSchema","locatorSchema","planSchema","AIResponseFormat","response","jsonContent","safeParseJson","callAiFnWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;;;;;AAkDO,SAASA;IACd,MAAMC,YAAYC,YAAYC;IAC9B,MAAMC,cAAcF,YAAYG;IAChC,MAAMC,eAAeJ,YAAYK;IACjC,MAAMC,iBAAiBN,YAAYO;IAEnC,IAAIR,WAAW,OAAO;IACtB,IAAIG,aAAa,OAAO;IACxB,IAAIE,cAAc,OAAO;IAEzB,OAAOI,QAAQF;AACjB;AAGA,IAAIG,yBAAyB;AAE7B,SAASC;IAEP,IAAID,wBAAwB;IAE5B,MAAME,oBAAoBC,qBAAqBC;IAC/C,IAAIC,cAAc;IAClB,IAAIH,mBAAmB;QACrBI,QAAQ,IAAI,CACV;QAEFD,cAAc;IAChB;IACA,MAAME,wBAAwBJ,qBAC5BK;IAEF,IAAID,uBAAuB;QACzBD,QAAQ,IAAI,CACV;QAGAD,cADEA,cACY,SAEA;IAElB;IACA,IAAIA,aACFI,YAAYJ;IAIdL,yBAAyB;AAC3B;AAGA,MAAMU,eAAe;AACd,SAASC;IACd,IAAIC,YAAYF;IAChB,MAAMG,eAAetB,YAAYuB;IACjC,IAAID,cACFD,YAAYC;IAEd,OAAOD;AACT;AAEA,eAAeG,iBAAiB,EAC9BC,iBAAiB,EAGlB;IAICf;IACA,IAAIgB;IACJ,MAAMC,cAAcC,kBAAkBrB;IAEtC,MAAMsB,aAAa7B,YAAY8B;IAC/B,MAAMC,YAAY/B,YAAYgC;IAE9B,IAAIC;IACJ,MAAMC,aAAaC,SAAS;IAC5B,IAAIJ,WAAW;QACbG,WAAW,oBAAoBH;QAC/BE,aAAa,IAAIG,gBAAgBL;IACnC,OAAO,IAAIF,YAAY;QACrBK,WAAW,qBAAqBL;QAChCI,aAAa,IAAII,gBAAgBR;IACnC;IAEA,IAAI7B,YAAYsC,mBAEdZ,SAAS,IAAIa,YAAY;QACvB,SAASvC,YAAYwC;QACrB,QAAQxC,YAAYC;QACpB,WAAWgC;QACX,GAAGN,WAAW;QACd,yBAAyB;IAC3B;SACK,IAAI3B,YAAYG,4BAA4B;QACjD,MAAMsC,mBAAmBb,kBACvBc;QAKF,MAAMC,QAAQ3C,YAAY4C;QAC1B,IAAIC;QACJ,IAAIF,OAAO;YACTG,OACE,CAACC,aACD;YAEF,MAAMC,aAAa,IAAIC;YAEvBH,OAAOH,OAAO;YACdE,gBAAgBK,uBAAuBF,YAAYL;YAEnDjB,SAAS,IAAIa,YAAY;gBACvB,sBAAsBM;gBACtB,UAAU7C,YAAYmD;gBACtB,YAAYnD,YAAYoD;gBACxB,YAAYpD,YAAYqD;gBACxB,GAAG1B,WAAW;gBACd,GAAGc,gBAAgB;YACrB;QACF,OAEEf,SAAS,IAAIa,YAAY;YACvB,QAAQvC,YAAYsD;YACpB,UAAUtD,YAAYmD;YACtB,YAAYnD,YAAYoD;YACxB,YAAYpD,YAAYqD;YACxB,yBAAyB;YACzB,GAAG1B,WAAW;YACd,GAAGc,gBAAgB;QACrB;IAEJ,OAAO,IAAI,CAACzC,YAAYuD,6BAA6B;QACnD,MAAMC,UAAUxD,YAAYwC;QAC5B,IAAI,AAAmB,YAAnB,OAAOgB,SACT;YAAA,IAAI,CAAC,eAAe,IAAI,CAACA,UACvB,MAAM,IAAIC,MACR,CAAC,gFAAgF,EAAED,QAAQ,2BAA2B,CAAC;QAE3H;QAGF9B,SAAS,IAAIgC,SAAO;YAClB,SAAS1D,YAAYwC;YACrB,QAAQxC,YAAYC;YACpB,WAAWgC;YACX,GAAGN,WAAW;YACd,gBAAgB;gBACd,GAAIA,AAAAA,CAAAA,QAAAA,cAAAA,KAAAA,IAAAA,YAAa,cAAc,AAAD,KAAK,CAAC,CAAC;gBACrC,CAACgC,kBAAkB,EAAElC,kBAAkB,QAAQ;YACjD;YACA,yBAAyB;QAC3B;IACF;IAEA,IAAIC,UAAUd,qBAAqBgD,2BAA2B;QAC5D,IAAIb,aACF,MAAM,IAAIU,MAAM;QAElB1C,QAAQ,GAAG,CAAC;QACZ,MAAM,EAAE8C,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC;QACpCnC,SAASmC,WAAWnC;IACtB;IAEA,IAAI,AAAkB,WAAXA,QACT,OAAO;QACL,YAAYA,OAAO,IAAI,CAAC,WAAW;QACnC,OAAO;IACT;IAIF,IAAI1B,YAAYuD,6BAA6B;QAC3C,MAAMO,SAAS9D,YAAYK;QAC3ByC,OAAOgB,QAAQ;QACfpC,SAAS,IAAIqC,UAAU;YACrBD;YACA,WAAW7B;YACX,yBAAyB;QAC3B;IACF;IAEA,IAAI,AAAkB,WAAXP,UAA2BA,OAAe,QAAQ,EAC3D,OAAO;QACL,YAAaA,OAAe,QAAQ;QACpC,OAAO;IACT;IAGF,MAAM,IAAI+B,MAAM;AAClB;AAEO,eAAeO,KACpBC,QAAsC,EACtCxC,iBAA+B,EAC/ByC,cAEmC,EACnCC,OAGC;IAEDrB,OACEhD,iBACA;IAGF,MAAM,EAAEsE,UAAU,EAAEC,KAAK,EAAE,GAAG,MAAM7C,iBAAiB;QACnDC;IACF;IAEA,MAAM6C,YAAYtE,YAAYuE;IAC9B,MAAMC,YAAYrC,SAAS;IAC3B,MAAMsC,oBAAoBtC,SAAS;IACnC,MAAMuC,qBAAqBvC,SAAS;IAEpC,MAAMwC,YAAYC,KAAK,GAAG;IAC1B,MAAMC,QAAQzD;IACd,MAAM0D,cAAcX,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIY;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAaC,AAAmB,kBAAnBA,iBAAmC,MAAM;QACtD,QAAQ,CAAC,CAACN;QACV,YACE,AAAqB,YAArB,OAAOR,YACHA,YACAe,OAAO,QAAQ,CAACf,aAAa,QAAQ;QAC3C,GAAIc,AAAmB,cAAnBA,iBACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACF,IAAIf,AAAU,aAAVA,OAAoB;YACtBG,UACE,CAAC,QAAQ,EAAEM,cAAc,eAAe,GAAG,WAAW,EAAED,OAAO;YAGjE,IAAIC,aAAa;gBACf,MAAMQ,SAAU,MAAMlB,WAAW,MAAM,CACrC;oBACES;oBACAZ;oBACA,iBAAiBC;oBACjB,GAAGiB,YAAY;gBACjB,GACA;oBACE,QAAQ;gBACV;gBAKF,WAAW,MAAMI,SAASD,OAAQ;wBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;oBAtBJ,MAAMX,UAAUS,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;oBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;oBAG3D,IAAIF,MAAM,KAAK,EACbN,QAAQM,MAAM,KAAK;oBAGrB,IAAIR,WAAWY,mBAAmB;wBAChCX,eAAeD;wBACf,MAAMa,YAAiC;4BACrCb;4BACAY;4BACAX;4BACA,YAAY;4BACZ,OAAOa;wBACT;wBACA1B,QAAQ,OAAO,CAAEyB;oBACnB;oBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;wBACrCR,WAAWN,KAAK,GAAG,KAAKD;wBAGxB,IAAI,CAACM,OAAO;4BAEV,MAAMa,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACf,YAAY,MAAM,GAAG;4BAElCC,QAAQ;gCACN,eAAea;gCACf,mBAAmBA;gCACnB,cAAcA,AAAkB,IAAlBA;4BAChB;wBACF;wBAGA,MAAME,aAAkC;4BACtC,SAAS;4BACThB;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO;gCACL,eAAeC,MAAM,aAAa,IAAI;gCACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gCAC9C,cAAcA,MAAM,YAAY,IAAI;gCACpC,WAAWC,YAAY;gCACvB,YAAYL;4BACd;wBACF;wBACAV,QAAQ,OAAO,CAAE6B;wBACjB;oBACF;gBACF;gBACAjB,UAAUC;gBACVP,kBACE,CAAC,iBAAiB,EAAEI,MAAM,QAAQ,EAAEO,kBAAkB,UAAU,WAAW,EAAEF,UAAU;YAE3F,OAAO;oBAUgHe,eAAyDC,gBAAwDC;gBATtO,MAAMC,SAAS,MAAMhC,WAAW,MAAM,CAAC;oBACrCS;oBACAZ;oBACA,iBAAiBC;oBACjB,GAAGiB,YAAY;gBACjB;gBACAD,WAAWN,KAAK,GAAG,KAAKD;gBAExBF,kBACE,CAAC,OAAO,EAAEI,MAAM,QAAQ,EAAEO,kBAAkB,UAAU,mBAAmB,EAAEiB,qBAAqB,iBAAiB,EAAEJ,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEjB,SAAS,aAAa,EAAEkB,OAAO,WAAW,IAAI,IAAI;gBAGtU1B,mBACE,CAAC,oBAAoB,EAAE4B,KAAK,SAAS,CAACF,OAAO,KAAK,GAAG;gBAGvDtD,OACEsD,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACF,SAAS;gBAEhErB,UAAUqB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;gBAC3CnB,QAAQmB,OAAO,KAAK;YACtB;YAEA5B,UAAU,CAAC,UAAU,EAAEO,SAAS;YAChCjC,OAAOiC,SAAS;QAClB,OAAO,IAAIV,AAAU,gBAAVA,OAAuB;YAChC,MAAMkC,sBAAsB,CAACxB;gBAC3B,IAAIA,AAAiB,gBAAjBA,QAAQ,IAAI,EAAkB;oBAChC,MAAMyB,YAAYzB,QAAQ,SAAS,CAAC,GAAG;oBACvCjC,OAAO0D,WAAW;oBAClB,MAAM,EAAEC,QAAQ,EAAEC,IAAI,EAAE,GAAGC,YAAY5B,QAAQ,SAAS,CAAC,GAAG;oBAC5D,OAAO;wBACL,QAAQ;4BACN,MAAM;4BACN,YAAY0B;4BACZ,MAAMC;wBACR;wBACA,MAAM;oBACR;gBACF;gBACA,OAAO3B;YACT;YAEA,IAAID,aAAa;gBACf,MAAMQ,SAAU,MAAMlB,WAAW,MAAM,CAAC;oBACtCS;oBACA,QAAQ;oBACR,UAAUZ,SAAS,GAAG,CAAC,CAAC2C,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiB1C;oBACjB,GAAGiB,YAAY;gBACjB;gBAEA,WAAW,MAAMI,SAASD,OAAQ;wBAChBwB;oBAAhB,MAAM/B,UAAU+B,AAAAA,SAAAA,CAAAA,eAAAA,MAAM,KAAK,AAAD,IAAVA,KAAAA,IAAAA,aAAa,IAAI,AAAD,KAAK;oBACrC,IAAI/B,SAAS;wBACXC,eAAeD;wBACf,MAAMa,YAAiC;4BACrCb;4BACAC;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAOa;wBACT;wBACA1B,QAAQ,OAAO,CAAEyB;oBACnB;oBAGA,IAAIL,AAAe,mBAAfA,MAAM,IAAI,EAAqB;wBACjCL,WAAWN,KAAK,GAAG,KAAKD;wBACxB,MAAMoC,iBAAiBxB,MAAM,KAAK;wBAGlC,MAAMS,aAAkC;4BACtC,SAAS;4BACThB;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO+B,iBACH;gCACE,eAAeA,eAAe,YAAY,IAAI;gCAC9C,mBAAmBA,eAAe,aAAa,IAAI;gCACnD,cACGA,AAAAA,CAAAA,eAAe,YAAY,IAAI,KAC/BA,CAAAA,eAAe,aAAa,IAAI;gCACnC,WAAW7B,YAAY;gCACvB,YAAYL;4BACd,IACAgB;wBACN;wBACA1B,QAAQ,OAAO,CAAE6B;wBACjB;oBACF;gBACF;gBACAjB,UAAUC;YACZ,OAAO;gBACL,MAAMoB,SAAS,MAAMhC,WAAW,MAAM,CAAC;oBACrCS;oBACA,QAAQ;oBACR,UAAUZ,SAAS,GAAG,CAAC,CAAC2C,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiB1C;oBACjB,GAAGiB,YAAY;gBACjB;gBACAD,WAAWN,KAAK,GAAG,KAAKD;gBACxBI,UAAWqB,OAAe,OAAO,CAAC,EAAE,CAAC,IAAI;gBACzCnB,QAAQmB,OAAO,KAAK;YACtB;YAEAtD,OAAOiC,SAAS;QAClB;QAEA,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMa,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEhB,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAea;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASf,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAYL;YACd,IACAgB;YACJ,YAAY,CAAC,CAACf;QAChB;IACF,EAAE,OAAOkC,GAAQ;QACfjG,QAAQ,KAAK,CAAC,kBAAkBiG;QAChC,MAAMC,WAAW,IAAIxD,MACnB,CAAC,eAAe,EAAEqB,cAAc,eAAe,GAAG,kBAAkB,EAAEkC,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,eAAeC,oBACpBjD,QAAsC,EACtCxC,iBAA+B;IAE/B,IAAIyC;IAKJ,MAAMW,QAAQzD;IAEd,IAAIyD,MAAM,QAAQ,CAAC,UACjB,OAAQpD;QACN,KAAK0F,aAAa,MAAM;YACtBjD,iBAAiBkD;YACjB;QACF,KAAKD,aAAa,eAAe;YAC/BjD,iBAAiBmD;YACjB;QACF,KAAKF,aAAa,IAAI;YACpBjD,iBAAiBoD;YACjB;QACF,KAAKH,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChCjD,iBAAiB;gBAAE,MAAMqD,iBAAiB,IAAI;YAAC;YAC/C;IACJ;IAIF,IAAI1C,AAAU,wBAAVA,OACFX,iBAAiB;QAAE,MAAMqD,iBAAiB,IAAI;IAAC;IAGjD,MAAMC,WAAW,MAAMxD,KAAKC,UAAUxC,mBAAmByC;IACzDpB,OAAO0E,UAAU;IACjB,MAAMC,cAAcC,cAAcF,SAAS,OAAO;IAClD,OAAO;QAAE,SAASC;QAAa,OAAOD,SAAS,KAAK;IAAC;AACvD;AAEO,eAAeG,2BACpBC,IAAY,EACZnG,iBAA+B;IAE/B,MAAM,EAAEsD,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,KAAK4D,MAAMnG;IAC5C,OAAO;QAAEsD;QAASE;IAAM;AAC1B;AAEO,SAAS4C,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASR,cAAcQ,KAAa;IACzC,MAAMC,kBAAkBN,yBAAyBK;IAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAAC/C;IACT;IACA,IAAI;QACF,OAAOiB,KAAK,KAAK,CAAC6B;IACpB,EAAE,OAAM,CAAC;IACT,IAAI;QACF,OAAO7B,KAAK,KAAK,CAAC+B,WAAWF;IAC/B,EAAE,OAAOnB,GAAG,CAAC;IAEb,IAAI5B,AAAmB,oBAAnBA,kBAAsCA,AAAmB,kBAAnBA,gBAAkC;QAC1E,MAAMkD,aAAaL,yBAAyBE;QAC5C,OAAO7B,KAAK,KAAK,CAAC+B,WAAWC;IAC/B;IACA,MAAM7E,MAAM,CAAC,+BAA+B,EAAEyE,OAAO;AACvD"}
+{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport { Anthropic } from '@anthropic-ai/sdk';\nimport {\n  DefaultAzureCredential,\n  getBearerTokenProvider,\n} from '@azure/identity';\nimport {\n  type IModelPreferences,\n  MIDSCENE_API_TYPE,\n  MIDSCENE_LANGSMITH_DEBUG,\n  OPENAI_MAX_TOKENS,\n  decideModelConfig,\n  getAIConfig,\n  getAIConfigInBoolean,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { parseBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ifInBrowser } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI, { AzureOpenAI } from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { locatorSchema } from '../prompt/llm-locator';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelPreferences,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelPreferences: IModelPreferences;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  style: 'openai' | 'anthropic';\n  modelName: string;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    openaiUseAzureDeprecated,\n    useAzureOpenai,\n    azureOpenaiScope,\n    azureOpenaiKey,\n    azureOpenaiEndpoint,\n    azureOpenaiApiVersion,\n    azureOpenaiDeployment,\n    azureExtraConfig,\n    useAnthropicSdk,\n    anthropicApiKey,\n  } = decideModelConfig(modelPreferences, true);\n\n  let openai: OpenAI | AzureOpenAI | undefined;\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  if (openaiUseAzureDeprecated) {\n    // this is deprecated\n    openai = new AzureOpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      dangerouslyAllowBrowser: true,\n    }) as OpenAI;\n  } else if (useAzureOpenai) {\n    // https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart?tabs=bash%2Cjavascript-key%2Ctypescript-keyless%2Cpython&pivots=programming-language-javascript#rest-api\n    // keyless authentication\n    let tokenProvider: any = undefined;\n    if (azureOpenaiScope) {\n      assert(\n        !ifInBrowser,\n        'Azure OpenAI is not supported in browser with Midscene.',\n      );\n      const credential = new DefaultAzureCredential();\n\n      tokenProvider = getBearerTokenProvider(credential, azureOpenaiScope);\n\n      openai = new AzureOpenAI({\n        azureADTokenProvider: tokenProvider,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    } else {\n      // endpoint, apiKey, apiVersion, deployment\n      openai = new AzureOpenAI({\n        apiKey: azureOpenaiKey,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        dangerouslyAllowBrowser: true,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    }\n  } else if (!useAnthropicSdk) {\n    openai = new OpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      defaultHeaders: {\n        ...(openaiExtraConfig?.defaultHeaders || {}),\n        [MIDSCENE_API_TYPE]: AIActionTypeValue.toString(),\n      },\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  if (openai && getAIConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    const { wrapOpenAI } = await import('langsmith/wrappers');\n    openai = wrapOpenAI(openai);\n  }\n\n  if (typeof openai !== 'undefined') {\n    return {\n      completion: openai.chat.completions,\n      style: 'openai',\n      modelName,\n    };\n  }\n\n  // Anthropic\n  if (useAnthropicSdk) {\n    openai = new Anthropic({\n      apiKey: anthropicApiKey,\n      httpAgent: proxyAgent,\n      dangerouslyAllowBrowser: true,\n    }) as any;\n  }\n\n  if (typeof openai !== 'undefined' && (openai as any).messages) {\n    return {\n      completion: (openai as any).messages,\n      style: 'anthropic',\n      modelName,\n    };\n  }\n\n  throw new Error('Openai SDK or Anthropic SDK is not initialized');\n}\n\nexport async function call(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const { completion, style, modelName } = await createChatClient({\n    AIActionTypeValue,\n    modelPreferences,\n  });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens = getAIConfig(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlLocateMode(modelPreferences) === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlLocateMode(modelPreferences) === 'qwen-vl' // qwen specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    if (style === 'openai') {\n      debugCall(\n        `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n      );\n\n      if (isStreaming) {\n        const stream = (await completion.create(\n          {\n            model: modelName,\n            messages,\n            response_format: responseFormat,\n            ...commonConfig,\n          },\n          {\n            stream: true,\n          },\n        )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n          _request_id?: string | null;\n        };\n\n        for await (const chunk of stream) {\n          const content = chunk.choices?.[0]?.delta?.content || '';\n          const reasoning_content =\n            (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n          // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n          if (chunk.usage) {\n            usage = chunk.usage;\n          }\n\n          if (content || reasoning_content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              reasoning_content,\n              accumulated,\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.choices?.[0]?.finish_reason) {\n            timeCost = Date.now() - startTime;\n\n            // If usage is not available from the stream, provide a basic usage info\n            if (!usage) {\n              // Estimate token counts based on content length (rough approximation)\n              const estimatedTokens = Math.max(\n                1,\n                Math.floor(accumulated.length / 4),\n              );\n              usage = {\n                prompt_tokens: estimatedTokens,\n                completion_tokens: estimatedTokens,\n                total_tokens: estimatedTokens * 2,\n              };\n            }\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: {\n                prompt_tokens: usage.prompt_tokens ?? 0,\n                completion_tokens: usage.completion_tokens ?? 0,\n                total_tokens: usage.total_tokens ?? 0,\n                time_cost: timeCost ?? 0,\n                model_name: modelName,\n              },\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n        debugProfileStats(\n          `streaming model, ${modelName}, mode, ${vlLocateMode(modelPreferences) || 'default'}, cost-ms, ${timeCost}`,\n        );\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n\n        debugProfileStats(\n          `model, ${modelName}, mode, ${vlLocateMode(modelPreferences) || 'default'}, ui-tars-version, ${uiTarsModelVersion(modelPreferences)}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n        );\n\n        debugProfileDetail(\n          `model usage detail: ${JSON.stringify(result.usage)}`,\n        );\n\n        assert(\n          result.choices,\n          `invalid response from LLM service: ${JSON.stringify(result)}`,\n        );\n        content = result.choices[0].message.content!;\n        usage = result.usage;\n      }\n\n      debugCall(`response: ${content}`);\n      assert(content, 'empty content');\n    } else if (style === 'anthropic') {\n      const convertImageContent = (content: any) => {\n        if (content.type === 'image_url') {\n          const imgBase64 = content.image_url.url;\n          assert(imgBase64, 'image_url is required');\n          const { mimeType, body } = parseBase64(content.image_url.url);\n          return {\n            source: {\n              type: 'base64',\n              media_type: mimeType,\n              data: body,\n            },\n            type: 'image',\n          };\n        }\n        return content;\n      };\n\n      if (isStreaming) {\n        const stream = (await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any)) as any;\n\n        for await (const chunk of stream) {\n          const content = chunk.delta?.text || '';\n          if (content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              accumulated,\n              reasoning_content: '',\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.type === 'message_stop') {\n            timeCost = Date.now() - startTime;\n            const anthropicUsage = chunk.usage;\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: anthropicUsage\n                ? {\n                    prompt_tokens: anthropicUsage.input_tokens ?? 0,\n                    completion_tokens: anthropicUsage.output_tokens ?? 0,\n                    total_tokens:\n                      (anthropicUsage.input_tokens ?? 0) +\n                      (anthropicUsage.output_tokens ?? 0),\n                    time_cost: timeCost ?? 0,\n                    model_name: modelName,\n                  }\n                : undefined,\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n        content = (result as any).content[0].text as string;\n        usage = result.usage;\n      }\n\n      assert(content, 'empty content');\n    }\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.INSPECT_ELEMENT:\n        responseFormat = locatorSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  if (modelName === 'gpt-4o-2024-05-13') {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callToGetJSONObject<T>(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const response = await call(messages, AIActionTypeValue, modelPreferences);\n  assert(response, 'empty response');\n  const jsonContent = safeParseJson(response.content, modelPreferences);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAiFnWithStringResponse<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await call(\n    msgs,\n    AIActionTypeValue,\n    modelPreferences,\n  );\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(\n  input: string,\n  modelPreferences: IModelPreferences,\n) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (\n    vlLocateMode(modelPreferences) === 'doubao-vision' ||\n    vlLocateMode(modelPreferences) === 'vlm-ui-tars'\n  ) {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["createChatClient","AIActionTypeValue","modelPreferences","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","openaiUseAzureDeprecated","useAzureOpenai","azureOpenaiScope","azureOpenaiKey","azureOpenaiEndpoint","azureOpenaiApiVersion","azureOpenaiDeployment","azureExtraConfig","useAnthropicSdk","anthropicApiKey","decideModelConfig","openai","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","AzureOpenAI","tokenProvider","assert","ifInBrowser","credential","DefaultAzureCredential","getBearerTokenProvider","OpenAI","MIDSCENE_API_TYPE","getAIConfigInBoolean","MIDSCENE_LANGSMITH_DEBUG","Error","console","wrapOpenAI","Anthropic","call","messages","options","completion","style","responseFormat","getResponseFormat","maxTokens","getAIConfig","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","vlLocateMode","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","uiTarsModelVersion","JSON","convertImageContent","imgBase64","mimeType","body","parseBase64","m","Array","_chunk_delta","anthropicUsage","e","newError","AIActionType","assertSchema","locatorSchema","planSchema","AIResponseFormat","callToGetJSONObject","response","jsonContent","safeParseJson","callAiFnWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;;;;;AAiCA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,gBAAgB,EAIjB;IAKC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,wBAAwB,EACxBC,cAAc,EACdC,gBAAgB,EAChBC,cAAc,EACdC,mBAAmB,EACnBC,qBAAqB,EACrBC,qBAAqB,EACrBC,gBAAgB,EAChBC,eAAe,EACfC,eAAe,EAChB,GAAGC,kBAAkBjB,kBAAkB;IAExC,IAAIkB;IAEJ,IAAIC;IACJ,MAAMC,aAAaC,SAAS;IAC5B,IAAInB,WAAW;QACbkB,WAAW,oBAAoBlB;QAC/BiB,aAAa,IAAIG,gBAAgBpB;IACnC,OAAO,IAAID,YAAY;QACrBmB,WAAW,qBAAqBnB;QAChCkB,aAAa,IAAII,gBAAgBtB;IACnC;IAEA,IAAIM,0BAEFW,SAAS,IAAIM,YAAY;QACvB,SAASpB;QACT,QAAQC;QACR,WAAWc;QACX,GAAGb,iBAAiB;QACpB,yBAAyB;IAC3B;SACK,IAAIE,gBAAgB;QAGzB,IAAIiB;QACJ,IAAIhB,kBAAkB;YACpBiB,OACE,CAACC,aACD;YAEF,MAAMC,aAAa,IAAIC;YAEvBJ,gBAAgBK,uBAAuBF,YAAYnB;YAEnDS,SAAS,IAAIM,YAAY;gBACvB,sBAAsBC;gBACtB,UAAUd;gBACV,YAAYC;gBACZ,YAAYC;gBACZ,GAAGP,iBAAiB;gBACpB,GAAGQ,gBAAgB;YACrB;QACF,OAEEI,SAAS,IAAIM,YAAY;YACvB,QAAQd;YACR,UAAUC;YACV,YAAYC;YACZ,YAAYC;YACZ,yBAAyB;YACzB,GAAGP,iBAAiB;YACpB,GAAGQ,gBAAgB;QACrB;IAEJ,OAAO,IAAI,CAACC,iBACVG,SAAS,IAAIa,SAAO;QAClB,SAAS3B;QACT,QAAQC;QACR,WAAWc;QACX,GAAGb,iBAAiB;QACpB,gBAAgB;YACd,GAAIA,AAAAA,CAAAA,QAAAA,oBAAAA,KAAAA,IAAAA,kBAAmB,cAAc,AAAD,KAAK,CAAC,CAAC;YAC3C,CAAC0B,kBAAkB,EAAEjC,kBAAkB,QAAQ;QACjD;QACA,yBAAyB;IAC3B;IAGF,IAAImB,UAAUe,qBAAqBC,2BAA2B;QAC5D,IAAIP,aACF,MAAM,IAAIQ,MAAM;QAElBC,QAAQ,GAAG,CAAC;QACZ,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC;QACpCnB,SAASmB,WAAWnB;IACtB;IAEA,IAAI,AAAkB,WAAXA,QACT,OAAO;QACL,YAAYA,OAAO,IAAI,CAAC,WAAW;QACnC,OAAO;QACPf;IACF;IAIF,IAAIY,iBACFG,SAAS,IAAIoB,UAAU;QACrB,QAAQtB;QACR,WAAWG;QACX,yBAAyB;IAC3B;IAGF,IAAI,AAAkB,WAAXD,UAA2BA,OAAe,QAAQ,EAC3D,OAAO;QACL,YAAaA,OAAe,QAAQ;QACpC,OAAO;QACPf;IACF;IAGF,MAAM,IAAIgC,MAAM;AAClB;AAEO,eAAeI,KACpBC,QAAsC,EACtCzC,iBAA+B,EAC/BC,gBAAmC,EACnCyC,OAGC;IAED,MAAM,EAAEC,UAAU,EAAEC,KAAK,EAAExC,SAAS,EAAE,GAAG,MAAML,iBAAiB;QAC9DC;QACAC;IACF;IAEA,MAAM4C,iBAAiBC,kBAAkB1C,WAAWJ;IAEpD,MAAM+C,YAAYC,YAAYC;IAC9B,MAAMC,YAAY5B,SAAS;IAC3B,MAAM6B,oBAAoB7B,SAAS;IACnC,MAAM8B,qBAAqB9B,SAAS;IAEpC,MAAM+B,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcb,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIc;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAaC,AAAmC,kBAAnCA,aAAa5D,oBAAsC,MAAM;QACtE,QAAQ,CAAC,CAACsD;QACV,YACE,AAAqB,YAArB,OAAOR,YACHA,YACAe,OAAO,QAAQ,CAACf,aAAa,QAAQ;QAC3C,GAAIc,AAAmC,cAAnCA,aAAa5D,oBACb;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACF,IAAI2C,AAAU,aAAVA,OAAoB;YACtBM,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAEnD,WAAW;YAGrE,IAAImD,aAAa;gBACf,MAAMQ,SAAU,MAAMpB,WAAW,MAAM,CACrC;oBACE,OAAOvC;oBACPqC;oBACA,iBAAiBI;oBACjB,GAAGe,YAAY;gBACjB,GACA;oBACE,QAAQ;gBACV;gBAKF,WAAW,MAAMI,SAASD,OAAQ;wBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;oBAtBJ,MAAMX,UAAUS,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;oBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;oBAG3D,IAAIF,MAAM,KAAK,EACbN,QAAQM,MAAM,KAAK;oBAGrB,IAAIR,WAAWY,mBAAmB;wBAChCX,eAAeD;wBACf,MAAMa,YAAiC;4BACrCb;4BACAY;4BACAX;4BACA,YAAY;4BACZ,OAAOa;wBACT;wBACA5B,QAAQ,OAAO,CAAE2B;oBACnB;oBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;wBACrCR,WAAWL,KAAK,GAAG,KAAKD;wBAGxB,IAAI,CAACK,OAAO;4BAEV,MAAMa,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACf,YAAY,MAAM,GAAG;4BAElCC,QAAQ;gCACN,eAAea;gCACf,mBAAmBA;gCACnB,cAAcA,AAAkB,IAAlBA;4BAChB;wBACF;wBAGA,MAAME,aAAkC;4BACtC,SAAS;4BACThB;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO;gCACL,eAAeC,MAAM,aAAa,IAAI;gCACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gCAC9C,cAAcA,MAAM,YAAY,IAAI;gCACpC,WAAWC,YAAY;gCACvB,YAAYvD;4BACd;wBACF;wBACAsC,QAAQ,OAAO,CAAE+B;wBACjB;oBACF;gBACF;gBACAjB,UAAUC;gBACVN,kBACE,CAAC,iBAAiB,EAAE/C,UAAU,QAAQ,EAAEyD,aAAa5D,qBAAqB,UAAU,WAAW,EAAE0D,UAAU;YAE/G,OAAO;oBAUoJe,eAAyDC,gBAAwDC;gBAT1Q,MAAMC,SAAS,MAAMlC,WAAW,MAAM,CAAC;oBACrC,OAAOvC;oBACPqC;oBACA,iBAAiBI;oBACjB,GAAGe,YAAY;gBACjB;gBACAD,WAAWL,KAAK,GAAG,KAAKD;gBAExBF,kBACE,CAAC,OAAO,EAAE/C,UAAU,QAAQ,EAAEyD,aAAa5D,qBAAqB,UAAU,mBAAmB,EAAE6E,mBAAmB7E,kBAAkB,iBAAiB,EAAEyE,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEjB,SAAS,aAAa,EAAEkB,OAAO,WAAW,IAAI,IAAI;gBAG1WzB,mBACE,CAAC,oBAAoB,EAAE2B,KAAK,SAAS,CAACF,OAAO,KAAK,GAAG;gBAGvDlD,OACEkD,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACF,SAAS;gBAEhErB,UAAUqB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;gBAC3CnB,QAAQmB,OAAO,KAAK;YACtB;YAEA3B,UAAU,CAAC,UAAU,EAAEM,SAAS;YAChC7B,OAAO6B,SAAS;QAClB,OAAO,IAAIZ,AAAU,gBAAVA,OAAuB;YAChC,MAAMoC,sBAAsB,CAACxB;gBAC3B,IAAIA,AAAiB,gBAAjBA,QAAQ,IAAI,EAAkB;oBAChC,MAAMyB,YAAYzB,QAAQ,SAAS,CAAC,GAAG;oBACvC7B,OAAOsD,WAAW;oBAClB,MAAM,EAAEC,QAAQ,EAAEC,IAAI,EAAE,GAAGC,YAAY5B,QAAQ,SAAS,CAAC,GAAG;oBAC5D,OAAO;wBACL,QAAQ;4BACN,MAAM;4BACN,YAAY0B;4BACZ,MAAMC;wBACR;wBACA,MAAM;oBACR;gBACF;gBACA,OAAO3B;YACT;YAEA,IAAID,aAAa;gBACf,MAAMQ,SAAU,MAAMpB,WAAW,MAAM,CAAC;oBACtC,OAAOvC;oBACP,QAAQ;oBACR,UAAUqC,SAAS,GAAG,CAAC,CAAC4C,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiBxC;oBACjB,GAAGe,YAAY;gBACjB;gBAEA,WAAW,MAAMI,SAASD,OAAQ;wBAChBwB;oBAAhB,MAAM/B,UAAU+B,AAAAA,SAAAA,CAAAA,eAAAA,MAAM,KAAK,AAAD,IAAVA,KAAAA,IAAAA,aAAa,IAAI,AAAD,KAAK;oBACrC,IAAI/B,SAAS;wBACXC,eAAeD;wBACf,MAAMa,YAAiC;4BACrCb;4BACAC;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAOa;wBACT;wBACA5B,QAAQ,OAAO,CAAE2B;oBACnB;oBAGA,IAAIL,AAAe,mBAAfA,MAAM,IAAI,EAAqB;wBACjCL,WAAWL,KAAK,GAAG,KAAKD;wBACxB,MAAMmC,iBAAiBxB,MAAM,KAAK;wBAGlC,MAAMS,aAAkC;4BACtC,SAAS;4BACThB;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO+B,iBACH;gCACE,eAAeA,eAAe,YAAY,IAAI;gCAC9C,mBAAmBA,eAAe,aAAa,IAAI;gCACnD,cACGA,AAAAA,CAAAA,eAAe,YAAY,IAAI,KAC/BA,CAAAA,eAAe,aAAa,IAAI;gCACnC,WAAW7B,YAAY;gCACvB,YAAYvD;4BACd,IACAkE;wBACN;wBACA5B,QAAQ,OAAO,CAAE+B;wBACjB;oBACF;gBACF;gBACAjB,UAAUC;YACZ,OAAO;gBACL,MAAMoB,SAAS,MAAMlC,WAAW,MAAM,CAAC;oBACrC,OAAOvC;oBACP,QAAQ;oBACR,UAAUqC,SAAS,GAAG,CAAC,CAAC4C,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiBxC;oBACjB,GAAGe,YAAY;gBACjB;gBACAD,WAAWL,KAAK,GAAG,KAAKD;gBACxBG,UAAWqB,OAAe,OAAO,CAAC,EAAE,CAAC,IAAI;gBACzCnB,QAAQmB,OAAO,KAAK;YACtB;YAEAlD,OAAO6B,SAAS;QAClB;QAEA,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMa,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEhB,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAea;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASf,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAYvD;YACd,IACAkE;YACJ,YAAY,CAAC,CAACf;QAChB;IACF,EAAE,OAAOkC,GAAQ;QACfpD,QAAQ,KAAK,CAAC,kBAAkBoD;QAChC,MAAMC,WAAW,IAAItD,MACnB,CAAC,eAAe,EAAEmB,cAAc,eAAe,GAAG,kBAAkB,EAAEkC,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,MAAM5C,oBAAoB,CAC/B1C,WACAJ;IAIA,IAAI6C;IAKJ,IAAIzC,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAK2F,aAAa,MAAM;YACtB9C,iBAAiB+C;YACjB;QACF,KAAKD,aAAa,eAAe;YAC/B9C,iBAAiBgD;YACjB;QACF,KAAKF,aAAa,IAAI;YACpB9C,iBAAiBiD;YACjB;QACF,KAAKH,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChC9C,iBAAiB;gBAAE,MAAMkD,iBAAiB,IAAI;YAAC;YAC/C;IACJ;IAIF,IAAI3F,AAAc,wBAAdA,WACFyC,iBAAiB;QAAE,MAAMkD,iBAAiB,IAAI;IAAC;IAGjD,OAAOlD;AACT;AAEO,eAAemD,oBACpBvD,QAAsC,EACtCzC,iBAA+B,EAC/BC,gBAAmC;IAEnC,MAAMgG,WAAW,MAAMzD,KAAKC,UAAUzC,mBAAmBC;IACzD0B,OAAOsE,UAAU;IACjB,MAAMC,cAAcC,cAAcF,SAAS,OAAO,EAAEhG;IACpD,OAAO;QAAE,SAASiG;QAAa,OAAOD,SAAS,KAAK;IAAC;AACvD;AAEO,eAAeG,2BACpBC,IAAY,EACZrG,iBAA+B,EAC/BC,gBAAmC;IAEnC,MAAM,EAAEuD,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMlB,KAC/B6D,MACArG,mBACAC;IAEF,OAAO;QAAEuD;QAASE;IAAM;AAC1B;AAEO,SAAS4C,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASR,cACdQ,KAAa,EACb1G,gBAAmC;IAEnC,MAAM2G,kBAAkBN,yBAAyBK;IAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAAC/C;IACT;IACA,IAAI;QACF,OAAOiB,KAAK,KAAK,CAAC6B;IACpB,EAAE,OAAM,CAAC;IACT,IAAI;QACF,OAAO7B,KAAK,KAAK,CAAC+B,WAAWF;IAC/B,EAAE,OAAOnB,GAAG,CAAC;IAEb,IACE5B,AAAmC,oBAAnCA,aAAa5D,qBACb4D,AAAmC,kBAAnCA,aAAa5D,mBACb;QACA,MAAM8G,aAAaL,yBAAyBE;QAC5C,OAAO7B,KAAK,KAAK,CAAC+B,WAAWC;IAC/B;IACA,MAAM3E,MAAM,CAAC,+BAA+B,EAAEuE,OAAO;AACvD"}

package/dist/es/ai-model/ui-tars-planning.mjs

@@ -16,7 +16,7 @@ const pointToBbox = (point, width, height)=>[
         Math.round(Math.min(point.y + bboxSize / 2, height))
     ];
 async function vlmPlanning(options) {
-    const { conversationHistory, userInstruction, size } = options;
+    const { conversationHistory, userInstruction, size, modelPreferences } = options;
     const systemPrompt = getUiTarsPlanningPrompt() + userInstruction;
     const res = await call([
         {
@@ -24,9 +24,9 @@ async function vlmPlanning(options) {
             content: systemPrompt
         },
         ...conversationHistory
-    ], AIActionType.INSPECT_ELEMENT);
+    ], AIActionType.INSPECT_ELEMENT, modelPreferences);
     const convertedText = convertBboxToCoordinates(res.content);
-    const modelVer = uiTarsModelVersion();
+    const modelVer = uiTarsModelVersion(modelPreferences);
     const { parsed } = actionParser({
         prediction: convertedText,
         factor: [
@@ -211,8 +211,8 @@ function getPoint(startBox, size) {
         y * size.height
     ];
 }
-async function resizeImageForUiTars(imageBase64, size) {
-    if ('vlm-ui-tars' === vlLocateMode() && uiTarsModelVersion() === UITarsModelVersion.V1_5) {
+async function resizeImageForUiTars(imageBase64, size, modelPreferences) {
+    if ('vlm-ui-tars' === vlLocateMode(modelPreferences) && uiTarsModelVersion(modelPreferences) === UITarsModelVersion.V1_5) {
         debug('ui-tars-v1.5, will check image size', size);
         const currentPixels = size.width * size.height;
         const maxPixels = 12845056;