@midscene/core 0.25.4-beta-20250807040242.0 → 0.25.4-beta-20250807062119.0

package/dist/es/{chunk-I5LBWOQA.js → chunk-G2JTYWI6.js}

@@ -5,35 +5,16 @@ import {
   getBearerTokenProvider
 } from "@azure/identity";
 import {
-  ANTHROPIC_API_KEY,
-  AZURE_OPENAI_API_VERSION,
-  AZURE_OPENAI_DEPLOYMENT,
-  AZURE_OPENAI_ENDPOINT,
-  AZURE_OPENAI_KEY,
   MIDSCENE_API_TYPE,
-  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
-  MIDSCENE_AZURE_OPENAI_SCOPE,
-  MIDSCENE_DEBUG_AI_PROFILE,
-  MIDSCENE_DEBUG_AI_RESPONSE,
   MIDSCENE_LANGSMITH_DEBUG,
-  MIDSCENE_MODEL_NAME,
-  MIDSCENE_OPENAI_HTTP_PROXY,
-  MIDSCENE_OPENAI_INIT_CONFIG_JSON,
-  MIDSCENE_OPENAI_SOCKS_PROXY,
-  MIDSCENE_USE_ANTHROPIC_SDK,
-  MIDSCENE_USE_AZURE_OPENAI,
-  OPENAI_API_KEY,
-  OPENAI_BASE_URL,
   OPENAI_MAX_TOKENS,
-  OPENAI_USE_AZURE,
-  getAIConfig,
-  getAIConfigInBoolean,
-  getAIConfigInJson,
+  getAIConfig as getAIConfig2,
+  getAIConfigInBoolean as getAIConfigInBoolean2,
   uiTarsModelVersion,
   vlLocateMode as vlLocateMode3
 } from "@midscene/shared/env";
-import { enableDebug, getDebug as getDebug2 } from "@midscene/shared/logger";
-import { assert as assert3 } from "@midscene/shared/utils";
+import { getDebug as getDebug3 } from "@midscene/shared/logger";
+import { assert as assert4 } from "@midscene/shared/utils";
 import { ifInBrowser } from "@midscene/shared/utils";
 import { HttpsProxyAgent } from "https-proxy-agent";
 import { jsonrepair } from "jsonrepair";
@@ -55,10 +36,11 @@ var AIActionType = /* @__PURE__ */ ((AIActionType2) => {
   AIActionType2[AIActionType2["DESCRIBE_ELEMENT"] = 4] = "DESCRIBE_ELEMENT";
   return AIActionType2;
 })(AIActionType || {});
-async function callAiFn(msgs, AIActionTypeValue) {
+async function callAiFn(msgs, AIActionTypeValue, modelPreferences) {
   const { content, usage } = await callToGetJSONObject(
     msgs,
-    AIActionTypeValue
+    AIActionTypeValue,
+    modelPreferences
   );
   return { content, usage };
 }
@@ -1163,24 +1145,57 @@ pageDescription:
   });
 };
 
-// src/ai-model/service-caller/index.ts
-function checkAIConfig() {
-  const openaiKey = getAIConfig(OPENAI_API_KEY);
-  const azureConfig = getAIConfig(MIDSCENE_USE_AZURE_OPENAI);
-  const anthropicKey = getAIConfig(ANTHROPIC_API_KEY);
-  const initConfigJson = getAIConfig(MIDSCENE_OPENAI_INIT_CONFIG_JSON);
-  if (openaiKey)
-    return true;
-  if (azureConfig)
-    return true;
-  if (anthropicKey)
-    return true;
-  return Boolean(initConfigJson);
+// src/ai-model/service-caller/utils.ts
+import {
+  ANTHROPIC_API_KEY,
+  AZURE_OPENAI_API_VERSION,
+  AZURE_OPENAI_DEPLOYMENT,
+  AZURE_OPENAI_ENDPOINT,
+  AZURE_OPENAI_KEY,
+  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
+  MIDSCENE_AZURE_OPENAI_SCOPE,
+  MIDSCENE_DEBUG_AI_PROFILE,
+  MIDSCENE_DEBUG_AI_RESPONSE,
+  MIDSCENE_MODEL_NAME,
+  MIDSCENE_OPENAI_HTTP_PROXY,
+  MIDSCENE_OPENAI_INIT_CONFIG_JSON,
+  MIDSCENE_OPENAI_SOCKS_PROXY,
+  MIDSCENE_USE_ANTHROPIC_SDK,
+  MIDSCENE_USE_AZURE_OPENAI,
+  MIDSCENE_VQA_ANTHROPIC_API_KEY,
+  MIDSCENE_VQA_AZURE_OPENAI_API_VERSION,
+  MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT,
+  MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT,
+  MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON,
+  MIDSCENE_VQA_AZURE_OPENAI_KEY,
+  MIDSCENE_VQA_AZURE_OPENAI_SCOPE,
+  MIDSCENE_VQA_MODEL_NAME,
+  MIDSCENE_VQA_OPENAI_API_KEY,
+  MIDSCENE_VQA_OPENAI_BASE_URL,
+  MIDSCENE_VQA_OPENAI_HTTP_PROXY,
+  MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON,
+  MIDSCENE_VQA_OPENAI_SOCKS_PROXY,
+  MIDSCENE_VQA_OPENAI_USE_AZURE,
+  MIDSCENE_VQA_USE_ANTHROPIC_SDK,
+  MIDSCENE_VQA_USE_AZURE_OPENAI,
+  OPENAI_API_KEY,
+  OPENAI_BASE_URL,
+  OPENAI_USE_AZURE,
+  getAIConfig,
+  getAIConfigInBoolean,
+  getAIConfigInJson
+} from "@midscene/shared/env";
+import { enableDebug, getDebug as getDebug2 } from "@midscene/shared/logger";
+import { assert as assert3 } from "@midscene/shared/utils";
+function getModelName() {
+  let modelName = "gpt-4o";
+  const nameInConfig = getAIConfig(MIDSCENE_MODEL_NAME);
+  if (nameInConfig) {
+    modelName = nameInConfig;
+  }
+  return modelName;
 }
-var debugConfigInitialized = false;
 function initDebugConfig() {
-  if (debugConfigInitialized)
-    return;
   const shouldPrintTiming = getAIConfigInBoolean(MIDSCENE_DEBUG_AI_PROFILE);
   let debugConfig = "";
   if (shouldPrintTiming) {
@@ -1205,27 +1220,232 @@ function initDebugConfig() {
   if (debugConfig) {
     enableDebug(debugConfig);
   }
-  debugConfigInitialized = true;
 }
-var defaultModel = "gpt-4o";
-function getModelName() {
-  let modelName = defaultModel;
-  const nameInConfig = getAIConfig(MIDSCENE_MODEL_NAME);
-  if (nameInConfig) {
-    modelName = nameInConfig;
+var createAssert = (modelNameKey, modelName) => (value, key, modelVendorFlag) => {
+  if (modelVendorFlag) {
+    assert3(
+      value,
+      `The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName} and ${modelVendorFlag} has also been specified, but got: ${value}
+Please check your config.`
+    );
+  } else {
+    assert3(
+      value,
+      `The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName}, but got: ${value}
+Please check your config.`
+    );
   }
-  return modelName;
-}
+};
+var getModelConfigFromEnv = (modelName, keys, valueAssert) => {
+  const socksProxy = getAIConfig(keys.socksProxy);
+  const httpProxy = getAIConfig(keys.httpProxy);
+  if (getAIConfig(keys.openaiUseAzureDeprecated)) {
+    const openaiBaseURL = getAIConfig(keys.openaiBaseURL);
+    const openaiApiKey = getAIConfig(keys.openaiApiKey);
+    const openaiExtraConfig = getAIConfigInJson(keys.openaiExtraConfig);
+    valueAssert(
+      openaiBaseURL,
+      keys.openaiBaseURL,
+      keys.openaiUseAzureDeprecated
+    );
+    valueAssert(openaiApiKey, keys.openaiApiKey, keys.openaiUseAzureDeprecated);
+    return {
+      socksProxy,
+      httpProxy,
+      modelName,
+      openaiUseAzureDeprecated: true,
+      openaiApiKey,
+      openaiBaseURL,
+      openaiExtraConfig
+    };
+  } else if (getAIConfig(keys.useAzureOpenai)) {
+    const azureOpenaiScope = getAIConfig(keys.azureOpenaiScope);
+    const azureOpenaiApiKey = getAIConfig(keys.azureOpenaiApiKey);
+    const azureOpenaiEndpoint = getAIConfig(keys.azureOpenaiEndpoint);
+    const azureOpenaiDeployment = getAIConfig(keys.azureOpenaiDeployment);
+    const azureOpenaiApiVersion = getAIConfig(keys.azureOpenaiApiVersion);
+    const azureExtraConfig = getAIConfigInJson(keys.azureExtraConfig);
+    const openaiExtraConfig = getAIConfigInJson(keys.openaiExtraConfig);
+    valueAssert(azureOpenaiApiKey, keys.azureOpenaiApiKey, keys.useAzureOpenai);
+    return {
+      socksProxy,
+      httpProxy,
+      modelName,
+      useAzureOpenai: true,
+      azureOpenaiScope,
+      azureOpenaiApiKey,
+      azureOpenaiEndpoint,
+      azureOpenaiDeployment,
+      azureOpenaiApiVersion,
+      azureExtraConfig,
+      openaiExtraConfig
+    };
+  } else if (getAIConfig(keys.useAnthropicSdk)) {
+    const anthropicApiKey = getAIConfig(keys.anthropicApiKey);
+    valueAssert(anthropicApiKey, keys.anthropicApiKey, keys.useAnthropicSdk);
+    return {
+      socksProxy,
+      httpProxy,
+      modelName,
+      useAnthropicSdk: true,
+      anthropicApiKey
+    };
+  } else {
+    const openaiBaseURL = getAIConfig(keys.openaiBaseURL);
+    const openaiApiKey = getAIConfig(keys.openaiApiKey);
+    const openaiExtraConfig = getAIConfigInJson(keys.openaiExtraConfig);
+    valueAssert(openaiBaseURL, keys.openaiBaseURL);
+    valueAssert(openaiApiKey, keys.openaiApiKey);
+    return {
+      socksProxy,
+      httpProxy,
+      modelName,
+      openaiBaseURL,
+      openaiApiKey,
+      openaiExtraConfig
+    };
+  }
+};
+var maskKey = (key, maskChar = "*") => {
+  if (typeof key !== "string" || key.length === 0) {
+    return key;
+  }
+  const prefixLen = 3;
+  const suffixLen = 3;
+  const keepLength = prefixLen + suffixLen;
+  if (key.length <= keepLength) {
+    return key;
+  }
+  const prefix = key.substring(0, prefixLen);
+  const suffix = key.substring(key.length - suffixLen);
+  const maskLength = key.length - keepLength;
+  const mask = maskChar.repeat(maskLength);
+  return `${prefix}${mask}${suffix}`;
+};
+var maskConfig = (config) => {
+  return Object.fromEntries(
+    Object.entries(config).map(([key, value]) => [
+      key,
+      ["openaiApiKey", "azureOpenaiApiKey", "anthropicApiKey"].includes(key) ? maskKey(value) : value
+    ])
+  );
+};
+var decideModelConfig = (modelPreferences) => {
+  initDebugConfig();
+  const debugLog = getDebug2("ai:decideModelConfig");
+  debugLog("modelPreferences", modelPreferences);
+  const isVQAIntent = modelPreferences?.intent === "VQA";
+  const vqaModelName = getAIConfig(MIDSCENE_VQA_MODEL_NAME);
+  if (isVQAIntent && vqaModelName) {
+    debugLog(
+      `current action is a VQA action and detected ${MIDSCENE_VQA_MODEL_NAME} ${vqaModelName}, will only read VQA related model config from process.env`
+    );
+    const config = getModelConfigFromEnv(
+      vqaModelName,
+      {
+        /**
+         * proxy
+         */
+        socksProxy: MIDSCENE_VQA_OPENAI_SOCKS_PROXY,
+        httpProxy: MIDSCENE_VQA_OPENAI_HTTP_PROXY,
+        /**
+         * OpenAI
+         */
+        openaiBaseURL: MIDSCENE_VQA_OPENAI_BASE_URL,
+        openaiApiKey: MIDSCENE_VQA_OPENAI_API_KEY,
+        openaiExtraConfig: MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON,
+        /**
+         * Azure
+         */
+        openaiUseAzureDeprecated: MIDSCENE_VQA_OPENAI_USE_AZURE,
+        useAzureOpenai: MIDSCENE_VQA_USE_AZURE_OPENAI,
+        azureOpenaiScope: MIDSCENE_VQA_AZURE_OPENAI_SCOPE,
+        azureOpenaiApiKey: MIDSCENE_VQA_AZURE_OPENAI_KEY,
+        azureOpenaiEndpoint: MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT,
+        azureOpenaiApiVersion: MIDSCENE_VQA_AZURE_OPENAI_API_VERSION,
+        azureOpenaiDeployment: MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT,
+        azureExtraConfig: MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON,
+        /**
+         * Anthropic
+         */
+        useAnthropicSdk: MIDSCENE_VQA_USE_ANTHROPIC_SDK,
+        anthropicApiKey: MIDSCENE_VQA_ANTHROPIC_API_KEY
+      },
+      createAssert(MIDSCENE_VQA_MODEL_NAME, vqaModelName)
+    );
+    debugLog("got model config for VQA usage:", maskConfig(config));
+    return config;
+  } else {
+    debugLog("read model config from process.env as normal.");
+    const commonModelName = getAIConfig(MIDSCENE_MODEL_NAME);
+    assert3(
+      commonModelName,
+      `${MIDSCENE_MODEL_NAME} is empty, please check your config.`
+    );
+    const config = getModelConfigFromEnv(
+      commonModelName,
+      {
+        /**
+         * proxy
+         */
+        socksProxy: MIDSCENE_OPENAI_SOCKS_PROXY,
+        httpProxy: MIDSCENE_OPENAI_HTTP_PROXY,
+        /**
+         * OpenAI
+         */
+        openaiBaseURL: OPENAI_BASE_URL,
+        openaiApiKey: OPENAI_API_KEY,
+        openaiExtraConfig: MIDSCENE_OPENAI_INIT_CONFIG_JSON,
+        /**
+         * Azure
+         */
+        openaiUseAzureDeprecated: OPENAI_USE_AZURE,
+        useAzureOpenai: MIDSCENE_USE_AZURE_OPENAI,
+        azureOpenaiScope: MIDSCENE_AZURE_OPENAI_SCOPE,
+        azureOpenaiApiKey: AZURE_OPENAI_KEY,
+        azureOpenaiEndpoint: AZURE_OPENAI_ENDPOINT,
+        azureOpenaiApiVersion: AZURE_OPENAI_API_VERSION,
+        azureOpenaiDeployment: AZURE_OPENAI_DEPLOYMENT,
+        azureExtraConfig: MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
+        /**
+         * Anthropic
+         */
+        useAnthropicSdk: MIDSCENE_USE_ANTHROPIC_SDK,
+        anthropicApiKey: ANTHROPIC_API_KEY
+      },
+      createAssert(MIDSCENE_MODEL_NAME, commonModelName)
+    );
+    debugLog("got model config for common usage:", maskConfig(config));
+    return config;
+  }
+};
+
+// src/ai-model/service-caller/index.ts
 async function createChatClient({
-  AIActionTypeValue
+  AIActionTypeValue,
+  modelPreferences
 }) {
-  initDebugConfig();
+  const {
+    socksProxy,
+    httpProxy,
+    modelName,
+    openaiBaseURL,
+    openaiApiKey,
+    openaiExtraConfig,
+    openaiUseAzureDeprecated,
+    useAzureOpenai,
+    azureOpenaiScope,
+    azureOpenaiApiKey,
+    azureOpenaiEndpoint,
+    azureOpenaiApiVersion,
+    azureOpenaiDeployment,
+    azureExtraConfig,
+    useAnthropicSdk,
+    anthropicApiKey
+  } = decideModelConfig(modelPreferences);
   let openai;
-  const extraConfig = getAIConfigInJson(MIDSCENE_OPENAI_INIT_CONFIG_JSON);
-  const socksProxy = getAIConfig(MIDSCENE_OPENAI_SOCKS_PROXY);
-  const httpProxy = getAIConfig(MIDSCENE_OPENAI_HTTP_PROXY);
   let proxyAgent = void 0;
-  const debugProxy = getDebug2("ai:call:proxy");
+  const debugProxy = getDebug3("ai:call:proxy");
   if (httpProxy) {
     debugProxy("using http proxy", httpProxy);
     proxyAgent = new HttpsProxyAgent(httpProxy);
@@ -1233,70 +1453,56 @@ async function createChatClient({
     debugProxy("using socks proxy", socksProxy);
     proxyAgent = new SocksProxyAgent(socksProxy);
   }
-  if (getAIConfig(OPENAI_USE_AZURE)) {
+  if (openaiUseAzureDeprecated) {
     openai = new AzureOpenAI({
-      baseURL: getAIConfig(OPENAI_BASE_URL),
-      apiKey: getAIConfig(OPENAI_API_KEY),
+      baseURL: openaiBaseURL,
+      apiKey: openaiApiKey,
       httpAgent: proxyAgent,
-      ...extraConfig,
+      ...openaiExtraConfig,
       dangerouslyAllowBrowser: true
     });
-  } else if (getAIConfig(MIDSCENE_USE_AZURE_OPENAI)) {
-    const extraAzureConfig = getAIConfigInJson(
-      MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON
-    );
-    const scope = getAIConfig(MIDSCENE_AZURE_OPENAI_SCOPE);
+  } else if (useAzureOpenai) {
     let tokenProvider = void 0;
-    if (scope) {
-      assert3(
+    if (azureOpenaiScope) {
+      assert4(
         !ifInBrowser,
         "Azure OpenAI is not supported in browser with Midscene."
       );
       const credential = new DefaultAzureCredential();
-      assert3(scope, "MIDSCENE_AZURE_OPENAI_SCOPE is required");
-      tokenProvider = getBearerTokenProvider(credential, scope);
+      tokenProvider = getBearerTokenProvider(credential, azureOpenaiScope);
       openai = new AzureOpenAI({
         azureADTokenProvider: tokenProvider,
-        endpoint: getAIConfig(AZURE_OPENAI_ENDPOINT),
-        apiVersion: getAIConfig(AZURE_OPENAI_API_VERSION),
-        deployment: getAIConfig(AZURE_OPENAI_DEPLOYMENT),
-        ...extraConfig,
-        ...extraAzureConfig
+        endpoint: azureOpenaiEndpoint,
+        apiVersion: azureOpenaiApiVersion,
+        deployment: azureOpenaiDeployment,
+        ...openaiExtraConfig,
+        ...azureExtraConfig
       });
     } else {
       openai = new AzureOpenAI({
-        apiKey: getAIConfig(AZURE_OPENAI_KEY),
-        endpoint: getAIConfig(AZURE_OPENAI_ENDPOINT),
-        apiVersion: getAIConfig(AZURE_OPENAI_API_VERSION),
-        deployment: getAIConfig(AZURE_OPENAI_DEPLOYMENT),
+        apiKey: azureOpenaiApiKey,
+        endpoint: azureOpenaiEndpoint,
+        apiVersion: azureOpenaiApiVersion,
+        deployment: azureOpenaiDeployment,
         dangerouslyAllowBrowser: true,
-        ...extraConfig,
-        ...extraAzureConfig
+        ...openaiExtraConfig,
+        ...azureExtraConfig
       });
     }
-  } else if (!getAIConfig(MIDSCENE_USE_ANTHROPIC_SDK)) {
-    const baseURL = getAIConfig(OPENAI_BASE_URL);
-    if (typeof baseURL === "string") {
-      if (!/^https?:\/\//.test(baseURL)) {
-        throw new Error(
-          `OPENAI_BASE_URL must be a valid URL starting with http:// or https://, but got: ${baseURL}
-Please check your config.`
-        );
-      }
-    }
+  } else if (!useAnthropicSdk) {
     openai = new OpenAI({
-      baseURL: getAIConfig(OPENAI_BASE_URL),
-      apiKey: getAIConfig(OPENAI_API_KEY),
+      baseURL: openaiBaseURL,
+      apiKey: openaiApiKey,
       httpAgent: proxyAgent,
-      ...extraConfig,
+      ...openaiExtraConfig,
       defaultHeaders: {
-        ...extraConfig?.defaultHeaders || {},
+        ...openaiExtraConfig?.defaultHeaders || {},
         [MIDSCENE_API_TYPE]: AIActionTypeValue.toString()
       },
       dangerouslyAllowBrowser: true
     });
   }
-  if (openai && getAIConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)) {
+  if (openai && getAIConfigInBoolean2(MIDSCENE_LANGSMITH_DEBUG)) {
     if (ifInBrowser) {
       throw new Error("langsmith is not supported in browser");
     }
@@ -1307,14 +1513,13 @@ Please check your config.`
   if (typeof openai !== "undefined") {
     return {
       completion: openai.chat.completions,
-      style: "openai"
+      style: "openai",
+      modelName
     };
   }
-  if (getAIConfig(MIDSCENE_USE_ANTHROPIC_SDK)) {
-    const apiKey = getAIConfig(ANTHROPIC_API_KEY);
-    assert3(apiKey, "ANTHROPIC_API_KEY is required");
+  if (useAnthropicSdk) {
     openai = new Anthropic({
-      apiKey,
+      apiKey: anthropicApiKey,
       httpAgent: proxyAgent,
       dangerouslyAllowBrowser: true
     });
@@ -1322,25 +1527,23 @@ Please check your config.`
   if (typeof openai !== "undefined" && openai.messages) {
     return {
       completion: openai.messages,
-      style: "anthropic"
+      style: "anthropic",
+      modelName
     };
   }
   throw new Error("Openai SDK or Anthropic SDK is not initialized");
 }
-async function call2(messages, AIActionTypeValue, responseFormat, options) {
-  assert3(
-    checkAIConfig(),
-    "Cannot find config for AI model service. If you are using a self-hosted model without validating the API key, please set `OPENAI_API_KEY` to any non-null value. https://midscenejs.com/model-provider.html"
-  );
-  const { completion, style } = await createChatClient({
-    AIActionTypeValue
+async function call2(messages, AIActionTypeValue, options, modelPreferences) {
+  const { completion, style, modelName } = await createChatClient({
+    AIActionTypeValue,
+    modelPreferences
   });
-  const maxTokens = getAIConfig(OPENAI_MAX_TOKENS);
-  const debugCall = getDebug2("ai:call");
-  const debugProfileStats = getDebug2("ai:profile:stats");
-  const debugProfileDetail = getDebug2("ai:profile:detail");
+  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);
+  const maxTokens = getAIConfig2(OPENAI_MAX_TOKENS);
+  const debugCall = getDebug3("ai:call");
+  const debugProfileStats = getDebug3("ai:profile:stats");
+  const debugProfileDetail = getDebug3("ai:profile:detail");
   const startTime = Date.now();
-  const model = getModelName();
   const isStreaming = options?.stream && options?.onChunk;
   let content;
   let accumulated = "";
@@ -1357,12 +1560,12 @@ async function call2(messages, AIActionTypeValue, responseFormat, options) {
   try {
     if (style === "openai") {
       debugCall(
-        `sending ${isStreaming ? "streaming " : ""}request to ${model}`
+        `sending ${isStreaming ? "streaming " : ""}request to ${modelName}`
       );
       if (isStreaming) {
         const stream = await completion.create(
           {
-            model,
+            model: modelName,
             messages,
             response_format: responseFormat,
             ...commonConfig
@@ -1419,23 +1622,23 @@ async function call2(messages, AIActionTypeValue, responseFormat, options) {
         }
         content = accumulated;
         debugProfileStats(
-          `streaming model, ${model}, mode, ${vlLocateMode3() || "default"}, cost-ms, ${timeCost}`
+          `streaming model, ${modelName}, mode, ${vlLocateMode3() || "default"}, cost-ms, ${timeCost}`
         );
       } else {
         const result = await completion.create({
-          model,
+          model: modelName,
           messages,
           response_format: responseFormat,
           ...commonConfig
         });
         timeCost = Date.now() - startTime;
         debugProfileStats(
-          `model, ${model}, mode, ${vlLocateMode3() || "default"}, ui-tars-version, ${uiTarsModelVersion()}, prompt-tokens, ${result.usage?.prompt_tokens || ""}, completion-tokens, ${result.usage?.completion_tokens || ""}, total-tokens, ${result.usage?.total_tokens || ""}, cost-ms, ${timeCost}, requestId, ${result._request_id || ""}`
+          `model, ${modelName}, mode, ${vlLocateMode3() || "default"}, ui-tars-version, ${uiTarsModelVersion()}, prompt-tokens, ${result.usage?.prompt_tokens || ""}, completion-tokens, ${result.usage?.completion_tokens || ""}, total-tokens, ${result.usage?.total_tokens || ""}, cost-ms, ${timeCost}, requestId, ${result._request_id || ""}`
         );
         debugProfileDetail(
           `model usage detail: ${JSON.stringify(result.usage)}`
         );
-        assert3(
+        assert4(
           result.choices,
           `invalid response from LLM service: ${JSON.stringify(result)}`
         );
@@ -1443,12 +1646,12 @@ async function call2(messages, AIActionTypeValue, responseFormat, options) {
         usage = result.usage;
       }
       debugCall(`response: ${content}`);
-      assert3(content, "empty content");
+      assert4(content, "empty content");
     } else if (style === "anthropic") {
       const convertImageContent = (content2) => {
         if (content2.type === "image_url") {
           const imgBase64 = content2.image_url.url;
-          assert3(imgBase64, "image_url is required");
+          assert4(imgBase64, "image_url is required");
           return {
             source: {
               type: "base64",
@@ -1462,7 +1665,7 @@ async function call2(messages, AIActionTypeValue, responseFormat, options) {
       };
       if (isStreaming) {
         const stream = await completion.create({
-          model,
+          model: modelName,
           system: "You are a versatile professional in software UI automation",
           messages: messages.map((m) => ({
             role: "user",
@@ -1506,7 +1709,7 @@ async function call2(messages, AIActionTypeValue, responseFormat, options) {
         content = accumulated;
       } else {
         const result = await completion.create({
-          model,
+          model: modelName,
           system: "You are a versatile professional in software UI automation",
           messages: messages.map((m) => ({
             role: "user",
@@ -1519,7 +1722,7 @@ async function call2(messages, AIActionTypeValue, responseFormat, options) {
         content = result.content[0].text;
         usage = result.usage;
       }
-      assert3(content, "empty content");
+      assert4(content, "empty content");
     }
     if (isStreaming && !usage) {
       const estimatedTokens = Math.max(
@@ -1553,10 +1756,9 @@ async function call2(messages, AIActionTypeValue, responseFormat, options) {
     throw newError;
   }
 }
-async function callToGetJSONObject(messages, AIActionTypeValue) {
+var getResponseFormat = (modelName, AIActionTypeValue) => {
   let responseFormat;
-  const model = getModelName();
-  if (model.includes("gpt-4")) {
+  if (modelName.includes("gpt-4")) {
     switch (AIActionTypeValue) {
       case 0 /* ASSERT */:
         responseFormat = assertSchema;
@@ -1573,11 +1775,19 @@ async function callToGetJSONObject(messages, AIActionTypeValue) {
         break;
     }
   }
-  if (model === "gpt-4o-2024-05-13") {
+  if (modelName === "gpt-4o-2024-05-13") {
     responseFormat = { type: "json_object" /* JSON */ };
   }
-  const response = await call2(messages, AIActionTypeValue, responseFormat);
-  assert3(response, "empty response");
+  return responseFormat;
+};
+async function callToGetJSONObject(messages, AIActionTypeValue, modelPreferences) {
+  const response = await call2(
+    messages,
+    AIActionTypeValue,
+    void 0,
+    modelPreferences
+  );
+  assert4(response, "empty response");
   const jsonContent = safeParseJson(response.content);
   return { content: jsonContent, usage: response.usage };
 }
@@ -1861,7 +2071,7 @@ Respond with YAML only, no explanations.`
       });
     }
     if (options.stream && options.onChunk) {
-      return await call2(prompt, 2 /* EXTRACT_DATA */, void 0, {
+      return await call2(prompt, 2 /* EXTRACT_DATA */, {
         stream: true,
         onChunk: options.onChunk
       });
@@ -1984,7 +2194,7 @@ ${PLAYWRIGHT_EXAMPLE_CODE}`;
     }
   ];
   if (options.stream && options.onChunk) {
-    return await call2(prompt, 2 /* EXTRACT_DATA */, void 0, {
+    return await call2(prompt, 2 /* EXTRACT_DATA */, {
       stream: true,
       onChunk: options.onChunk
     });
@@ -2005,7 +2215,7 @@ ${PLAYWRIGHT_EXAMPLE_CODE}`;
 import {
   MIDSCENE_USE_QWEN_VL,
   MIDSCENE_USE_VLM_UI_TARS,
-  getAIConfigInBoolean as getAIConfigInBoolean2,
+  getAIConfigInBoolean as getAIConfigInBoolean3,
   vlLocateMode as vlLocateMode4
 } from "@midscene/shared/env";
 import {
@@ -2013,8 +2223,8 @@ import {
   paddingToMatchBlockByBase64,
   preProcessImageUrl
 } from "@midscene/shared/img";
-import { getDebug as getDebug3 } from "@midscene/shared/logger";
-import { assert as assert4 } from "@midscene/shared/utils";
+import { getDebug as getDebug4 } from "@midscene/shared/logger";
+import { assert as assert5 } from "@midscene/shared/utils";
 
 // src/ai-model/prompt/extraction.ts
 import { PromptTemplate as PromptTemplate3 } from "@langchain/core/prompts";
@@ -2169,8 +2379,8 @@ var sectionLocatorInstruction = new PromptTemplate4({
 });
 
 // src/ai-model/inspect.ts
-var debugInspect = getDebug3("ai:inspect");
-var debugSection = getDebug3("ai:section");
+var debugInspect = getDebug4("ai:inspect");
+var debugSection = getDebug4("ai:section");
 var extraTextFromUserPrompt = (prompt) => {
   if (typeof prompt === "string") {
     return prompt;
@@ -2224,7 +2434,7 @@ async function AiLocateElement(options) {
   const { context, targetElementDescription, callAI } = options;
   const { screenshotBase64 } = context;
   const { description, elementById, insertElementByPosition } = await describeUserPage(context);
-  assert4(
+  assert5(
     targetElementDescription,
     "cannot find the target element description"
   );
@@ -2235,11 +2445,11 @@ async function AiLocateElement(options) {
   const systemPrompt = systemPromptToLocateElement(vlLocateMode4());
   let imagePayload = screenshotBase64;
   if (options.searchConfig) {
-    assert4(
+    assert5(
       options.searchConfig.rect,
       "searchArea is provided but its rect cannot be found. Failed to locate element"
     );
-    assert4(
+    assert5(
       options.searchConfig.imageBase64,
       "searchArea is provided but its imageBase64 cannot be found. Failed to locate element"
     );
@@ -2391,7 +2601,7 @@ async function AiLocateSection(options) {
     imageBase64 = await cropByRect(
       screenshotBase64,
       sectionRect,
-      getAIConfigInBoolean2(MIDSCENE_USE_QWEN_VL)
+      getAIConfigInBoolean3(MIDSCENE_USE_QWEN_VL)
     );
   }
   return {
@@ -2403,7 +2613,13 @@ async function AiLocateSection(options) {
   };
 }
 async function AiExtractElementInfo(options) {
-  const { dataQuery, context, extractOption, multimodalPrompt } = options;
+  const {
+    dataQuery,
+    context,
+    extractOption,
+    multimodalPrompt,
+    modelPreferences
+  } = options;
   const systemPrompt = systemPromptToExtract();
   const { screenshotBase64 } = context;
   const { description, elementById } = await describeUserPage(context, {
@@ -2452,7 +2668,8 @@ async function AiExtractElementInfo(options) {
   }
   const result = await callAiFn(
     msgs,
-    2 /* EXTRACT_DATA */
+    2 /* EXTRACT_DATA */,
+    modelPreferences
   );
   return {
     parseResult: result.content,
@@ -2462,10 +2679,10 @@ async function AiExtractElementInfo(options) {
 }
 async function AiAssert(options) {
   const { assertion, context } = options;
-  assert4(assertion, "assertion should not be empty");
+  assert5(assertion, "assertion should not be empty");
   const { screenshotBase64 } = context;
   const systemPrompt = systemPromptToAssert({
-    isUITars: getAIConfigInBoolean2(MIDSCENE_USE_VLM_UI_TARS)
+    isUITars: getAIConfigInBoolean3(MIDSCENE_USE_VLM_UI_TARS)
   });
   const assertionText = extraTextFromUserPrompt(assertion);
   const msgs = [
@@ -2512,7 +2729,7 @@ ${assertionText}
 // src/ai-model/llm-planning.ts
 import { vlLocateMode as vlLocateMode5 } from "@midscene/shared/env";
 import { paddingToMatchBlockByBase64 as paddingToMatchBlockByBase642 } from "@midscene/shared/img";
-import { assert as assert5 } from "@midscene/shared/utils";
+import { assert as assert6 } from "@midscene/shared/utils";
 async function plan(userInstruction, opts) {
   const { callAI, context } = opts || {};
   const { screenshotBase64, size } = context;
@@ -2574,7 +2791,7 @@ async function plan(userInstruction, opts) {
     usage,
     yamlFlow: buildYamlFlowFromPlans(actions, planFromAI.sleep)
   };
-  assert5(planFromAI, "can't get plans from AI");
+  assert6(planFromAI, "can't get plans from AI");
   if (vlLocateMode5()) {
     actions.forEach((action) => {
       if (action.locate) {
@@ -2590,7 +2807,7 @@ async function plan(userInstruction, opts) {
         }
       }
     });
-    assert5(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);
+    assert6(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);
   } else {
     actions.forEach((action) => {
       if (action.locate?.id) {
@@ -2618,8 +2835,8 @@ import {
 } from "@midscene/shared/env";
 import { resizeImgBase64 } from "@midscene/shared/img";
 import { transformHotkeyInput } from "@midscene/shared/keyboard-layout";
-import { getDebug as getDebug4 } from "@midscene/shared/logger";
-import { assert as assert6 } from "@midscene/shared/utils";
+import { getDebug as getDebug5 } from "@midscene/shared/logger";
+import { assert as assert7 } from "@midscene/shared/utils";
 import { actionParser } from "@ui-tars/action-parser";
 
 // src/ai-model/prompt/ui-tars-planning.ts
@@ -2658,7 +2875,7 @@ finished(content='xxx') # Use escape characters \\', \\", and \\n in content par
 var getSummary = (prediction) => prediction.replace(/Reflection:[\s\S]*?(?=Action_Summary:|Action:|$)/g, "").trim();
 
 // src/ai-model/ui-tars-planning.ts
-var debug = getDebug4("ui-tars-planning");
+var debug = getDebug5("ui-tars-planning");
 var bboxSize = 10;
 var pointToBbox = (point, width, height) => {
   return [
@@ -2696,7 +2913,7 @@ async function vlmPlanning(options) {
   const transformActions = [];
   parsed.forEach((action) => {
     if (action.action_type === "click") {
-      assert6(action.action_inputs.start_box, "start_box is required");
+      assert7(action.action_inputs.start_box, "start_box is required");
       const point = getPoint(action.action_inputs.start_box, size);
       transformActions.push({
         type: "Locate",
@@ -2723,8 +2940,8 @@ async function vlmPlanning(options) {
         param: action.thought || ""
       });
     } else if (action.action_type === "drag") {
-      assert6(action.action_inputs.start_box, "start_box is required");
-      assert6(action.action_inputs.end_box, "end_box is required");
+      assert7(action.action_inputs.start_box, "start_box is required");
+      assert7(action.action_inputs.end_box, "end_box is required");
       const startPoint = getPoint(action.action_inputs.start_box, size);
       const endPoint = getPoint(action.action_inputs.end_box, size);
       transformActions.push({
@@ -2806,7 +3023,7 @@ async function vlmPlanning(options) {
         param: {}
       });
     } else if (action.action_type === "androidLongPress") {
-      assert6(
+      assert7(
         action.action_inputs.start_coords,
         "start_coords is required for androidLongPress"
       );
@@ -2922,4 +3139,4 @@ export {
   resizeImageForUiTars
 };
 
-//# sourceMappingURL=chunk-I5LBWOQA.js.map
+//# sourceMappingURL=chunk-G2JTYWI6.js.map