@midscene/shared 0.30.10 → 1.0.0

package/dist/es/build/rspack-config.mjs

@@ -0,0 +1,4 @@
+const commonIgnoreWarnings = [
+    /Critical dependency: the request of a dependency is an expression/
+];
+export { commonIgnoreWarnings };

package/dist/es/constants/example-code.mjs

@@ -113,12 +113,12 @@ tasks:
       # Auto Planning (.ai)
       # ----------------
 
-      # Perform an interaction. \`ai\` is a shorthand for \`aiAction\`.
+      # Perform an interaction. \`ai\` is a shorthand for \`aiAct\`.
       - ai: <prompt>
         cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
 
       # This usage is the same as \`ai\`.
-      - aiAction: <prompt>
+      - aiAct: <prompt>
         cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
 
       # Instant Action (.aiTap, .aiDoubleClick, .aiHover, .aiInput, .aiKeyboardPress, .aiScroll)
@@ -166,8 +166,8 @@ tasks:
         xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Midscene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.
         cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
 
-      # Log the current screenshot with a description in the report file.
-      - logScreenshot: <title> # Optional, the title of the screenshot. If not provided, the title will be 'untitled'.
+      # Record the current screenshot with a description in the report file.
+      - recordToReport: <title> # Optional, the title of the screenshot. If not provided, the title will be 'untitled'.
         content: <content> # Optional, the description of the screenshot.
 
       # Data Extraction

package/dist/es/env/constants.mjs

@@ -1,79 +1,33 @@
-import { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MIDSCENE_ANTHROPIC_API_KEY, MIDSCENE_AZURE_OPENAI_API_VERSION, MIDSCENE_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_AZURE_OPENAI_ENDPOINT, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_KEY, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_GROUNDING_ANTHROPIC_API_KEY, MIDSCENE_GROUNDING_AZURE_OPENAI_API_VERSION, MIDSCENE_GROUNDING_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_GROUNDING_AZURE_OPENAI_ENDPOINT, MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_GROUNDING_AZURE_OPENAI_KEY, MIDSCENE_GROUNDING_AZURE_OPENAI_SCOPE, MIDSCENE_GROUNDING_MODEL_NAME, MIDSCENE_GROUNDING_OPENAI_API_KEY, MIDSCENE_GROUNDING_OPENAI_BASE_URL, MIDSCENE_GROUNDING_OPENAI_HTTP_PROXY, MIDSCENE_GROUNDING_OPENAI_INIT_CONFIG_JSON, MIDSCENE_GROUNDING_OPENAI_SOCKS_PROXY, MIDSCENE_GROUNDING_OPENAI_USE_AZURE, MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK, MIDSCENE_GROUNDING_USE_AZURE_OPENAI, MIDSCENE_GROUNDING_VL_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_API_KEY, MIDSCENE_OPENAI_BASE_URL, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_OPENAI_USE_AZURE, MIDSCENE_PLANNING_ANTHROPIC_API_KEY, MIDSCENE_PLANNING_AZURE_OPENAI_API_VERSION, MIDSCENE_PLANNING_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_PLANNING_AZURE_OPENAI_ENDPOINT, MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_PLANNING_AZURE_OPENAI_KEY, MIDSCENE_PLANNING_AZURE_OPENAI_SCOPE, MIDSCENE_PLANNING_MODEL_NAME, MIDSCENE_PLANNING_OPENAI_API_KEY, MIDSCENE_PLANNING_OPENAI_BASE_URL, MIDSCENE_PLANNING_OPENAI_HTTP_PROXY, MIDSCENE_PLANNING_OPENAI_INIT_CONFIG_JSON, MIDSCENE_PLANNING_OPENAI_SOCKS_PROXY, MIDSCENE_PLANNING_OPENAI_USE_AZURE, MIDSCENE_PLANNING_USE_ANTHROPIC_SDK, MIDSCENE_PLANNING_USE_AZURE_OPENAI, MIDSCENE_PLANNING_VL_MODE, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_VL_MODE, MIDSCENE_VQA_ANTHROPIC_API_KEY, MIDSCENE_VQA_AZURE_OPENAI_API_VERSION, MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT, MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_VQA_AZURE_OPENAI_KEY, MIDSCENE_VQA_AZURE_OPENAI_SCOPE, MIDSCENE_VQA_MODEL_NAME, MIDSCENE_VQA_OPENAI_API_KEY, MIDSCENE_VQA_OPENAI_BASE_URL, MIDSCENE_VQA_OPENAI_HTTP_PROXY, MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON, MIDSCENE_VQA_OPENAI_SOCKS_PROXY, MIDSCENE_VQA_OPENAI_USE_AZURE, MIDSCENE_VQA_USE_ANTHROPIC_SDK, MIDSCENE_VQA_USE_AZURE_OPENAI, MIDSCENE_VQA_VL_MODE, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_USE_AZURE } from "./types.mjs";
-const VQA_MODEL_CONFIG_KEYS = {
-    modelName: MIDSCENE_VQA_MODEL_NAME,
-    socksProxy: MIDSCENE_VQA_OPENAI_SOCKS_PROXY,
-    httpProxy: MIDSCENE_VQA_OPENAI_HTTP_PROXY,
-    openaiBaseURL: MIDSCENE_VQA_OPENAI_BASE_URL,
-    openaiApiKey: MIDSCENE_VQA_OPENAI_API_KEY,
-    openaiExtraConfig: MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON,
-    openaiUseAzureDeprecated: MIDSCENE_VQA_OPENAI_USE_AZURE,
-    useAzureOpenai: MIDSCENE_VQA_USE_AZURE_OPENAI,
-    azureOpenaiScope: MIDSCENE_VQA_AZURE_OPENAI_SCOPE,
-    azureOpenaiKey: MIDSCENE_VQA_AZURE_OPENAI_KEY,
-    azureOpenaiEndpoint: MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT,
-    azureOpenaiApiVersion: MIDSCENE_VQA_AZURE_OPENAI_API_VERSION,
-    azureOpenaiDeployment: MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT,
-    azureExtraConfig: MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON,
-    useAnthropicSdk: MIDSCENE_VQA_USE_ANTHROPIC_SDK,
-    anthropicApiKey: MIDSCENE_VQA_ANTHROPIC_API_KEY,
-    vlMode: MIDSCENE_VQA_VL_MODE
-};
-const GROUNDING_MODEL_CONFIG_KEYS = {
-    modelName: MIDSCENE_GROUNDING_MODEL_NAME,
-    socksProxy: MIDSCENE_GROUNDING_OPENAI_SOCKS_PROXY,
-    httpProxy: MIDSCENE_GROUNDING_OPENAI_HTTP_PROXY,
-    openaiBaseURL: MIDSCENE_GROUNDING_OPENAI_BASE_URL,
-    openaiApiKey: MIDSCENE_GROUNDING_OPENAI_API_KEY,
-    openaiExtraConfig: MIDSCENE_GROUNDING_OPENAI_INIT_CONFIG_JSON,
-    openaiUseAzureDeprecated: MIDSCENE_GROUNDING_OPENAI_USE_AZURE,
-    useAzureOpenai: MIDSCENE_GROUNDING_USE_AZURE_OPENAI,
-    azureOpenaiScope: MIDSCENE_GROUNDING_AZURE_OPENAI_SCOPE,
-    azureOpenaiKey: MIDSCENE_GROUNDING_AZURE_OPENAI_KEY,
-    azureOpenaiEndpoint: MIDSCENE_GROUNDING_AZURE_OPENAI_ENDPOINT,
-    azureOpenaiApiVersion: MIDSCENE_GROUNDING_AZURE_OPENAI_API_VERSION,
-    azureOpenaiDeployment: MIDSCENE_GROUNDING_AZURE_OPENAI_DEPLOYMENT,
-    azureExtraConfig: MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON,
-    useAnthropicSdk: MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK,
-    anthropicApiKey: MIDSCENE_GROUNDING_ANTHROPIC_API_KEY,
-    vlMode: MIDSCENE_GROUNDING_VL_MODE
+import { MIDSCENE_INSIGHT_MODEL_API_KEY, MIDSCENE_INSIGHT_MODEL_BASE_URL, MIDSCENE_INSIGHT_MODEL_HTTP_PROXY, MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON, MIDSCENE_INSIGHT_MODEL_NAME, MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY, MIDSCENE_INSIGHT_MODEL_TIMEOUT, MIDSCENE_MODEL_API_KEY, MIDSCENE_MODEL_BASE_URL, MIDSCENE_MODEL_FAMILY, MIDSCENE_MODEL_HTTP_PROXY, MIDSCENE_MODEL_INIT_CONFIG_JSON, MIDSCENE_MODEL_NAME, MIDSCENE_MODEL_SOCKS_PROXY, MIDSCENE_MODEL_TIMEOUT, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_PLANNING_MODEL_API_KEY, MIDSCENE_PLANNING_MODEL_BASE_URL, MIDSCENE_PLANNING_MODEL_HTTP_PROXY, MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON, MIDSCENE_PLANNING_MODEL_NAME, MIDSCENE_PLANNING_MODEL_SOCKS_PROXY, MIDSCENE_PLANNING_MODEL_TIMEOUT, OPENAI_API_KEY, OPENAI_BASE_URL } from "./types.mjs";
+const INSIGHT_MODEL_CONFIG_KEYS = {
+    modelName: MIDSCENE_INSIGHT_MODEL_NAME,
+    socksProxy: MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY,
+    httpProxy: MIDSCENE_INSIGHT_MODEL_HTTP_PROXY,
+    openaiBaseURL: MIDSCENE_INSIGHT_MODEL_BASE_URL,
+    openaiApiKey: MIDSCENE_INSIGHT_MODEL_API_KEY,
+    openaiExtraConfig: MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON,
+    modelFamily: 'THERE_IS_NO_MODEL_FAMILY_FOR_INSIGHT',
+    timeout: MIDSCENE_INSIGHT_MODEL_TIMEOUT
 };
 const PLANNING_MODEL_CONFIG_KEYS = {
     modelName: MIDSCENE_PLANNING_MODEL_NAME,
-    socksProxy: MIDSCENE_PLANNING_OPENAI_SOCKS_PROXY,
-    httpProxy: MIDSCENE_PLANNING_OPENAI_HTTP_PROXY,
-    openaiBaseURL: MIDSCENE_PLANNING_OPENAI_BASE_URL,
-    openaiApiKey: MIDSCENE_PLANNING_OPENAI_API_KEY,
-    openaiExtraConfig: MIDSCENE_PLANNING_OPENAI_INIT_CONFIG_JSON,
-    openaiUseAzureDeprecated: MIDSCENE_PLANNING_OPENAI_USE_AZURE,
-    useAzureOpenai: MIDSCENE_PLANNING_USE_AZURE_OPENAI,
-    azureOpenaiScope: MIDSCENE_PLANNING_AZURE_OPENAI_SCOPE,
-    azureOpenaiKey: MIDSCENE_PLANNING_AZURE_OPENAI_KEY,
-    azureOpenaiEndpoint: MIDSCENE_PLANNING_AZURE_OPENAI_ENDPOINT,
-    azureOpenaiApiVersion: MIDSCENE_PLANNING_AZURE_OPENAI_API_VERSION,
-    azureOpenaiDeployment: MIDSCENE_PLANNING_AZURE_OPENAI_DEPLOYMENT,
-    azureExtraConfig: MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON,
-    useAnthropicSdk: MIDSCENE_PLANNING_USE_ANTHROPIC_SDK,
-    anthropicApiKey: MIDSCENE_PLANNING_ANTHROPIC_API_KEY,
-    vlMode: MIDSCENE_PLANNING_VL_MODE
+    socksProxy: MIDSCENE_PLANNING_MODEL_SOCKS_PROXY,
+    httpProxy: MIDSCENE_PLANNING_MODEL_HTTP_PROXY,
+    openaiBaseURL: MIDSCENE_PLANNING_MODEL_BASE_URL,
+    openaiApiKey: MIDSCENE_PLANNING_MODEL_API_KEY,
+    openaiExtraConfig: MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON,
+    modelFamily: 'THERE_IS_NO_MODEL_FAMILY_FOR_PLANNING',
+    timeout: MIDSCENE_PLANNING_MODEL_TIMEOUT
 };
 const DEFAULT_MODEL_CONFIG_KEYS = {
     modelName: MIDSCENE_MODEL_NAME,
-    socksProxy: MIDSCENE_OPENAI_SOCKS_PROXY,
-    httpProxy: MIDSCENE_OPENAI_HTTP_PROXY,
-    openaiBaseURL: MIDSCENE_OPENAI_BASE_URL,
-    openaiApiKey: MIDSCENE_OPENAI_API_KEY,
-    openaiExtraConfig: MIDSCENE_OPENAI_INIT_CONFIG_JSON,
-    openaiUseAzureDeprecated: MIDSCENE_OPENAI_USE_AZURE,
-    useAzureOpenai: MIDSCENE_USE_AZURE_OPENAI,
-    azureOpenaiScope: MIDSCENE_AZURE_OPENAI_SCOPE,
-    azureOpenaiKey: MIDSCENE_AZURE_OPENAI_KEY,
-    azureOpenaiEndpoint: MIDSCENE_AZURE_OPENAI_ENDPOINT,
-    azureOpenaiApiVersion: MIDSCENE_AZURE_OPENAI_API_VERSION,
-    azureOpenaiDeployment: MIDSCENE_AZURE_OPENAI_DEPLOYMENT,
-    azureExtraConfig: MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
-    useAnthropicSdk: MIDSCENE_USE_ANTHROPIC_SDK,
-    anthropicApiKey: MIDSCENE_ANTHROPIC_API_KEY,
-    vlMode: MIDSCENE_VL_MODE
+    socksProxy: MIDSCENE_MODEL_SOCKS_PROXY,
+    httpProxy: MIDSCENE_MODEL_HTTP_PROXY,
+    openaiBaseURL: MIDSCENE_MODEL_BASE_URL,
+    openaiApiKey: MIDSCENE_MODEL_API_KEY,
+    openaiExtraConfig: MIDSCENE_MODEL_INIT_CONFIG_JSON,
+    modelFamily: MIDSCENE_MODEL_FAMILY,
+    timeout: MIDSCENE_MODEL_TIMEOUT
 };
 const DEFAULT_MODEL_CONFIG_KEYS_LEGACY = {
     modelName: MIDSCENE_MODEL_NAME,
@@ -82,16 +36,7 @@ const DEFAULT_MODEL_CONFIG_KEYS_LEGACY = {
     openaiBaseURL: OPENAI_BASE_URL,
     openaiApiKey: OPENAI_API_KEY,
     openaiExtraConfig: MIDSCENE_OPENAI_INIT_CONFIG_JSON,
-    openaiUseAzureDeprecated: OPENAI_USE_AZURE,
-    useAzureOpenai: MIDSCENE_USE_AZURE_OPENAI,
-    azureOpenaiScope: MIDSCENE_AZURE_OPENAI_SCOPE,
-    azureOpenaiKey: AZURE_OPENAI_KEY,
-    azureOpenaiEndpoint: AZURE_OPENAI_ENDPOINT,
-    azureOpenaiApiVersion: AZURE_OPENAI_API_VERSION,
-    azureOpenaiDeployment: AZURE_OPENAI_DEPLOYMENT,
-    azureExtraConfig: MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
-    useAnthropicSdk: MIDSCENE_USE_ANTHROPIC_SDK,
-    anthropicApiKey: ANTHROPIC_API_KEY,
-    vlMode: 'DEFAULT_MODEL_CONFIG_KEYS has no vlMode key'
+    modelFamily: 'DEFAULT_MODEL_CONFIG_KEYS has no modelFamily key',
+    timeout: MIDSCENE_MODEL_TIMEOUT
 };
-export { DEFAULT_MODEL_CONFIG_KEYS, DEFAULT_MODEL_CONFIG_KEYS_LEGACY, GROUNDING_MODEL_CONFIG_KEYS, PLANNING_MODEL_CONFIG_KEYS, VQA_MODEL_CONFIG_KEYS };
+export { DEFAULT_MODEL_CONFIG_KEYS, DEFAULT_MODEL_CONFIG_KEYS_LEGACY, INSIGHT_MODEL_CONFIG_KEYS, PLANNING_MODEL_CONFIG_KEYS };

package/dist/es/env/global-config-manager.mjs

@@ -30,8 +30,8 @@ class GlobalConfigManager {
     }
     getEnvConfigValue(key) {
         const allConfig = this.getAllEnvConfig();
+        if (key === MATCH_BY_POSITION) throw new Error('MATCH_BY_POSITION is discarded, use MIDSCENE_MODEL_FAMILY instead');
         if (!STRING_ENV_KEYS.includes(key)) throw new Error(`getEnvConfigValue with key ${key} is not supported.`);
-        if (key === MATCH_BY_POSITION) throw new Error('MATCH_BY_POSITION is deprecated, use MIDSCENE_USE_VL_MODEL instead');
         const value = allConfig[key];
         this.keysHaveBeenRead[key] = true;
         if ('string' == typeof value) return value.trim();
@@ -58,7 +58,6 @@ class GlobalConfigManager {
         this.globalModelConfigManager = globalModelConfigManager;
     }
     overrideAIConfig(newConfig, extendMode = false) {
-        var _this_override;
         for(const key in newConfig){
             if (![
                 ...GLOBAL_ENV_KEYS,
@@ -69,7 +68,7 @@ class GlobalConfigManager {
             if (this.keysHaveBeenRead[key]) console.warn(`Warning: try to override AI config with key ${key} ,but it has been read.`);
         }
         const savedNewConfig = extendMode ? {
-            ...null == (_this_override = this.override) ? void 0 : _this_override.newConfig,
+            ...this.override?.newConfig,
             ...newConfig
         } : newConfig;
         this.override = {

package/dist/es/env/helper.mjs

@@ -1,4 +1,3 @@
-import { assert } from "../utils.mjs";
 const maskKey = (key, maskChar = '*')=>{
     if ('string' != typeof key || 0 === key.length) return key;
     const prefixLen = 3;
@@ -12,21 +11,21 @@ const maskKey = (key, maskChar = '*')=>{
     return `${prefix}${mask}${suffix}`;
 };
 const maskConfig = (config)=>Object.fromEntries(Object.entries(config).map(([key, value])=>{
-        if ([
-            'openaiApiKey',
-            'azureOpenaiKey',
-            'anthropicApiKey'
-        ].includes(key)) return [
+        if (!value) return [
             key,
-            maskKey(value)
+            value
         ];
-        if ([
-            'openaiExtraConfig',
-            'azureExtraConfig'
-        ].includes(key)) return [
+        if ('string' == typeof value && /key/i.test(key)) return [
             key,
-            maskKey(JSON.stringify(value))
+            maskKey(value)
         ];
+        if ('object' == typeof value) {
+            const valueStr = JSON.stringify(value);
+            if (/key/i.test(valueStr)) return [
+                key,
+                maskKey(valueStr)
+            ];
+        }
         return [
             key,
             value
@@ -41,8 +40,4 @@ const parseJson = (key, value)=>{
         });
     }
 };
-const createAssert = (modelNameKey, provider, modelName)=>(value, key, modelVendorFlag)=>{
-        if (modelName) modelVendorFlag ? assert(value, `The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName} and ${modelVendorFlag} has also been specified in ${provider}, but got: ${value}. Please check your config.`) : assert(value, `The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName} in ${provider}, but got: ${value}. Please check your config.`);
-        else assert(value, `The ${key} must be a non-empty string, but got: ${value}. Please check your config.`);
-    };
-export { createAssert, maskConfig, parseJson };
+export { maskConfig, parseJson };

package/dist/es/env/init-debug.mjs

@@ -1,16 +1,16 @@
 import { enableDebug } from "../logger.mjs";
 import { getBasicEnvValue } from "./basic.mjs";
-import { MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE } from "./types.mjs";
+import { MIDSCENE_DEBUG_MODEL_PROFILE, MIDSCENE_DEBUG_MODEL_RESPONSE } from "./types.mjs";
 const initDebugConfig = ()=>{
-    const shouldPrintTiming = getBasicEnvValue(MIDSCENE_DEBUG_AI_PROFILE);
+    const shouldPrintTiming = getBasicEnvValue(MIDSCENE_DEBUG_MODEL_PROFILE);
     let debugConfig = '';
     if (shouldPrintTiming) {
-        console.warn('MIDSCENE_DEBUG_AI_PROFILE is deprecated, use DEBUG=midscene:ai:profile instead');
+        console.warn('MIDSCENE_DEBUG_MODEL_PROFILE is deprecated, use DEBUG=midscene:ai:profile instead');
         debugConfig = 'ai:profile';
     }
-    const shouldPrintAIResponse = getBasicEnvValue(MIDSCENE_DEBUG_AI_RESPONSE);
-    if (shouldPrintAIResponse) {
-        console.warn('MIDSCENE_DEBUG_AI_RESPONSE is deprecated, use DEBUG=midscene:ai:response instead');
+    const shouldPrintModelResponse = getBasicEnvValue(MIDSCENE_DEBUG_MODEL_RESPONSE);
+    if (shouldPrintModelResponse) {
+        console.warn('MIDSCENE_DEBUG_MODEL_RESPONSE is deprecated, use DEBUG=midscene:ai:response instead');
         debugConfig = debugConfig ? 'ai:*' : 'ai:call';
     }
     if (debugConfig) enableDebug(debugConfig);

package/dist/es/env/model-config-manager.mjs

@@ -1,4 +1,4 @@
-import { decideModelConfigFromEnv, decideModelConfigFromIntentConfig } from "./decide-model-config.mjs";
+import { decideModelConfigFromIntentConfig } from "./parse-model-config.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -9,91 +9,71 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const ALL_INTENTS = [
-    'VQA',
-    'default',
-    'grounding',
-    'planning'
-];
 class ModelConfigManager {
-    calcIntentConfigMap(modelConfigFn) {
-        const intentConfigMap = {
-            VQA: void 0,
-            default: void 0,
-            grounding: void 0,
-            planning: void 0
+    initialize() {
+        if (this.isInitialized) return;
+        let configMap;
+        if (this.modelConfig) {
+            this.isolatedMode = true;
+            configMap = this.normalizeModelConfig(this.modelConfig);
+        } else configMap = this.globalConfigManager?.getAllEnvConfig() || {};
+        const defaultConfig = decideModelConfigFromIntentConfig('default', configMap);
+        if (!defaultConfig) throw new Error('default model config is not found, which should not happen');
+        const insightConfig = decideModelConfigFromIntentConfig('insight', configMap);
+        const planningConfig = decideModelConfigFromIntentConfig('planning', configMap);
+        this.modelConfigMap = {
+            default: {
+                ...defaultConfig,
+                createOpenAIClient: this.createOpenAIClientFn
+            },
+            insight: {
+                ...insightConfig || defaultConfig,
+                createOpenAIClient: this.createOpenAIClientFn
+            },
+            planning: {
+                ...planningConfig || defaultConfig,
+                createOpenAIClient: this.createOpenAIClientFn
+            }
         };
-        for (const i of ALL_INTENTS){
-            const result = modelConfigFn({
-                intent: i
-            });
-            if (!result) throw new Error(`The agent has an option named modelConfig is a function, but it return ${result} when call with intent ${i}, which should be a object.`);
-            intentConfigMap[i] = result;
-        }
-        return intentConfigMap;
+        this.isInitialized = true;
     }
-    calcModelConfigMapBaseOnIntent(intentConfigMap) {
-        const modelConfigMap = {
-            VQA: void 0,
-            default: void 0,
-            grounding: void 0,
-            planning: void 0
-        };
-        for (const i of ALL_INTENTS){
-            const result = decideModelConfigFromIntentConfig(i, intentConfigMap[i]);
-            modelConfigMap[i] = result;
-        }
-        return modelConfigMap;
-    }
-    calcModelConfigMapBaseOnEnv(allEnvConfig) {
-        const modelConfigMap = {
-            VQA: void 0,
-            default: void 0,
-            grounding: void 0,
-            planning: void 0
-        };
-        for (const i of ALL_INTENTS){
-            const result = decideModelConfigFromEnv(i, allEnvConfig);
-            modelConfigMap[i] = result;
-        }
-        return modelConfigMap;
+    normalizeModelConfig(config) {
+        return Object.entries(config).reduce((acc, [key, value])=>{
+            if (null == value) return acc;
+            acc[key] = String(value);
+            return acc;
+        }, Object.create(null));
     }
     clearModelConfigMap() {
         if (this.isolatedMode) throw new Error('ModelConfigManager work in isolated mode, so clearModelConfigMap should not be called');
-        this.modelConfigMap = void 0;
+        this.isInitialized = false;
     }
     getModelConfig(intent) {
-        if (this.isolatedMode) {
-            if (!this.modelConfigMap) throw new Error('modelConfigMap is not initialized in isolated mode, which should not happen');
-            return this.modelConfigMap[intent];
-        }
-        if (!this.modelConfigMap) {
-            if (!this.globalConfigManager) throw new Error('globalConfigManager is not registered, which should not happen');
-            this.modelConfigMap = this.calcModelConfigMapBaseOnEnv(this.globalConfigManager.getAllEnvConfig());
-        }
+        if (!this.isInitialized) this.initialize();
+        if (!this.modelConfigMap) throw new Error('modelConfigMap is not initialized, which should not happen');
         return this.modelConfigMap[intent];
     }
     getUploadTestServerUrl() {
         const { openaiExtraConfig } = this.getModelConfig('default');
-        const serverUrl = null == openaiExtraConfig ? void 0 : openaiExtraConfig.REPORT_SERVER_URL;
+        const serverUrl = openaiExtraConfig?.REPORT_SERVER_URL;
         return serverUrl;
     }
     registerGlobalConfigManager(globalConfigManager) {
         this.globalConfigManager = globalConfigManager;
     }
-    throwErrorIfNonVLModel(intent = 'grounding') {
-        const modelConfig = this.getModelConfig(intent);
-        if (!modelConfig.vlMode) throw new Error('No visual language model (VL model) detected for the current scenario. Element localization may be inaccurate. Please verify your model configuration. Learn more: https://midscenejs.com/choose-a-model');
+    throwErrorIfNonVLModel() {
+        const modelConfig = this.getModelConfig('default');
+        if (!modelConfig.vlMode) throw new Error('MIDSCENE_MODEL_FAMILY is not set to a visual language model (VL model), the element localization can not be achieved. Check your model configuration. See https://midscenejs.com/model-strategy.html');
     }
-    constructor(modelConfigFn){
+    constructor(modelConfig, createOpenAIClientFn){
         _define_property(this, "modelConfigMap", void 0);
+        _define_property(this, "isInitialized", false);
         _define_property(this, "isolatedMode", false);
         _define_property(this, "globalConfigManager", void 0);
-        if (modelConfigFn) {
-            this.isolatedMode = true;
-            const intentConfigMap = this.calcIntentConfigMap(modelConfigFn);
-            this.modelConfigMap = this.calcModelConfigMapBaseOnIntent(intentConfigMap);
-        }
+        _define_property(this, "modelConfig", void 0);
+        _define_property(this, "createOpenAIClientFn", void 0);
+        this.modelConfig = modelConfig;
+        this.createOpenAIClientFn = createOpenAIClientFn;
     }
 }
 export { ModelConfigManager };

package/dist/es/env/parse-model-config.mjs

@@ -0,0 +1,112 @@
+import { DEFAULT_MODEL_CONFIG_KEYS, INSIGHT_MODEL_CONFIG_KEYS, PLANNING_MODEL_CONFIG_KEYS } from "./constants.mjs";
+import { MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN3_VL, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MODEL_FAMILY_VALUES, OPENAI_API_KEY, OPENAI_BASE_URL, UITarsModelVersion } from "./types.mjs";
+import { getDebug } from "../logger.mjs";
+import { assert } from "../utils.mjs";
+import { maskConfig, parseJson } from "./helper.mjs";
+import { initDebugConfig } from "./init-debug.mjs";
+const KEYS_MAP = {
+    insight: INSIGHT_MODEL_CONFIG_KEYS,
+    planning: PLANNING_MODEL_CONFIG_KEYS,
+    default: DEFAULT_MODEL_CONFIG_KEYS
+};
+const modelFamilyToVLConfig = (modelFamily)=>{
+    if (!modelFamily) return {
+        vlMode: void 0,
+        uiTarsVersion: void 0
+    };
+    if ('vlm-ui-tars' === modelFamily) return {
+        vlMode: 'vlm-ui-tars',
+        uiTarsVersion: UITarsModelVersion.V1_0
+    };
+    if ('vlm-ui-tars-doubao' === modelFamily || 'vlm-ui-tars-doubao-1.5' === modelFamily) return {
+        vlMode: 'vlm-ui-tars',
+        uiTarsVersion: UITarsModelVersion.DOUBAO_1_5_20B
+    };
+    if (!MODEL_FAMILY_VALUES.includes(modelFamily)) throw new Error(`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}`);
+    return {
+        vlMode: modelFamily,
+        uiTarsVersion: void 0
+    };
+};
+const legacyConfigToModelFamily = (provider)=>{
+    const isDoubao = provider[MIDSCENE_USE_DOUBAO_VISION];
+    const isQwen = provider[MIDSCENE_USE_QWEN_VL];
+    const isQwen3 = provider[MIDSCENE_USE_QWEN3_VL];
+    const isUiTars = provider[MIDSCENE_USE_VLM_UI_TARS];
+    const isGemini = provider[MIDSCENE_USE_GEMINI];
+    const enabledModes = [
+        isDoubao && MIDSCENE_USE_DOUBAO_VISION,
+        isQwen && MIDSCENE_USE_QWEN_VL,
+        isQwen3 && MIDSCENE_USE_QWEN3_VL,
+        isUiTars && MIDSCENE_USE_VLM_UI_TARS,
+        isGemini && MIDSCENE_USE_GEMINI
+    ].filter(Boolean);
+    if (enabledModes.length > 1) throw new Error(`Only one vision mode can be enabled at a time. Currently enabled modes: ${enabledModes.join(', ')}. Please disable all but one mode.`);
+    if (isQwen3) return 'qwen3-vl';
+    if (isQwen) return 'qwen2.5-vl';
+    if (isDoubao) return 'doubao-vision';
+    if (isGemini) return 'gemini';
+    if (isUiTars) if ('1' === isUiTars) return 'vlm-ui-tars';
+    else if ('DOUBAO' === isUiTars || 'DOUBAO-1.5' === isUiTars) return 'vlm-ui-tars-doubao-1.5';
+    else return 'vlm-ui-tars-doubao';
+};
+const parseOpenaiSdkConfig = ({ keys, provider, useLegacyLogic = false })=>{
+    initDebugConfig();
+    const debugLog = getDebug('ai:config');
+    debugLog('enter parseOpenaiSdkConfig with keys:', keys);
+    const legacyAPIKey = useLegacyLogic ? provider[OPENAI_API_KEY] : void 0;
+    const legacyBaseURL = useLegacyLogic ? provider[OPENAI_BASE_URL] : void 0;
+    const legacySocksProxy = useLegacyLogic ? provider[MIDSCENE_OPENAI_SOCKS_PROXY] : void 0;
+    const legacyHttpProxy = useLegacyLogic ? provider[MIDSCENE_OPENAI_HTTP_PROXY] : void 0;
+    const legacyOpenaiExtraConfig = useLegacyLogic ? provider[MIDSCENE_OPENAI_INIT_CONFIG_JSON] : void 0;
+    const legacyModelFamily = useLegacyLogic ? legacyConfigToModelFamily(provider) : void 0;
+    const modelFamilyRaw = provider[keys.modelFamily] || legacyModelFamily;
+    const openaiApiKey = provider[keys.openaiApiKey] || legacyAPIKey;
+    const openaiBaseURL = provider[keys.openaiBaseURL] || legacyBaseURL;
+    const socksProxy = provider[keys.socksProxy] || legacySocksProxy;
+    const httpProxy = provider[keys.httpProxy] || legacyHttpProxy;
+    const modelName = provider[keys.modelName];
+    const openaiExtraConfigStr = provider[keys.openaiExtraConfig];
+    const openaiExtraConfig = parseJson(keys.openaiExtraConfig, openaiExtraConfigStr || legacyOpenaiExtraConfig);
+    const { vlMode, uiTarsVersion } = modelFamilyToVLConfig(modelFamilyRaw);
+    const getModelDescription = (vlMode, uiTarsVersion)=>{
+        if (vlMode) if (uiTarsVersion) return `UI-TARS=${uiTarsVersion}`;
+        else return `${vlMode} mode`;
+        return '';
+    };
+    const modelDescription = getModelDescription(vlMode, uiTarsVersion);
+    return {
+        socksProxy,
+        httpProxy,
+        vlModeRaw: vlMode,
+        openaiBaseURL,
+        openaiApiKey,
+        openaiExtraConfig,
+        vlMode,
+        uiTarsModelVersion: uiTarsVersion,
+        modelName: modelName,
+        modelDescription,
+        intent: '-',
+        timeout: provider[keys.timeout] ? Number(provider[keys.timeout]) : void 0
+    };
+};
+const decideModelConfigFromIntentConfig = (intent, configMap)=>{
+    const debugLog = getDebug('ai:config');
+    debugLog('will decideModelConfig base on agent.modelConfig()', intent, maskConfig(configMap));
+    const keysForFn = KEYS_MAP[intent];
+    const modelName = configMap[keysForFn.modelName];
+    if (!modelName) return void debugLog('no modelName found for intent', intent);
+    const finalResult = parseOpenaiSdkConfig({
+        keys: keysForFn,
+        provider: configMap,
+        useLegacyLogic: 'default' === intent
+    });
+    finalResult.intent = intent;
+    debugLog('decideModelConfig result by agent.modelConfig() with intent', intent, maskConfig({
+        ...finalResult
+    }));
+    assert(finalResult.openaiBaseURL, `failed to get base URL of model (intent=${intent}). See https://midscenejs.com/model-strategy`);
+    if (!finalResult.modelName) console.warn(`modelName is not set for intent ${intent}, this may cause unexpected behavior. See https://midscenejs.com/model-strategy`);
+    return finalResult;
+};
+export { decideModelConfigFromIntentConfig, legacyConfigToModelFamily, modelFamilyToVLConfig, parseOpenaiSdkConfig };