@midscene/shared 0.30.10 → 1.0.0

package/src/env/global-config-manager.ts

@@ -1,4 +1,3 @@
-import { getDebug } from '../logger';
 import { initDebugConfig } from './init-debug';
 import type { ModelConfigManager } from './model-config-manager';
 import {
@@ -62,14 +61,16 @@ export class GlobalConfigManager {
   getEnvConfigValue(key: (typeof STRING_ENV_KEYS)[number]) {
     const allConfig = this.getAllEnvConfig();
 
-    if (!STRING_ENV_KEYS.includes(key)) {
-      throw new Error(`getEnvConfigValue with key ${key} is not supported.`);
-    }
     if (key === MATCH_BY_POSITION) {
       throw new Error(
-        'MATCH_BY_POSITION is deprecated, use MIDSCENE_USE_VL_MODEL instead',
+        'MATCH_BY_POSITION is discarded, use MIDSCENE_MODEL_FAMILY instead',
       );
     }
+
+    if (!STRING_ENV_KEYS.includes(key)) {
+      throw new Error(`getEnvConfigValue with key ${key} is not supported.`);
+    }
+
     const value = allConfig[key];
     this.keysHaveBeenRead[key] = true;
     if (typeof value === 'string') {
@@ -122,8 +123,7 @@ export class GlobalConfigManager {
   }
 
   /**
-   * for overrideAIConfig
-   * can only override keys in MODEL_ENV_KEYS
+   * @deprecated use the modelConfig param in Agent constructor instead
    */
   overrideAIConfig(
     newConfig: Partial<

package/src/env/helper.ts

@@ -22,13 +22,19 @@ const maskKey = (key: string, maskChar = '*') => {
   return `${prefix}${mask}${suffix}`;
 };
 
-export const maskConfig = (config: IModelConfig) => {
+export const maskConfig = (config: Record<string, unknown>) => {
   return Object.fromEntries(
     Object.entries(config).map(([key, value]) => {
-      if (['openaiApiKey', 'azureOpenaiKey', 'anthropicApiKey'].includes(key)) {
+      if (!value) return [key, value];
+
+      if (typeof value === 'string' && /key/i.test(key)) {
         return [key, maskKey(value)];
-      } else if (['openaiExtraConfig', 'azureExtraConfig'].includes(key)) {
-        return [key, maskKey(JSON.stringify(value))];
+      }
+      if (typeof value === 'object') {
+        const valueStr = JSON.stringify(value);
+        if (/key/i.test(valueStr)) {
+          return [key, maskKey(valueStr)];
+        }
       }
       return [key, value];
     }),
@@ -50,30 +56,3 @@ export const parseJson = (key: string, value: string | undefined) => {
   }
   return undefined;
 };
-
-export const createAssert =
-  (
-    modelNameKey: string,
-    provider: 'process.env' | 'modelConfig',
-    modelName?: string,
-  ) =>
-  (value: string | undefined, key: string, modelVendorFlag?: string) => {
-    if (modelName) {
-      if (modelVendorFlag) {
-        assert(
-          value,
-          `The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName} and ${modelVendorFlag} has also been specified in ${provider}, but got: ${value}. Please check your config.`,
-        );
-      } else {
-        assert(
-          value,
-          `The ${key} must be a non-empty string because of the ${modelNameKey} is declared as ${modelName} in ${provider}, but got: ${value}. Please check your config.`,
-        );
-      }
-    } else {
-      assert(
-        value,
-        `The ${key} must be a non-empty string, but got: ${value}. Please check your config.`,
-      );
-    }
-  };

package/src/env/init-debug.ts

@@ -1,21 +1,26 @@
 import { enableDebug } from '../logger';
 import { getBasicEnvValue } from './basic';
-import { MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE } from './types';
+import {
+  MIDSCENE_DEBUG_MODEL_PROFILE,
+  MIDSCENE_DEBUG_MODEL_RESPONSE,
+} from './types';
 
 export const initDebugConfig = () => {
-  const shouldPrintTiming = getBasicEnvValue(MIDSCENE_DEBUG_AI_PROFILE);
+  const shouldPrintTiming = getBasicEnvValue(MIDSCENE_DEBUG_MODEL_PROFILE);
   let debugConfig = '';
   if (shouldPrintTiming) {
     console.warn(
-      'MIDSCENE_DEBUG_AI_PROFILE is deprecated, use DEBUG=midscene:ai:profile instead',
+      'MIDSCENE_DEBUG_MODEL_PROFILE is deprecated, use DEBUG=midscene:ai:profile instead',
     );
     debugConfig = 'ai:profile';
   }
-  const shouldPrintAIResponse = getBasicEnvValue(MIDSCENE_DEBUG_AI_RESPONSE);
+  const shouldPrintModelResponse = getBasicEnvValue(
+    MIDSCENE_DEBUG_MODEL_RESPONSE,
+  );
 
-  if (shouldPrintAIResponse) {
+  if (shouldPrintModelResponse) {
     console.warn(
-      'MIDSCENE_DEBUG_AI_RESPONSE is deprecated, use DEBUG=midscene:ai:response instead',
+      'MIDSCENE_DEBUG_MODEL_RESPONSE is deprecated, use DEBUG=midscene:ai:response instead',
     );
     if (debugConfig) {
       debugConfig = 'ai:*';

package/src/env/model-config-manager.ts

@@ -1,87 +1,101 @@
-import {
-  decideModelConfigFromEnv,
-  decideModelConfigFromIntentConfig,
-} from './decide-model-config';
 import type { GlobalConfigManager } from './global-config-manager';
+import { decideModelConfigFromIntentConfig } from './parse-model-config';
 
-import type { IModelConfig, TIntent, TModelConfigFn } from './types';
-
-const ALL_INTENTS: TIntent[] = ['VQA', 'default', 'grounding', 'planning'];
-
-export type TIntentConfigMap = Record<
+import type {
+  CreateOpenAIClientFn,
+  IModelConfig,
   TIntent,
-  ReturnType<TModelConfigFn> | undefined
->;
+  TModelConfig,
+} from './types';
 
 export class ModelConfigManager {
   private modelConfigMap: Record<TIntent, IModelConfig> | undefined = undefined;
 
-  // once modelConfigFn is set, isolatedMode will be true
-  // modelConfigMap will only depend on modelConfigFn and not effect by process.env
+  private isInitialized = false;
+
+  // once modelConfig is set, isolatedMode will be true
+  // modelConfigMap will only depend on provided config and not be affected by process.env
   private isolatedMode = false;
 
   private globalConfigManager: GlobalConfigManager | undefined = undefined;
 
-  constructor(modelConfigFn?: TModelConfigFn) {
-    if (modelConfigFn) {
-      this.isolatedMode = true;
-      const intentConfigMap = this.calcIntentConfigMap(modelConfigFn);
-      this.modelConfigMap =
-        this.calcModelConfigMapBaseOnIntent(intentConfigMap);
-    }
+  private modelConfig?: TModelConfig;
+  private createOpenAIClientFn?: CreateOpenAIClientFn;
+
+  constructor(
+    modelConfig?: TModelConfig,
+    createOpenAIClientFn?: CreateOpenAIClientFn,
+  ) {
+    this.modelConfig = modelConfig;
+    this.createOpenAIClientFn = createOpenAIClientFn;
   }
 
-  private calcIntentConfigMap(modelConfigFn: TModelConfigFn): TIntentConfigMap {
-    const intentConfigMap: TIntentConfigMap = {
-      VQA: undefined,
-      default: undefined,
-      grounding: undefined,
-      planning: undefined,
-    };
+  private initialize() {
+    if (this.isInitialized) {
+      return;
+    }
 
-    for (const i of ALL_INTENTS) {
-      const result = modelConfigFn({ intent: i });
-      if (!result) {
-        throw new Error(
-          `The agent has an option named modelConfig is a function, but it return ${result} when call with intent ${i}, which should be a object.`,
-        );
-      }
-      intentConfigMap[i] = result;
+    let configMap: Record<string, string | undefined>;
+    if (this.modelConfig) {
+      this.isolatedMode = true;
+      configMap = this.normalizeModelConfig(this.modelConfig);
+    } else {
+      configMap = this.globalConfigManager?.getAllEnvConfig() || {};
     }
-    return intentConfigMap;
-  }
 
-  private calcModelConfigMapBaseOnIntent(intentConfigMap: TIntentConfigMap) {
-    const modelConfigMap: Record<TIntent, IModelConfig | undefined> = {
-      VQA: undefined,
-      default: undefined,
-      grounding: undefined,
-      planning: undefined,
-    };
-    for (const i of ALL_INTENTS) {
-      const result = decideModelConfigFromIntentConfig(
-        i,
-        intentConfigMap[i] as unknown as Record<string, string | undefined>,
+    const defaultConfig = decideModelConfigFromIntentConfig(
+      'default',
+      configMap,
+    );
+    if (!defaultConfig) {
+      throw new Error(
+        'default model config is not found, which should not happen',
       );
-      modelConfigMap[i] = result;
     }
-    return modelConfigMap as Record<TIntent, IModelConfig>;
-  }
 
-  private calcModelConfigMapBaseOnEnv(
-    allEnvConfig: Record<string, string | undefined>,
-  ) {
-    const modelConfigMap: Record<TIntent, IModelConfig | undefined> = {
-      VQA: undefined,
-      default: undefined,
-      grounding: undefined,
-      planning: undefined,
+    const insightConfig = decideModelConfigFromIntentConfig(
+      'insight',
+      configMap,
+    );
+
+    const planningConfig = decideModelConfigFromIntentConfig(
+      'planning',
+      configMap,
+    );
+
+    // Each intent uses its own timeout from parsed config (MIDSCENE_MODEL_TIMEOUT,
+    // MIDSCENE_INSIGHT_MODEL_TIMEOUT, MIDSCENE_PLANNING_MODEL_TIMEOUT).
+    this.modelConfigMap = {
+      default: {
+        ...defaultConfig,
+        createOpenAIClient: this.createOpenAIClientFn,
+      },
+      insight: {
+        ...(insightConfig || defaultConfig),
+        createOpenAIClient: this.createOpenAIClientFn,
+      },
+      planning: {
+        ...(planningConfig || defaultConfig),
+        createOpenAIClient: this.createOpenAIClientFn,
+      },
     };
-    for (const i of ALL_INTENTS) {
-      const result = decideModelConfigFromEnv(i, allEnvConfig);
-      modelConfigMap[i] = result;
-    }
-    return modelConfigMap as Record<TIntent, IModelConfig>;
+
+    this.isInitialized = true;
+  }
+
+  private normalizeModelConfig(
+    config: TModelConfig,
+  ): Record<string, string | undefined> {
+    return Object.entries(config).reduce<Record<string, string | undefined>>(
+      (acc, [key, value]) => {
+        if (value === undefined || value === null) {
+          return acc;
+        }
+        acc[key] = String(value);
+        return acc;
+      },
+      Object.create(null),
+    );
   }
 
   /**
@@ -93,7 +107,7 @@ export class ModelConfigManager {
         'ModelConfigManager work in isolated mode, so clearModelConfigMap should not be called',
       );
     }
-    this.modelConfigMap = undefined;
+    this.isInitialized = false;
   }
 
   /**
@@ -101,26 +115,16 @@ export class ModelConfigManager {
    * if isolatedMode is false, modelConfigMap can be changed by process.env so we need to recalculate it when it's undefined
    */
   getModelConfig(intent: TIntent): IModelConfig {
-    if (this.isolatedMode) {
-      if (!this.modelConfigMap) {
-        throw new Error(
-          'modelConfigMap is not initialized in isolated mode, which should not happen',
-        );
-      }
-      return this.modelConfigMap[intent];
-    } else {
-      if (!this.modelConfigMap) {
-        if (!this.globalConfigManager) {
-          throw new Error(
-            'globalConfigManager is not registered, which should not happen',
-          );
-        }
-        this.modelConfigMap = this.calcModelConfigMapBaseOnEnv(
-          this.globalConfigManager.getAllEnvConfig(),
-        );
-      }
-      return this.modelConfigMap[intent];
+    // check if initialized
+    if (!this.isInitialized) {
+      this.initialize();
+    }
+    if (!this.modelConfigMap) {
+      throw new Error(
+        'modelConfigMap is not initialized, which should not happen',
+      );
     }
+    return this.modelConfigMap[intent];
   }
 
   getUploadTestServerUrl(): string | undefined {
@@ -133,12 +137,12 @@ export class ModelConfigManager {
     this.globalConfigManager = globalConfigManager;
   }
 
-  throwErrorIfNonVLModel(intent: TIntent = 'grounding') {
-    const modelConfig = this.getModelConfig(intent);
+  throwErrorIfNonVLModel() {
+    const modelConfig = this.getModelConfig('default');
 
     if (!modelConfig.vlMode) {
       throw new Error(
-        'No visual language model (VL model) detected for the current scenario. Element localization may be inaccurate. Please verify your model configuration. Learn more: https://midscenejs.com/choose-a-model',
+        'MIDSCENE_MODEL_FAMILY is not set to a visual language model (VL model), the element localization can not be achieved. Check your model configuration. See https://midscenejs.com/model-strategy.html',
       );
     }
   }

package/src/env/parse-model-config.ts

@@ -0,0 +1,265 @@
+import {
+  DEFAULT_MODEL_CONFIG_KEYS,
+  type DEFAULT_MODEL_CONFIG_KEYS_LEGACY,
+  INSIGHT_MODEL_CONFIG_KEYS,
+  PLANNING_MODEL_CONFIG_KEYS,
+} from './constants';
+import {
+  type IModelConfig,
+  MIDSCENE_MODEL_FAMILY,
+  MIDSCENE_OPENAI_HTTP_PROXY,
+  MIDSCENE_OPENAI_INIT_CONFIG_JSON,
+  MIDSCENE_OPENAI_SOCKS_PROXY,
+  MIDSCENE_USE_DOUBAO_VISION,
+  MIDSCENE_USE_GEMINI,
+  MIDSCENE_USE_QWEN3_VL,
+  MIDSCENE_USE_QWEN_VL,
+  MIDSCENE_USE_VLM_UI_TARS,
+  MODEL_FAMILY_VALUES,
+  OPENAI_API_KEY,
+  OPENAI_BASE_URL,
+  type TIntent,
+  type TModelFamily,
+  type TVlModeTypes,
+  UITarsModelVersion,
+} from './types';
+
+import { getDebug } from '../logger';
+import { assert } from '../utils';
+import { maskConfig, parseJson } from './helper';
+import { initDebugConfig } from './init-debug';
+
+type TModelConfigKeys =
+  | typeof INSIGHT_MODEL_CONFIG_KEYS
+  | typeof PLANNING_MODEL_CONFIG_KEYS
+  | typeof DEFAULT_MODEL_CONFIG_KEYS
+  | typeof DEFAULT_MODEL_CONFIG_KEYS_LEGACY;
+
+const KEYS_MAP: Record<TIntent, TModelConfigKeys> = {
+  insight: INSIGHT_MODEL_CONFIG_KEYS,
+  planning: PLANNING_MODEL_CONFIG_KEYS,
+  default: DEFAULT_MODEL_CONFIG_KEYS,
+} as const;
+
+/**
+ * Convert model family to VL configuration
+ * @param modelFamily - The model family value
+ * @returns Object containing vlMode and uiTarsVersion
+ */
+export const modelFamilyToVLConfig = (
+  modelFamily?: TModelFamily,
+): {
+  vlMode?: TVlModeTypes;
+  uiTarsVersion?: UITarsModelVersion;
+} => {
+  if (!modelFamily) {
+    return { vlMode: undefined, uiTarsVersion: undefined };
+  }
+
+  // UI-TARS variants with version handling
+  if (modelFamily === 'vlm-ui-tars') {
+    return { vlMode: 'vlm-ui-tars', uiTarsVersion: UITarsModelVersion.V1_0 };
+  }
+
+  if (
+    modelFamily === 'vlm-ui-tars-doubao' ||
+    modelFamily === 'vlm-ui-tars-doubao-1.5'
+  ) {
+    return {
+      vlMode: 'vlm-ui-tars',
+      uiTarsVersion: UITarsModelVersion.DOUBAO_1_5_20B,
+    };
+  }
+
+  // Check if the modelFamily is valid
+  if (!MODEL_FAMILY_VALUES.includes(modelFamily as any)) {
+    throw new Error(`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}`);
+  }
+
+  // For other model families, they directly map to vlMode
+  return { vlMode: modelFamily as TVlModeTypes, uiTarsVersion: undefined };
+};
+
+/**
+ * Convert legacy environment variables to model family
+ * @param provider - Environment variable provider (e.g., process.env)
+ * @returns The corresponding model family value, or undefined if no legacy config is found
+ */
+export const legacyConfigToModelFamily = (
+  provider: Record<string, string | undefined>,
+): TModelFamily | undefined => {
+  // Step 1: Parse legacy environment variables to get vlMode and uiTarsVersion
+  const isDoubao = provider[MIDSCENE_USE_DOUBAO_VISION];
+  const isQwen = provider[MIDSCENE_USE_QWEN_VL];
+  const isQwen3 = provider[MIDSCENE_USE_QWEN3_VL];
+  const isUiTars = provider[MIDSCENE_USE_VLM_UI_TARS];
+  const isGemini = provider[MIDSCENE_USE_GEMINI];
+
+  const enabledModes = [
+    isDoubao && MIDSCENE_USE_DOUBAO_VISION,
+    isQwen && MIDSCENE_USE_QWEN_VL,
+    isQwen3 && MIDSCENE_USE_QWEN3_VL,
+    isUiTars && MIDSCENE_USE_VLM_UI_TARS,
+    isGemini && MIDSCENE_USE_GEMINI,
+  ].filter(Boolean);
+
+  if (enabledModes.length > 1) {
+    throw new Error(
+      `Only one vision mode can be enabled at a time. Currently enabled modes: ${enabledModes.join(', ')}. Please disable all but one mode.`,
+    );
+  }
+
+  // Step 2: Map to model family based on detected mode
+  // Simple modes that directly map to model family
+  if (isQwen3) return 'qwen3-vl';
+  if (isQwen) return 'qwen2.5-vl';
+  if (isDoubao) return 'doubao-vision';
+  if (isGemini) return 'gemini';
+
+  // UI-TARS with version detection
+  if (isUiTars) {
+    if (isUiTars === '1') {
+      return 'vlm-ui-tars';
+    } else if (isUiTars === 'DOUBAO' || isUiTars === 'DOUBAO-1.5') {
+      return 'vlm-ui-tars-doubao-1.5';
+    } else {
+      // Handle other UI-TARS versions
+      return 'vlm-ui-tars-doubao';
+    }
+  }
+
+  return undefined;
+};
+
+/**
+ * Parse OpenAI SDK config
+ */
+export const parseOpenaiSdkConfig = ({
+  keys,
+  provider,
+  useLegacyLogic = false,
+}: {
+  keys: TModelConfigKeys;
+  provider: Record<string, string | undefined>;
+  useLegacyLogic?: boolean;
+}): IModelConfig => {
+  initDebugConfig();
+  const debugLog = getDebug('ai:config');
+
+  debugLog('enter parseOpenaiSdkConfig with keys:', keys);
+
+  const legacyAPIKey = useLegacyLogic ? provider[OPENAI_API_KEY] : undefined;
+  const legacyBaseURL = useLegacyLogic ? provider[OPENAI_BASE_URL] : undefined;
+  const legacySocksProxy = useLegacyLogic
+    ? provider[MIDSCENE_OPENAI_SOCKS_PROXY]
+    : undefined;
+  const legacyHttpProxy = useLegacyLogic
+    ? provider[MIDSCENE_OPENAI_HTTP_PROXY]
+    : undefined;
+  const legacyOpenaiExtraConfig = useLegacyLogic
+    ? provider[MIDSCENE_OPENAI_INIT_CONFIG_JSON]
+    : undefined;
+  const legacyModelFamily = useLegacyLogic
+    ? legacyConfigToModelFamily(provider)
+    : undefined;
+
+  const modelFamilyRaw = provider[keys.modelFamily] || legacyModelFamily;
+  const openaiApiKey: string | undefined =
+    provider[keys.openaiApiKey] || legacyAPIKey;
+  const openaiBaseURL: string | undefined =
+    provider[keys.openaiBaseURL] || legacyBaseURL;
+  const socksProxy: string | undefined =
+    provider[keys.socksProxy] || legacySocksProxy;
+  const httpProxy: string | undefined =
+    provider[keys.httpProxy] || legacyHttpProxy;
+  const modelName: string | undefined = provider[keys.modelName];
+  const openaiExtraConfigStr: string | undefined =
+    provider[keys.openaiExtraConfig];
+  const openaiExtraConfig = parseJson(
+    keys.openaiExtraConfig,
+    openaiExtraConfigStr || legacyOpenaiExtraConfig,
+  );
+
+  const { vlMode, uiTarsVersion } = modelFamilyToVLConfig(
+    modelFamilyRaw as unknown as TModelFamily,
+  );
+
+  const getModelDescription = (
+    vlMode: TVlModeTypes | undefined,
+    uiTarsVersion: UITarsModelVersion | undefined,
+  ) => {
+    if (vlMode) {
+      if (uiTarsVersion) {
+        return `UI-TARS=${uiTarsVersion}`;
+      } else {
+        return `${vlMode} mode`;
+      }
+    }
+    return '';
+  };
+  const modelDescription = getModelDescription(vlMode, uiTarsVersion);
+
+  return {
+    socksProxy,
+    httpProxy,
+    vlModeRaw: vlMode,
+    openaiBaseURL,
+    openaiApiKey,
+    openaiExtraConfig,
+    vlMode,
+    uiTarsModelVersion: uiTarsVersion,
+    modelName: modelName!,
+    modelDescription,
+    intent: '-' as any,
+    timeout: provider[keys.timeout]
+      ? Number(provider[keys.timeout])
+      : undefined,
+  };
+};
+
+export const decideModelConfigFromIntentConfig = (
+  intent: TIntent,
+  configMap: Record<string, string | undefined>,
+): IModelConfig | undefined => {
+  const debugLog = getDebug('ai:config');
+
+  debugLog(
+    'will decideModelConfig base on agent.modelConfig()',
+    intent,
+    maskConfig(configMap),
+  );
+
+  const keysForFn = KEYS_MAP[intent];
+  const modelName = configMap[keysForFn.modelName];
+
+  if (!modelName) {
+    debugLog('no modelName found for intent', intent);
+    return undefined;
+  }
+
+  const finalResult = parseOpenaiSdkConfig({
+    keys: keysForFn,
+    provider: configMap,
+    useLegacyLogic: intent === 'default',
+  });
+  finalResult.intent = intent;
+
+  debugLog(
+    'decideModelConfig result by agent.modelConfig() with intent',
+    intent,
+    maskConfig({ ...finalResult }),
+  );
+
+  assert(
+    finalResult.openaiBaseURL,
+    `failed to get base URL of model (intent=${intent}). See https://midscenejs.com/model-strategy`,
+  );
+
+  if (!finalResult.modelName) {
+    console.warn(
+      `modelName is not set for intent ${intent}, this may cause unexpected behavior. See https://midscenejs.com/model-strategy`,
+    );
+  }
+
+  return finalResult;
+};