@xiuchang-midscene/shared 2.0.2

package/src/env/types.ts

@@ -0,0 +1,547 @@
+// config keys
+export const MIDSCENE_MODEL_INIT_CONFIG_JSON =
+  'MIDSCENE_MODEL_INIT_CONFIG_JSON';
+export const MIDSCENE_MODEL_EXTRA_BODY_JSON = 'MIDSCENE_MODEL_EXTRA_BODY_JSON';
+export const MIDSCENE_MODEL_NAME = 'MIDSCENE_MODEL_NAME';
+export const MIDSCENE_DEBUG_MODEL_PROFILE = 'MIDSCENE_DEBUG_MODEL_PROFILE';
+export const MIDSCENE_DEBUG_MODEL_RESPONSE = 'MIDSCENE_DEBUG_MODEL_RESPONSE';
+export const MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG =
+  'MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG';
+export const MIDSCENE_DEBUG_MODE = 'MIDSCENE_DEBUG_MODE';
+export const MIDSCENE_MCP_USE_PUPPETEER_MODE =
+  'MIDSCENE_MCP_USE_PUPPETEER_MODE';
+export const MIDSCENE_MCP_CHROME_PATH = 'MIDSCENE_MCP_CHROME_PATH';
+export const MIDSCENE_MCP_ANDROID_MODE = 'MIDSCENE_MCP_ANDROID_MODE';
+export const DOCKER_CONTAINER = 'DOCKER_CONTAINER';
+
+// Observability
+export const MIDSCENE_LANGSMITH_DEBUG = 'MIDSCENE_LANGSMITH_DEBUG';
+export const MIDSCENE_LANGFUSE_DEBUG = 'MIDSCENE_LANGFUSE_DEBUG';
+
+export const MIDSCENE_MODEL_SOCKS_PROXY = 'MIDSCENE_MODEL_SOCKS_PROXY';
+export const MIDSCENE_MODEL_HTTP_PROXY = 'MIDSCENE_MODEL_HTTP_PROXY';
+
+// New primary names for public API
+export const MIDSCENE_MODEL_API_KEY = 'MIDSCENE_MODEL_API_KEY';
+export const MIDSCENE_MODEL_BASE_URL = 'MIDSCENE_MODEL_BASE_URL';
+export const MIDSCENE_MODEL_MAX_TOKENS = 'MIDSCENE_MODEL_MAX_TOKENS';
+export const MIDSCENE_MODEL_TIMEOUT = 'MIDSCENE_MODEL_TIMEOUT';
+export const MIDSCENE_MODEL_TEMPERATURE = 'MIDSCENE_MODEL_TEMPERATURE';
+export const MIDSCENE_MODEL_RETRY_COUNT = 'MIDSCENE_MODEL_RETRY_COUNT';
+export const MIDSCENE_MODEL_RETRY_INTERVAL = 'MIDSCENE_MODEL_RETRY_INTERVAL';
+export const MIDSCENE_MODEL_REASONING_EFFORT =
+  'MIDSCENE_MODEL_REASONING_EFFORT';
+export const MIDSCENE_MODEL_REASONING_ENABLED =
+  'MIDSCENE_MODEL_REASONING_ENABLED';
+export const MIDSCENE_MODEL_REASONING_BUDGET =
+  'MIDSCENE_MODEL_REASONING_BUDGET';
+
+/**
+ * @deprecated Use MIDSCENE_MODEL_API_KEY instead. This is kept for backward compatibility.
+ */
+export const OPENAI_API_KEY = 'OPENAI_API_KEY';
+/**
+ * @deprecated Use MIDSCENE_MODEL_BASE_URL instead. This is kept for backward compatibility.
+ */
+export const OPENAI_BASE_URL = 'OPENAI_BASE_URL';
+/**
+ * @deprecated Use MIDSCENE_MODEL_INIT_CONFIG_JSON instead. This is kept for backward compatibility.
+ */
+export const MIDSCENE_OPENAI_INIT_CONFIG_JSON =
+  'MIDSCENE_OPENAI_INIT_CONFIG_JSON';
+/**
+ * @deprecated Use MIDSCENE_MODEL_HTTP_PROXY instead. This is kept for backward compatibility.
+ */
+export const MIDSCENE_OPENAI_HTTP_PROXY = 'MIDSCENE_OPENAI_HTTP_PROXY';
+/**
+ * @deprecated Use MIDSCENE_MODEL_SOCKS_PROXY instead. This is kept for backward compatibility.
+ */
+export const MIDSCENE_OPENAI_SOCKS_PROXY = 'MIDSCENE_OPENAI_SOCKS_PROXY';
+/**
+ * @deprecated Use MIDSCENE_MODEL_MAX_TOKENS instead. This is kept for backward compatibility.
+ */
+export const OPENAI_MAX_TOKENS = 'OPENAI_MAX_TOKENS';
+
+export const MIDSCENE_ADB_PATH = 'MIDSCENE_ADB_PATH';
+export const MIDSCENE_ADB_REMOTE_HOST = 'MIDSCENE_ADB_REMOTE_HOST';
+export const MIDSCENE_ADB_REMOTE_PORT = 'MIDSCENE_ADB_REMOTE_PORT';
+export const MIDSCENE_ANDROID_IME_STRATEGY = 'MIDSCENE_ANDROID_IME_STRATEGY';
+
+export const MIDSCENE_IOS_DEVICE_UDID = 'MIDSCENE_IOS_DEVICE_UDID';
+export const MIDSCENE_IOS_SIMULATOR_UDID = 'MIDSCENE_IOS_SIMULATOR_UDID';
+
+export const MIDSCENE_CACHE = 'MIDSCENE_CACHE';
+export const MIDSCENE_USE_VLM_UI_TARS = 'MIDSCENE_USE_VLM_UI_TARS';
+export const MIDSCENE_USE_QWEN_VL = 'MIDSCENE_USE_QWEN_VL';
+export const MIDSCENE_USE_QWEN3_VL = 'MIDSCENE_USE_QWEN3_VL';
+export const MIDSCENE_USE_DOUBAO_VISION = 'MIDSCENE_USE_DOUBAO_VISION';
+export const MIDSCENE_USE_GEMINI = 'MIDSCENE_USE_GEMINI';
+export const MIDSCENE_USE_VL_MODEL = 'MIDSCENE_USE_VL_MODEL';
+export const MATCH_BY_POSITION = 'MATCH_BY_POSITION';
+export const MIDSCENE_REPORT_TAG_NAME = 'MIDSCENE_REPORT_TAG_NAME';
+export const MIDSCENE_REPORT_QUIET = 'MIDSCENE_REPORT_QUIET';
+
+export const MIDSCENE_PREFERRED_LANGUAGE = 'MIDSCENE_PREFERRED_LANGUAGE';
+
+export const MIDSCENE_CACHE_MAX_FILENAME_LENGTH =
+  'MIDSCENE_CACHE_MAX_FILENAME_LENGTH';
+
+export const MIDSCENE_REPLANNING_CYCLE_LIMIT =
+  'MIDSCENE_REPLANNING_CYCLE_LIMIT';
+
+export const MIDSCENE_RUN_DIR = 'MIDSCENE_RUN_DIR';
+
+// OSS Report Upload
+export const MIDSCENE_OSS_ENABLE = 'MIDSCENE_OSS_ENABLE';
+export const MIDSCENE_OSS_TOKEN_URL = 'MIDSCENE_OSS_TOKEN_URL';
+export const MIDSCENE_OSS_TOKEN_RESPONSE_PATH =
+  'MIDSCENE_OSS_TOKEN_RESPONSE_PATH';
+export const MIDSCENE_OSS_REGION = 'MIDSCENE_OSS_REGION';
+export const MIDSCENE_OSS_BUCKET = 'MIDSCENE_OSS_BUCKET';
+export const MIDSCENE_OSS_ENDPOINT = 'MIDSCENE_OSS_ENDPOINT';
+export const MIDSCENE_OSS_PREFIX = 'MIDSCENE_OSS_PREFIX';
+export const MIDSCENE_OSS_DOMAIN = 'MIDSCENE_OSS_DOMAIN';
+
+// Report optimization
+export const MIDSCENE_REPORT_JPEG_QUALITY = 'MIDSCENE_REPORT_JPEG_QUALITY';
+
+// INSIGHT (unified VQA and Grounding)
+export const MIDSCENE_INSIGHT_MODEL_NAME = 'MIDSCENE_INSIGHT_MODEL_NAME';
+export const MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY =
+  'MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY';
+export const MIDSCENE_INSIGHT_MODEL_HTTP_PROXY =
+  'MIDSCENE_INSIGHT_MODEL_HTTP_PROXY';
+export const MIDSCENE_INSIGHT_MODEL_BASE_URL =
+  'MIDSCENE_INSIGHT_MODEL_BASE_URL';
+export const MIDSCENE_INSIGHT_MODEL_API_KEY = 'MIDSCENE_INSIGHT_MODEL_API_KEY';
+export const MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON =
+  'MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON';
+export const MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON =
+  'MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON';
+export const MIDSCENE_INSIGHT_MODEL_TIMEOUT = 'MIDSCENE_INSIGHT_MODEL_TIMEOUT';
+export const MIDSCENE_INSIGHT_MODEL_TEMPERATURE =
+  'MIDSCENE_INSIGHT_MODEL_TEMPERATURE';
+export const MIDSCENE_INSIGHT_MODEL_RETRY_COUNT =
+  'MIDSCENE_INSIGHT_MODEL_RETRY_COUNT';
+export const MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL =
+  'MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL';
+export const MIDSCENE_INSIGHT_MODEL_FAMILY = 'MIDSCENE_INSIGHT_MODEL_FAMILY';
+export const MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT =
+  'MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT';
+export const MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED =
+  'MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED';
+export const MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET =
+  'MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET';
+
+// PLANNING
+export const MIDSCENE_PLANNING_MODEL_NAME = 'MIDSCENE_PLANNING_MODEL_NAME';
+export const MIDSCENE_PLANNING_MODEL_SOCKS_PROXY =
+  'MIDSCENE_PLANNING_MODEL_SOCKS_PROXY';
+export const MIDSCENE_PLANNING_MODEL_HTTP_PROXY =
+  'MIDSCENE_PLANNING_MODEL_HTTP_PROXY';
+export const MIDSCENE_PLANNING_MODEL_BASE_URL =
+  'MIDSCENE_PLANNING_MODEL_BASE_URL';
+export const MIDSCENE_PLANNING_MODEL_API_KEY =
+  'MIDSCENE_PLANNING_MODEL_API_KEY';
+export const MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON =
+  'MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON';
+export const MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON =
+  'MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON';
+export const MIDSCENE_PLANNING_MODEL_TIMEOUT =
+  'MIDSCENE_PLANNING_MODEL_TIMEOUT';
+export const MIDSCENE_PLANNING_MODEL_TEMPERATURE =
+  'MIDSCENE_PLANNING_MODEL_TEMPERATURE';
+export const MIDSCENE_PLANNING_MODEL_RETRY_COUNT =
+  'MIDSCENE_PLANNING_MODEL_RETRY_COUNT';
+export const MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL =
+  'MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL';
+export const MIDSCENE_PLANNING_MODEL_FAMILY = 'MIDSCENE_PLANNING_MODEL_FAMILY';
+export const MIDSCENE_PLANNING_MODEL_REASONING_EFFORT =
+  'MIDSCENE_PLANNING_MODEL_REASONING_EFFORT';
+export const MIDSCENE_PLANNING_MODEL_REASONING_ENABLED =
+  'MIDSCENE_PLANNING_MODEL_REASONING_ENABLED';
+export const MIDSCENE_PLANNING_MODEL_REASONING_BUDGET =
+  'MIDSCENE_PLANNING_MODEL_REASONING_BUDGET';
+export const MIDSCENE_MODEL_FAMILY = 'MIDSCENE_MODEL_FAMILY';
+
+/**
+ * env keys declared but unused
+ */
+export const UNUSED_ENV_KEYS = [MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG];
+
+/**
+ * env keys for debug or basic run
+ * can not be override by overrideAIConfig
+ */
+export const BASIC_ENV_KEYS = [
+  MIDSCENE_DEBUG_MODE,
+  MIDSCENE_DEBUG_MODEL_PROFILE,
+  MIDSCENE_DEBUG_MODEL_RESPONSE,
+  MIDSCENE_RUN_DIR,
+] as const;
+
+export const BOOLEAN_ENV_KEYS = [
+  MIDSCENE_CACHE,
+  MIDSCENE_MCP_USE_PUPPETEER_MODE,
+  MIDSCENE_MCP_ANDROID_MODE,
+  MIDSCENE_LANGSMITH_DEBUG,
+  MIDSCENE_LANGFUSE_DEBUG,
+  MIDSCENE_REPORT_QUIET,
+] as const;
+
+export const NUMBER_ENV_KEYS = [
+  MIDSCENE_MODEL_MAX_TOKENS,
+  MIDSCENE_CACHE_MAX_FILENAME_LENGTH,
+  MIDSCENE_REPLANNING_CYCLE_LIMIT,
+] as const;
+
+export const STRING_ENV_KEYS = [
+  MIDSCENE_MODEL_MAX_TOKENS,
+  OPENAI_MAX_TOKENS,
+  MIDSCENE_ADB_PATH,
+  MIDSCENE_ADB_REMOTE_HOST,
+  MIDSCENE_ADB_REMOTE_PORT,
+  MIDSCENE_ANDROID_IME_STRATEGY,
+  MIDSCENE_IOS_DEVICE_UDID,
+  MIDSCENE_IOS_SIMULATOR_UDID,
+  MIDSCENE_REPORT_TAG_NAME,
+  MIDSCENE_PREFERRED_LANGUAGE,
+  MATCH_BY_POSITION,
+  MIDSCENE_MCP_CHROME_PATH,
+  DOCKER_CONTAINER,
+] as const;
+
+/**
+ * Non model related env keys, used for globally controlling the behavior of midscene
+ * Can not be override by agent.modelConfig but can be override by overrideAIConfig
+ * Can be access at any time
+ */
+export const GLOBAL_ENV_KEYS = [
+  ...BOOLEAN_ENV_KEYS,
+  ...NUMBER_ENV_KEYS,
+  ...STRING_ENV_KEYS,
+] as const;
+
+/**
+ * Model related eve keys, used for declare which model to use.
+ * Can be override by both agent.modelConfig and overrideAIConfig
+ * Can only be access after agent.constructor
+ */
+export const MODEL_ENV_KEYS = [
+  // model default
+  MIDSCENE_MODEL_NAME,
+  MIDSCENE_MODEL_INIT_CONFIG_JSON,
+  MIDSCENE_MODEL_EXTRA_BODY_JSON,
+  MIDSCENE_MODEL_API_KEY,
+  MIDSCENE_MODEL_BASE_URL,
+  MIDSCENE_MODEL_SOCKS_PROXY,
+  MIDSCENE_MODEL_HTTP_PROXY,
+  MIDSCENE_MODEL_TIMEOUT,
+  MIDSCENE_MODEL_TEMPERATURE,
+  MIDSCENE_MODEL_RETRY_COUNT,
+  MIDSCENE_MODEL_RETRY_INTERVAL,
+  MIDSCENE_MODEL_REASONING_EFFORT,
+  MIDSCENE_MODEL_REASONING_ENABLED,
+  MIDSCENE_MODEL_REASONING_BUDGET,
+  MIDSCENE_USE_VLM_UI_TARS,
+  MIDSCENE_USE_QWEN_VL,
+  MIDSCENE_USE_QWEN3_VL,
+  MIDSCENE_USE_DOUBAO_VISION,
+  MIDSCENE_USE_GEMINI,
+  MIDSCENE_USE_VL_MODEL,
+  // model default legacy
+  OPENAI_API_KEY,
+  OPENAI_BASE_URL,
+  MIDSCENE_OPENAI_INIT_CONFIG_JSON,
+  MIDSCENE_OPENAI_HTTP_PROXY,
+  MIDSCENE_OPENAI_SOCKS_PROXY,
+  // INSIGHT (unified VQA and Grounding)
+  MIDSCENE_INSIGHT_MODEL_NAME,
+  MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY,
+  MIDSCENE_INSIGHT_MODEL_HTTP_PROXY,
+  MIDSCENE_INSIGHT_MODEL_BASE_URL,
+  MIDSCENE_INSIGHT_MODEL_API_KEY,
+  MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON,
+  MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON,
+  MIDSCENE_INSIGHT_MODEL_TIMEOUT,
+  MIDSCENE_INSIGHT_MODEL_TEMPERATURE,
+  MIDSCENE_INSIGHT_MODEL_RETRY_COUNT,
+  MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL,
+  MIDSCENE_INSIGHT_MODEL_FAMILY,
+  MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT,
+  MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED,
+  MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET,
+  // PLANNING
+  MIDSCENE_PLANNING_MODEL_NAME,
+  MIDSCENE_PLANNING_MODEL_SOCKS_PROXY,
+  MIDSCENE_PLANNING_MODEL_HTTP_PROXY,
+  MIDSCENE_PLANNING_MODEL_BASE_URL,
+  MIDSCENE_PLANNING_MODEL_API_KEY,
+  MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON,
+  MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON,
+  MIDSCENE_PLANNING_MODEL_TIMEOUT,
+  MIDSCENE_PLANNING_MODEL_TEMPERATURE,
+  MIDSCENE_PLANNING_MODEL_RETRY_COUNT,
+  MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL,
+  MIDSCENE_PLANNING_MODEL_FAMILY,
+  MIDSCENE_PLANNING_MODEL_REASONING_EFFORT,
+  MIDSCENE_PLANNING_MODEL_REASONING_ENABLED,
+  MIDSCENE_PLANNING_MODEL_REASONING_BUDGET,
+  MIDSCENE_MODEL_FAMILY,
+] as const;
+
+export const ALL_ENV_KEYS = [
+  ...UNUSED_ENV_KEYS,
+  ...BASIC_ENV_KEYS,
+  ...GLOBAL_ENV_KEYS,
+  ...MODEL_ENV_KEYS,
+] as const;
+
+export type TEnvKeys = (typeof ALL_ENV_KEYS)[number];
+export type TGlobalConfig = Record<TEnvKeys, string | undefined>;
+
+/**
+ * valid Model family types
+ */
+export type TModelFamily =
+  | 'qwen2.5-vl'
+  | 'qwen3-vl'
+  | 'qwen3.5'
+  | 'doubao-vision'
+  | 'doubao-seed'
+  | 'gemini'
+  | 'vlm-ui-tars'
+  | 'vlm-ui-tars-doubao'
+  | 'vlm-ui-tars-doubao-1.5'
+  | 'glm-v'
+  | 'auto-glm'
+  | 'auto-glm-multilingual'
+  | 'gpt-5';
+
+export const MODEL_FAMILY_VALUES: TModelFamily[] = [
+  'doubao-vision',
+  'doubao-seed',
+  'gemini',
+  'qwen2.5-vl',
+  'qwen3-vl',
+  'qwen3.5',
+  'vlm-ui-tars',
+  'vlm-ui-tars-doubao',
+  'vlm-ui-tars-doubao-1.5',
+  'glm-v',
+  'auto-glm',
+  'auto-glm-multilingual',
+  'gpt-5',
+];
+
+export interface IModelConfigForInsight {
+  // model name
+  [MIDSCENE_INSIGHT_MODEL_NAME]: string;
+  // proxy
+  [MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY]?: string;
+  [MIDSCENE_INSIGHT_MODEL_HTTP_PROXY]?: string;
+  // OpenAI
+  [MIDSCENE_INSIGHT_MODEL_BASE_URL]?: string;
+  [MIDSCENE_INSIGHT_MODEL_API_KEY]?: string;
+  [MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON]?: string;
+  [MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON]?: string;
+  // timeout
+  [MIDSCENE_INSIGHT_MODEL_TIMEOUT]?: string;
+  // temperature
+  [MIDSCENE_INSIGHT_MODEL_TEMPERATURE]?: string;
+  // model family
+  [MIDSCENE_INSIGHT_MODEL_FAMILY]?: TModelFamily;
+}
+
+export interface IModelConfigForPlanning {
+  // model name
+  [MIDSCENE_PLANNING_MODEL_NAME]: string;
+  // proxy
+  [MIDSCENE_PLANNING_MODEL_SOCKS_PROXY]?: string;
+  [MIDSCENE_PLANNING_MODEL_HTTP_PROXY]?: string;
+  // OpenAI
+  [MIDSCENE_PLANNING_MODEL_BASE_URL]?: string;
+  [MIDSCENE_PLANNING_MODEL_API_KEY]?: string;
+  [MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON]?: string;
+  [MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON]?: string;
+  // timeout
+  [MIDSCENE_PLANNING_MODEL_TIMEOUT]?: string;
+  // temperature
+  [MIDSCENE_PLANNING_MODEL_TEMPERATURE]?: string;
+  // model family
+  [MIDSCENE_PLANNING_MODEL_FAMILY]?: TModelFamily;
+}
+
+/**
+ * Model configuration for Planning intent.
+ *
+ * IMPORTANT: Planning MUST use a vision language model (VL mode).
+ * DOM-based planning is not supported.
+ *
+ * Required: MIDSCENE_MODEL_FAMILY must be set to one of 「TModelFamily」
+ */
+export interface IModelConfigForDefault {
+  // model name
+  [MIDSCENE_MODEL_NAME]: string;
+  // proxy
+  [MIDSCENE_MODEL_SOCKS_PROXY]?: string;
+  [MIDSCENE_MODEL_HTTP_PROXY]?: string;
+  // OpenAI
+  [MIDSCENE_MODEL_BASE_URL]?: string;
+  [MIDSCENE_MODEL_API_KEY]?: string;
+  [MIDSCENE_MODEL_INIT_CONFIG_JSON]?: string;
+  [MIDSCENE_MODEL_EXTRA_BODY_JSON]?: string;
+  // extra
+  [MIDSCENE_MODEL_FAMILY]?: TModelFamily;
+  // temperature
+  [MIDSCENE_MODEL_TEMPERATURE]?: string;
+  // reasoning effort
+  [MIDSCENE_MODEL_REASONING_EFFORT]?: string;
+  // enable reasoning (boolean as string)
+  [MIDSCENE_MODEL_REASONING_ENABLED]?: string;
+  // reasoning budget (number as string)
+  [MIDSCENE_MODEL_REASONING_BUDGET]?: string;
+}
+
+export interface IModelConfigForDefaultLegacy {
+  // model name
+  [MIDSCENE_MODEL_NAME]: string;
+  // proxy
+  [MIDSCENE_OPENAI_SOCKS_PROXY]?: string;
+  [MIDSCENE_OPENAI_HTTP_PROXY]?: string;
+  // OpenAI
+  [OPENAI_BASE_URL]?: string;
+  [OPENAI_API_KEY]?: string;
+  [MIDSCENE_OPENAI_INIT_CONFIG_JSON]?: string;
+}
+
+/**
+ * - insight: Visual Question Answering and Visual Grounding (unified)
+ * - planning: planning
+ * - default: all except insight、planning
+ */
+export type TIntent = 'insight' | 'planning' | 'default';
+
+/**
+ * Env-style model configuration map supplied directly to the agent.
+ * Numbers are allowed so callers can pass numeric env values (e.g. limits) without casting.
+ */
+export type TModelConfig = Record<string, string | number>;
+
+export enum UITarsModelVersion {
+  V1_0 = '1.0',
+  V1_5 = '1.5',
+  DOUBAO_1_5_15B = 'doubao-1.5-15B',
+  DOUBAO_1_5_20B = 'doubao-1.5-20B',
+}
+
+/**
+ * Callback to create custom OpenAI client instance
+ * @param config - Resolved model configuration including apiKey, baseURL, modelName, intent, etc.
+ * @returns OpenAI client instance (can be wrapped with langsmith, langfuse, etc.)
+ *
+ * Note: Wrapper functions like langsmith's wrapOpenAI() return the same OpenAI instance
+ * with enhanced behavior, so the return type remains compatible with OpenAI.
+ *
+ * Note: The return type is `any` in the shared package to avoid requiring openai as a dependency.
+ * The actual implementation should return an OpenAI instance.
+ *
+ * @example
+ * ```typescript
+ * import OpenAI from 'openai';
+ * import { wrapOpenAI } from 'langsmith/wrappers';
+ *
+ * createOpenAIClient: async (openai, opts) => {
+ *   // Wrap with langsmith for planning tasks
+ *   if (opts.baseURL?.includes('planning')) {
+ *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });
+ *   }
+ *
+ *   return openai;
+ * }
+ * ```
+ */
+export type CreateOpenAIClientFn = (
+  openAIInstance: any,
+  options: Record<string, unknown>,
+) => Promise<any>; // OpenAI instance, but typed as `any` to avoid dependency
+
+export interface IModelConfig {
+  /**
+   * proxy
+   */
+  socksProxy?: string;
+  httpProxy?: string;
+  /**
+   * model
+   */
+  modelName: string;
+  /**
+   * OpenAI
+   */
+  openaiBaseURL?: string;
+  openaiApiKey?: string;
+  openaiExtraConfig?: Record<string, unknown>;
+  /**
+   * Extra body parameters merged into each chat completion request body.
+   * Unlike openaiExtraConfig (which configures the OpenAI client instance),
+   * this is spread directly into the completion.create() call body.
+   * Example: { "chat_template_kwargs": { "enable_thinking": true } }
+   */
+  extraBody?: Record<string, unknown>;
+  /**
+   * Timeout for API calls in milliseconds.
+   * If not set, uses OpenAI SDK default (10 minutes).
+   */
+  timeout?: number;
+  /**
+   * Temperature for model sampling.
+   */
+  temperature?: number;
+  /**
+   * Number of retries when AI call fails.
+   * Default is 1 (retry once after failure).
+   */
+  retryCount?: number;
+  /**
+   * Interval between retries in milliseconds.
+   * Default is 2000.
+   */
+  retryInterval?: number;
+  /**
+   * Reasoning effort level for the model.
+   * Passed through to model-family-specific parameters (e.g., reasoning_effort for doubao, reasoning.effort for gpt-5).
+   */
+  reasoningEffort?: string;
+  /**
+   * Enable/disable reasoning for the model.
+   * Passed through to model-family-specific parameters (e.g., enable_thinking for qwen, thinking.type for doubao/glm-v).
+   */
+  reasoningEnabled?: boolean;
+  /**
+   * Reasoning token budget for the model.
+   * Passed through to model-family-specific parameters (e.g., thinking_budget for qwen).
+   */
+  reasoningBudget?: number;
+  /**
+   * Model family - unified model configuration
+   * Maps directly to model families like 'qwen2.5-vl', 'qwen3-vl', 'doubao-vision', 'doubao-seed', etc.
+   */
+  modelFamily?: TModelFamily;
+  uiTarsModelVersion?: UITarsModelVersion;
+  modelDescription: string;
+  /**
+   * original intent from the config
+   */
+  intent: TIntent;
+  /**
+   * Custom OpenAI client factory function
+   *
+   * If provided, this function will be called to create OpenAI client instances
+   * for each AI call, allowing you to:
+   * - Wrap clients with observability tools (langsmith, langfuse)
+   * - Use custom OpenAI-compatible clients
+   * - Apply different configurations based on intent
+   */
+  createOpenAIClient?: CreateOpenAIClientFn;
+}

package/src/env/utils.ts

@@ -0,0 +1,89 @@
+import { GlobalConfigManager } from './global-config-manager';
+import { ModelConfigManager } from './model-config-manager';
+import {
+  type GLOBAL_ENV_KEYS,
+  MIDSCENE_PREFERRED_LANGUAGE,
+  type MODEL_ENV_KEYS,
+} from './types';
+
+export const globalModelConfigManager = new ModelConfigManager();
+
+export const globalConfigManager = new GlobalConfigManager();
+
+globalConfigManager.registerModelConfigManager(globalModelConfigManager);
+globalModelConfigManager.registerGlobalConfigManager(globalConfigManager);
+
+/**
+ * Interface for devices that support getTimestamp method.
+ * This is a minimal interface to avoid circular dependencies with @midscene/core.
+ */
+export interface DeviceWithTimestamp {
+  getTimestamp?: () => Promise<number>;
+}
+
+/**
+ * Get the current timestamp, optionally from the target device.
+ *
+ * When useDeviceTimestamp is enabled and a device with getTimestamp is provided,
+ * this function will return the device's time. Otherwise, it returns the system time.
+ *
+ * This is useful when:
+ * - Testing on devices with different time zones
+ * - Debugging time-sensitive features
+ * - The system clock and device clock are not synchronized
+ *
+ * @param device Optional device interface that supports getTimestamp
+ * @param useDeviceTimestamp Whether to use device timestamp (from agent config)
+ * @returns Timestamp in milliseconds
+ *
+ * @example
+ * // Without device - always returns system time
+ * const systemTime = await getCurrentTime();
+ *
+ * @example
+ * // With device and config enabled - returns device time
+ * const deviceTime = await getCurrentTime(androidDevice, true);
+ */
+export async function getCurrentTime(
+  device?: DeviceWithTimestamp,
+  useDeviceTimestamp?: boolean,
+): Promise<number> {
+  if (useDeviceTimestamp && device?.getTimestamp) {
+    try {
+      return await device.getTimestamp();
+    } catch (error) {
+      // Fall back to system time if device time retrieval fails
+      console.warn(
+        `Failed to get device time, falling back to system time: ${error}`,
+      );
+      return Date.now();
+    }
+  }
+
+  return Date.now();
+}
+
+export const getPreferredLanguage = () => {
+  const prefer = globalConfigManager.getEnvConfigValue(
+    MIDSCENE_PREFERRED_LANGUAGE,
+  );
+  if (prefer) {
+    return prefer;
+  }
+
+  const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+  const isChina = timeZone === 'Asia/Shanghai';
+  return isChina ? 'Chinese' : 'English';
+};
+
+export const overrideAIConfig = (
+  newConfig: Partial<
+    Record<
+      (typeof GLOBAL_ENV_KEYS)[number] | (typeof MODEL_ENV_KEYS)[number],
+      string
+    >
+  >,
+  extendMode = false, // true: merge with global config, false: override global config
+) => {
+  globalConfigManager.overrideAIConfig(newConfig, extendMode);
+};

package/src/extractor/constants.ts

@@ -0,0 +1,5 @@
+export {
+  NodeType,
+  TEXT_MAX_SIZE,
+  TEXT_SIZE_THRESHOLD,
+} from '../constants/index';

package/src/extractor/debug.ts

@@ -0,0 +1,10 @@
+import { webExtractTextWithPosition } from '.';
+import {
+  setExtractTextWithPositionOnWindow,
+  setMidsceneVisibleRectOnWindow,
+} from './util';
+
+console.log(webExtractTextWithPosition(document.body, true));
+console.log(JSON.stringify(webExtractTextWithPosition(document.body, true)));
+setExtractTextWithPositionOnWindow();
+setMidsceneVisibleRectOnWindow();