@midscene/shared 0.30.10 → 1.0.0

package/dist/types/mcp/base-tools.d.ts

@@ -0,0 +1,55 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { BaseAgent, BaseDevice, IMidsceneTools, ToolDefinition } from './types';
+/**
+ * Base class for platform-specific MCP tools
+ * Generic type TAgent allows subclasses to use their specific agent types
+ */
+export declare abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent> implements IMidsceneTools {
+    protected mcpServer?: McpServer;
+    protected agent?: TAgent;
+    protected toolDefinitions: ToolDefinition[];
+    /**
+     * Ensure agent is initialized and ready for use.
+     * Must be implemented by subclasses to create platform-specific agent.
+     * @param initParam Optional initialization parameter (platform-specific, e.g., URL, device ID)
+     * @returns Promise resolving to initialized agent instance
+     * @throws Error if agent initialization fails
+     */
+    protected abstract ensureAgent(initParam?: string): Promise<TAgent>;
+    /**
+     * Optional: prepare platform-specific tools (e.g., device connection)
+     */
+    protected preparePlatformTools(): ToolDefinition[];
+    /**
+     * Must be implemented by subclasses to create a temporary device instance
+     * This allows getting real actionSpace without connecting to device
+     */
+    protected abstract createTemporaryDevice(): BaseDevice;
+    /**
+     * Initialize all tools by querying actionSpace
+     * Uses two-layer fallback strategy:
+     * 1. Try to get actionSpace from connected agent (if available)
+     * 2. Create temporary device instance to read actionSpace (always succeeds)
+     */
+    initTools(): Promise<void>;
+    /**
+     * Attach to MCP server and register all tools
+     */
+    attachToServer(server: McpServer): void;
+    /**
+     * Wrapper for auto-destroy behavior
+     */
+    private toolWithAutoDestroy;
+    /**
+     * Cleanup method - destroy agent and release resources
+     */
+    closeBrowser(): Promise<void>;
+    /**
+     * Helper: Convert base64 screenshot to image content array
+     */
+    protected buildScreenshotContent(screenshot: string): {
+        type: "image";
+        data: string;
+        mimeType: string;
+    }[];
+}

package/dist/types/mcp/index.d.ts

@@ -0,0 +1,5 @@
+export * from './base-server';
+export * from './base-tools';
+export * from './tool-generator';
+export * from './types';
+export * from './inject-report-html-plugin';

package/dist/types/mcp/inject-report-html-plugin.d.ts

@@ -0,0 +1,18 @@
+interface RslibPluginApi {
+    onAfterBuild: (callback: () => void) => void;
+}
+/**
+ * Rslib plugin to inject report HTML from @midscene/core dist into MCP bundle.
+ * This runs after build and reads the already-injected HTML from core.
+ *
+ * Prerequisites:
+ * - @midscene/report must be in devDependencies to ensure correct build order
+ * - @midscene/core dist must exist with injected HTML
+ *
+ * @param packageDir - The directory of the MCP package (use __dirname)
+ */
+export declare function injectReportHtmlFromCore(packageDir: string): {
+    name: string;
+    setup(api: RslibPluginApi): void;
+};
+export {};

package/dist/types/mcp/tool-generator.d.ts

@@ -0,0 +1,11 @@
+import type { ActionSpaceItem, BaseAgent, ToolDefinition } from './types';
+/**
+ * Converts DeviceAction from actionSpace into MCP ToolDefinition
+ * This is the core logic that removes need for hardcoded tool definitions
+ */
+export declare function generateToolsFromActionSpace(actionSpace: ActionSpaceItem[], getAgent: () => Promise<BaseAgent>): ToolDefinition[];
+/**
+ * Generate common tools (screenshot, waitFor)
+ * SIMPLIFIED: Only keep essential helper tools, removed assert
+ */
+export declare function generateCommonTools(getAgent: () => Promise<BaseAgent>): ToolDefinition[];

package/dist/types/mcp/types.d.ts

@@ -0,0 +1,100 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { z } from 'zod';
+/**
+ * Default timeout constants for app loading verification
+ */
+export declare const defaultAppLoadingTimeoutMs = 10000;
+export declare const defaultAppLoadingCheckIntervalMs = 2000;
+/**
+ * Content item types for tool results (MCP compatible)
+ */
+export type ToolResultContent = {
+    type: 'text';
+    text: string;
+} | {
+    type: 'image';
+    data: string;
+    mimeType: string;
+} | {
+    type: 'audio';
+    data: string;
+    mimeType: string;
+} | {
+    type: 'resource';
+    resource: {
+        text: string;
+        uri: string;
+        mimeType?: string;
+    } | {
+        uri: string;
+        blob: string;
+        mimeType?: string;
+    };
+};
+/**
+ * Result type for tool execution (MCP compatible)
+ */
+export interface ToolResult {
+    [x: string]: unknown;
+    content: ToolResultContent[];
+    isError?: boolean;
+    _meta?: Record<string, unknown>;
+}
+/**
+ * Tool handler function type
+ * Takes parsed arguments and returns a tool result
+ */
+export type ToolHandler<T = Record<string, unknown>> = (args: T) => Promise<ToolResult>;
+/**
+ * Tool schema type using Zod
+ */
+export type ToolSchema = Record<string, z.ZodTypeAny>;
+/**
+ * Tool definition for MCP server
+ */
+export interface ToolDefinition<T = Record<string, unknown>> {
+    name: string;
+    description: string;
+    schema: ToolSchema;
+    handler: ToolHandler<T>;
+    autoDestroy?: boolean;
+}
+/**
+ * Action space item definition
+ * Note: Intentionally no index signature to maintain compatibility with DeviceAction
+ */
+export interface ActionSpaceItem {
+    name: string;
+    description?: string;
+    args?: Record<string, unknown>;
+    paramSchema?: z.ZodTypeAny;
+}
+/**
+ * Base agent interface
+ * Represents a platform-specific agent (Android, iOS, Web)
+ * Note: Return types use `unknown` for compatibility with platform-specific implementations
+ */
+export interface BaseAgent {
+    getActionSpace(): Promise<ActionSpaceItem[]>;
+    destroy?(): Promise<void>;
+    page?: {
+        screenshotBase64(): Promise<string>;
+    };
+    aiAction?: (description: string, params?: Record<string, unknown>) => Promise<unknown>;
+    aiWaitFor?: (assertion: string, options: Record<string, unknown>) => Promise<unknown>;
+}
+/**
+ * Base device interface for temporary device instances
+ */
+export interface BaseDevice {
+    actionSpace(): ActionSpaceItem[];
+    destroy?(): Promise<void>;
+}
+/**
+ * Interface for platform-specific MCP tools manager
+ */
+export interface IMidsceneTools {
+    attachToServer(server: McpServer): void;
+    initTools(): Promise<void>;
+    closeBrowser?(): Promise<void>;
+}

package/dist/types/types/index.d.ts

@@ -14,7 +14,6 @@ export type Rect = Point & Size & {
 };
 export declare abstract class BaseElement {
     abstract id: string;
-    abstract indexId?: number;
     abstract attributes: {
         nodeType: NodeType;
         [key: string]: string;
@@ -22,7 +21,6 @@ export declare abstract class BaseElement {
     abstract content: string;
     abstract rect: Rect;
     abstract center: [number, number];
-    abstract xpaths?: string[];
     abstract isVisible: boolean;
 }
 export interface ElementTreeNode<ElementType extends BaseElement = BaseElement> {
@@ -32,3 +30,8 @@ export interface ElementTreeNode<ElementType extends BaseElement = BaseElement>
 export interface WebElementInfo extends ElementInfo {
     zoom: number;
 }
+export type LocateResultElement = {
+    description: string;
+    center: [number, number];
+    rect: Rect;
+};

package/dist/types/zod-schema-utils.d.ts

@@ -0,0 +1,23 @@
+import type { z } from 'zod';
+/**
+ * Recursively unwrap optional, nullable, default, and effects wrapper types
+ * to get the actual inner Zod type
+ */
+export declare function unwrapZodField(field: unknown): unknown;
+/**
+ * Check if a field is a Midscene locator field
+ * Checks for either:
+ * 1. midscene_location_field_flag in shape (result schema)
+ * 2. prompt field in shape (input schema)
+ */
+export declare function isMidsceneLocatorField(field: unknown): boolean;
+/**
+ * Get type name string from a Zod schema field
+ * @param field - Zod schema field
+ * @param locatorTypeDescription - Optional description for MidsceneLocation fields (used by core)
+ */
+export declare function getZodTypeName(field: unknown, locatorTypeDescription?: string): string;
+/**
+ * Get description from a Zod schema field
+ */
+export declare function getZodDescription(field: z.ZodTypeAny): string | null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/shared",
-  "version": "0.30.10",
+  "version": "1.0.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",
@@ -57,6 +57,16 @@
       "import": "./dist/es/common.mjs",
       "require": "./dist/lib/common.js"
     },
+    "./mcp": {
+      "types": "./dist/types/mcp/index.d.ts",
+      "import": "./dist/es/mcp/index.mjs",
+      "require": "./dist/lib/mcp/index.js"
+    },
+    "./logger": {
+      "types": "./dist/types/logger.d.ts",
+      "import": "./dist/es/logger.mjs",
+      "require": "./dist/lib/logger.js"
+    },
     "./*": {
       "types": "./dist/types/*.d.ts",
       "import": "./dist/es/*.mjs",
@@ -72,20 +82,25 @@
     "@silvia-odwyer/photon": "0.3.3",
     "@silvia-odwyer/photon-node": "0.3.3",
     "debug": "4.4.0",
+    "express": "^4.21.2",
     "jimp": "0.22.12",
     "js-sha256": "0.11.0",
     "sharp": "^0.34.3",
     "uuid": "11.1.0"
   },
   "devDependencies": {
-    "@rslib/core": "^0.11.2",
+    "@rslib/core": "^0.18.3",
+    "@modelcontextprotocol/sdk": "1.10.2",
     "@types/debug": "4.1.12",
+    "@types/express": "^4.17.21",
     "@types/node": "^18.0.0",
     "@ui-tars/shared": "1.2.0",
     "dotenv": "^16.4.5",
+    "openai": "6.3.0",
     "rimraf": "~3.0.2",
     "typescript": "^5.8.3",
-    "vitest": "3.0.5"
+    "vitest": "3.0.5",
+    "zod": "3.24.3"
   },
   "sideEffects": [],
   "publishConfig": {
@@ -98,7 +113,7 @@
     "build": "npm run build:script && npm run build:pkg",
     "build:pkg": "rslib build",
     "build:script": "rslib build -c ./rslib.inspect.config.ts",
-    "build:watch": "npm run build:script && rslib build --watch",
+    "build:watch": "npm run build:script && rslib build --watch --no-clean",
     "reset": "rimraf ./**/node_modules",
     "test": "vitest --run",
     "test:u": "vitest --run -u"

package/src/build/rspack-config.ts

@@ -0,0 +1,12 @@
+/**
+ * Common Rspack configuration helpers for rsbuild projects
+ */
+
+/**
+ * Common warning patterns to ignore in Rspack builds.
+ * These warnings are typically from optional dependencies or known non-critical issues.
+ */
+export const commonIgnoreWarnings = [
+  // Ignore dynamic import warnings from langsmith/langfuse optional dependencies
+  /Critical dependency: the request of a dependency is an expression/,
+];

package/src/constants/example-code.ts

@@ -114,12 +114,12 @@ tasks:
       # Auto Planning (.ai)
       # ----------------
 
-      # Perform an interaction. \`ai\` is a shorthand for \`aiAction\`.
+      # Perform an interaction. \`ai\` is a shorthand for \`aiAct\`.
       - ai: <prompt>
         cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
 
       # This usage is the same as \`ai\`.
-      - aiAction: <prompt>
+      - aiAct: <prompt>
         cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
 
       # Instant Action (.aiTap, .aiDoubleClick, .aiHover, .aiInput, .aiKeyboardPress, .aiScroll)
@@ -167,8 +167,8 @@ tasks:
         xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Midscene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.
         cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
 
-      # Log the current screenshot with a description in the report file.
-      - logScreenshot: <title> # Optional, the title of the screenshot. If not provided, the title will be 'untitled'.
+      # Record the current screenshot with a description in the report file.
+      - recordToReport: <title> # Optional, the title of the screenshot. If not provided, the title will be 'untitled'.
         content: <content> # Optional, the description of the screenshot.
 
       # Data Extraction

package/src/env/constants.ts

@@ -1,81 +1,31 @@
 import {
-  ANTHROPIC_API_KEY,
-  AZURE_OPENAI_API_VERSION,
-  AZURE_OPENAI_DEPLOYMENT,
-  AZURE_OPENAI_ENDPOINT,
-  AZURE_OPENAI_KEY,
-  MIDSCENE_ANTHROPIC_API_KEY,
-  MIDSCENE_AZURE_OPENAI_API_VERSION,
-  MIDSCENE_AZURE_OPENAI_DEPLOYMENT,
-  MIDSCENE_AZURE_OPENAI_ENDPOINT,
-  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
-  MIDSCENE_AZURE_OPENAI_KEY,
-  MIDSCENE_AZURE_OPENAI_SCOPE,
-  MIDSCENE_GROUNDING_ANTHROPIC_API_KEY,
-  MIDSCENE_GROUNDING_AZURE_OPENAI_API_VERSION,
-  MIDSCENE_GROUNDING_AZURE_OPENAI_DEPLOYMENT,
-  MIDSCENE_GROUNDING_AZURE_OPENAI_ENDPOINT,
-  MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON,
-  MIDSCENE_GROUNDING_AZURE_OPENAI_KEY,
-  MIDSCENE_GROUNDING_AZURE_OPENAI_SCOPE,
-  MIDSCENE_GROUNDING_MODEL_NAME,
-  MIDSCENE_GROUNDING_OPENAI_API_KEY,
-  MIDSCENE_GROUNDING_OPENAI_BASE_URL,
-  MIDSCENE_GROUNDING_OPENAI_HTTP_PROXY,
-  MIDSCENE_GROUNDING_OPENAI_INIT_CONFIG_JSON,
-  MIDSCENE_GROUNDING_OPENAI_SOCKS_PROXY,
-  MIDSCENE_GROUNDING_OPENAI_USE_AZURE,
-  MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK,
-  MIDSCENE_GROUNDING_USE_AZURE_OPENAI,
-  MIDSCENE_GROUNDING_VL_MODE,
+  MIDSCENE_INSIGHT_MODEL_API_KEY,
+  MIDSCENE_INSIGHT_MODEL_BASE_URL,
+  MIDSCENE_INSIGHT_MODEL_HTTP_PROXY,
+  MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON,
+  MIDSCENE_INSIGHT_MODEL_NAME,
+  MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY,
+  MIDSCENE_INSIGHT_MODEL_TIMEOUT,
+  MIDSCENE_MODEL_API_KEY,
+  MIDSCENE_MODEL_BASE_URL,
+  MIDSCENE_MODEL_FAMILY,
+  MIDSCENE_MODEL_HTTP_PROXY,
+  MIDSCENE_MODEL_INIT_CONFIG_JSON,
   MIDSCENE_MODEL_NAME,
-  MIDSCENE_OPENAI_API_KEY,
-  MIDSCENE_OPENAI_BASE_URL,
+  MIDSCENE_MODEL_SOCKS_PROXY,
+  MIDSCENE_MODEL_TIMEOUT,
   MIDSCENE_OPENAI_HTTP_PROXY,
   MIDSCENE_OPENAI_INIT_CONFIG_JSON,
   MIDSCENE_OPENAI_SOCKS_PROXY,
-  MIDSCENE_OPENAI_USE_AZURE,
-  MIDSCENE_PLANNING_ANTHROPIC_API_KEY,
-  MIDSCENE_PLANNING_AZURE_OPENAI_API_VERSION,
-  MIDSCENE_PLANNING_AZURE_OPENAI_DEPLOYMENT,
-  MIDSCENE_PLANNING_AZURE_OPENAI_ENDPOINT,
-  MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON,
-  MIDSCENE_PLANNING_AZURE_OPENAI_KEY,
-  MIDSCENE_PLANNING_AZURE_OPENAI_SCOPE,
+  MIDSCENE_PLANNING_MODEL_API_KEY,
+  MIDSCENE_PLANNING_MODEL_BASE_URL,
+  MIDSCENE_PLANNING_MODEL_HTTP_PROXY,
+  MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON,
   MIDSCENE_PLANNING_MODEL_NAME,
-  MIDSCENE_PLANNING_OPENAI_API_KEY,
-  MIDSCENE_PLANNING_OPENAI_BASE_URL,
-  MIDSCENE_PLANNING_OPENAI_HTTP_PROXY,
-  MIDSCENE_PLANNING_OPENAI_INIT_CONFIG_JSON,
-  MIDSCENE_PLANNING_OPENAI_SOCKS_PROXY,
-  MIDSCENE_PLANNING_OPENAI_USE_AZURE,
-  MIDSCENE_PLANNING_USE_ANTHROPIC_SDK,
-  MIDSCENE_PLANNING_USE_AZURE_OPENAI,
-  MIDSCENE_PLANNING_VL_MODE,
-  MIDSCENE_USE_ANTHROPIC_SDK,
-  MIDSCENE_USE_AZURE_OPENAI,
-  MIDSCENE_VL_MODE,
-  MIDSCENE_VQA_ANTHROPIC_API_KEY,
-  MIDSCENE_VQA_AZURE_OPENAI_API_VERSION,
-  MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT,
-  MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT,
-  MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON,
-  MIDSCENE_VQA_AZURE_OPENAI_KEY,
-  MIDSCENE_VQA_AZURE_OPENAI_SCOPE,
-  // VQA
-  MIDSCENE_VQA_MODEL_NAME,
-  MIDSCENE_VQA_OPENAI_API_KEY,
-  MIDSCENE_VQA_OPENAI_BASE_URL,
-  MIDSCENE_VQA_OPENAI_HTTP_PROXY,
-  MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON,
-  MIDSCENE_VQA_OPENAI_SOCKS_PROXY,
-  MIDSCENE_VQA_OPENAI_USE_AZURE,
-  MIDSCENE_VQA_USE_ANTHROPIC_SDK,
-  MIDSCENE_VQA_USE_AZURE_OPENAI,
-  MIDSCENE_VQA_VL_MODE,
+  MIDSCENE_PLANNING_MODEL_SOCKS_PROXY,
+  MIDSCENE_PLANNING_MODEL_TIMEOUT,
   OPENAI_API_KEY,
   OPENAI_BASE_URL,
-  OPENAI_USE_AZURE,
 } from './types';
 
 interface IModelConfigKeys {
@@ -92,95 +42,36 @@ interface IModelConfigKeys {
   openaiApiKey: string;
   openaiExtraConfig: string;
   /**
-   * Azure
+   * Extra
    */
-  openaiUseAzureDeprecated: string;
-  useAzureOpenai: string;
-  azureOpenaiScope: string;
-  azureOpenaiKey: string;
-  azureOpenaiEndpoint: string;
-  azureOpenaiApiVersion: string;
-  azureOpenaiDeployment: string;
-  azureExtraConfig: string;
+  modelFamily: string;
   /**
-   * Anthropic
+   * Timeout
    */
-  useAnthropicSdk: string;
-  anthropicApiKey: string;
-  /**
-   * Extra
-   */
-  vlMode: string;
+  timeout: string;
 }
 
-export const VQA_MODEL_CONFIG_KEYS: IModelConfigKeys = {
-  modelName: MIDSCENE_VQA_MODEL_NAME,
+export const INSIGHT_MODEL_CONFIG_KEYS: IModelConfigKeys = {
+  modelName: MIDSCENE_INSIGHT_MODEL_NAME,
   /**
    * proxy
    */
-  socksProxy: MIDSCENE_VQA_OPENAI_SOCKS_PROXY,
-  httpProxy: MIDSCENE_VQA_OPENAI_HTTP_PROXY,
+  socksProxy: MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY,
+  httpProxy: MIDSCENE_INSIGHT_MODEL_HTTP_PROXY,
   /**
    * OpenAI
    */
-  openaiBaseURL: MIDSCENE_VQA_OPENAI_BASE_URL,
-  openaiApiKey: MIDSCENE_VQA_OPENAI_API_KEY,
-  openaiExtraConfig: MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON,
-  /**
-   * Azure
-   */
-  openaiUseAzureDeprecated: MIDSCENE_VQA_OPENAI_USE_AZURE,
-  useAzureOpenai: MIDSCENE_VQA_USE_AZURE_OPENAI,
-  azureOpenaiScope: MIDSCENE_VQA_AZURE_OPENAI_SCOPE,
-  azureOpenaiKey: MIDSCENE_VQA_AZURE_OPENAI_KEY,
-  azureOpenaiEndpoint: MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT,
-  azureOpenaiApiVersion: MIDSCENE_VQA_AZURE_OPENAI_API_VERSION,
-  azureOpenaiDeployment: MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT,
-  azureExtraConfig: MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON,
-  /**
-   * Anthropic
-   */
-  useAnthropicSdk: MIDSCENE_VQA_USE_ANTHROPIC_SDK,
-  anthropicApiKey: MIDSCENE_VQA_ANTHROPIC_API_KEY,
+  openaiBaseURL: MIDSCENE_INSIGHT_MODEL_BASE_URL,
+  openaiApiKey: MIDSCENE_INSIGHT_MODEL_API_KEY,
+  openaiExtraConfig: MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON,
   /**
    * Extra
    */
-  vlMode: MIDSCENE_VQA_VL_MODE,
-} as const;
-
-export const GROUNDING_MODEL_CONFIG_KEYS: IModelConfigKeys = {
-  modelName: MIDSCENE_GROUNDING_MODEL_NAME,
-  /**
-   * proxy
-   */
-  socksProxy: MIDSCENE_GROUNDING_OPENAI_SOCKS_PROXY,
-  httpProxy: MIDSCENE_GROUNDING_OPENAI_HTTP_PROXY,
-  /**
-   * OpenAI
-   */
-  openaiBaseURL: MIDSCENE_GROUNDING_OPENAI_BASE_URL,
-  openaiApiKey: MIDSCENE_GROUNDING_OPENAI_API_KEY,
-  openaiExtraConfig: MIDSCENE_GROUNDING_OPENAI_INIT_CONFIG_JSON,
-  /**
-   * Azure
-   */
-  openaiUseAzureDeprecated: MIDSCENE_GROUNDING_OPENAI_USE_AZURE,
-  useAzureOpenai: MIDSCENE_GROUNDING_USE_AZURE_OPENAI,
-  azureOpenaiScope: MIDSCENE_GROUNDING_AZURE_OPENAI_SCOPE,
-  azureOpenaiKey: MIDSCENE_GROUNDING_AZURE_OPENAI_KEY,
-  azureOpenaiEndpoint: MIDSCENE_GROUNDING_AZURE_OPENAI_ENDPOINT,
-  azureOpenaiApiVersion: MIDSCENE_GROUNDING_AZURE_OPENAI_API_VERSION,
-  azureOpenaiDeployment: MIDSCENE_GROUNDING_AZURE_OPENAI_DEPLOYMENT,
-  azureExtraConfig: MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON,
-  /**
-   * Anthropic
-   */
-  useAnthropicSdk: MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK,
-  anthropicApiKey: MIDSCENE_GROUNDING_ANTHROPIC_API_KEY,
+  modelFamily: 'THERE_IS_NO_MODEL_FAMILY_FOR_INSIGHT',
   /**
-   * Extra
+   * Timeout
    */
-  vlMode: MIDSCENE_GROUNDING_VL_MODE,
+  timeout: MIDSCENE_INSIGHT_MODEL_TIMEOUT,
 } as const;
 
 export const PLANNING_MODEL_CONFIG_KEYS: IModelConfigKeys = {
@@ -188,34 +79,22 @@ export const PLANNING_MODEL_CONFIG_KEYS: IModelConfigKeys = {
   /**
    * proxy
    */
-  socksProxy: MIDSCENE_PLANNING_OPENAI_SOCKS_PROXY,
-  httpProxy: MIDSCENE_PLANNING_OPENAI_HTTP_PROXY,
+  socksProxy: MIDSCENE_PLANNING_MODEL_SOCKS_PROXY,
+  httpProxy: MIDSCENE_PLANNING_MODEL_HTTP_PROXY,
   /**
    * OpenAI
    */
-  openaiBaseURL: MIDSCENE_PLANNING_OPENAI_BASE_URL,
-  openaiApiKey: MIDSCENE_PLANNING_OPENAI_API_KEY,
-  openaiExtraConfig: MIDSCENE_PLANNING_OPENAI_INIT_CONFIG_JSON,
+  openaiBaseURL: MIDSCENE_PLANNING_MODEL_BASE_URL,
+  openaiApiKey: MIDSCENE_PLANNING_MODEL_API_KEY,
+  openaiExtraConfig: MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON,
   /**
-   * Azure
-   */
-  openaiUseAzureDeprecated: MIDSCENE_PLANNING_OPENAI_USE_AZURE,
-  useAzureOpenai: MIDSCENE_PLANNING_USE_AZURE_OPENAI,
-  azureOpenaiScope: MIDSCENE_PLANNING_AZURE_OPENAI_SCOPE,
-  azureOpenaiKey: MIDSCENE_PLANNING_AZURE_OPENAI_KEY,
-  azureOpenaiEndpoint: MIDSCENE_PLANNING_AZURE_OPENAI_ENDPOINT,
-  azureOpenaiApiVersion: MIDSCENE_PLANNING_AZURE_OPENAI_API_VERSION,
-  azureOpenaiDeployment: MIDSCENE_PLANNING_AZURE_OPENAI_DEPLOYMENT,
-  azureExtraConfig: MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON,
-  /**
-   * Anthropic
+   * Extra
    */
-  useAnthropicSdk: MIDSCENE_PLANNING_USE_ANTHROPIC_SDK,
-  anthropicApiKey: MIDSCENE_PLANNING_ANTHROPIC_API_KEY,
+  modelFamily: 'THERE_IS_NO_MODEL_FAMILY_FOR_PLANNING',
   /**
-   * Extra
+   * Timeout
    */
-  vlMode: MIDSCENE_PLANNING_VL_MODE,
+  timeout: MIDSCENE_PLANNING_MODEL_TIMEOUT,
 } as const;
 
 // modelConfig return default
@@ -224,68 +103,44 @@ export const DEFAULT_MODEL_CONFIG_KEYS: IModelConfigKeys = {
   /**
    * proxy
    */
-  socksProxy: MIDSCENE_OPENAI_SOCKS_PROXY,
-  httpProxy: MIDSCENE_OPENAI_HTTP_PROXY,
+  socksProxy: MIDSCENE_MODEL_SOCKS_PROXY,
+  httpProxy: MIDSCENE_MODEL_HTTP_PROXY,
   /**
    * OpenAI
    */
-  openaiBaseURL: MIDSCENE_OPENAI_BASE_URL,
-  openaiApiKey: MIDSCENE_OPENAI_API_KEY,
-  openaiExtraConfig: MIDSCENE_OPENAI_INIT_CONFIG_JSON,
+  openaiBaseURL: MIDSCENE_MODEL_BASE_URL,
+  openaiApiKey: MIDSCENE_MODEL_API_KEY,
+  openaiExtraConfig: MIDSCENE_MODEL_INIT_CONFIG_JSON,
   /**
-   * Azure
-   */
-  openaiUseAzureDeprecated: MIDSCENE_OPENAI_USE_AZURE,
-  useAzureOpenai: MIDSCENE_USE_AZURE_OPENAI,
-  azureOpenaiScope: MIDSCENE_AZURE_OPENAI_SCOPE,
-  azureOpenaiKey: MIDSCENE_AZURE_OPENAI_KEY,
-  azureOpenaiEndpoint: MIDSCENE_AZURE_OPENAI_ENDPOINT,
-  azureOpenaiApiVersion: MIDSCENE_AZURE_OPENAI_API_VERSION,
-  azureOpenaiDeployment: MIDSCENE_AZURE_OPENAI_DEPLOYMENT,
-  azureExtraConfig: MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
-  /**
-   * Anthropic
+   * Extra
    */
-  useAnthropicSdk: MIDSCENE_USE_ANTHROPIC_SDK,
-  anthropicApiKey: MIDSCENE_ANTHROPIC_API_KEY,
+  modelFamily: MIDSCENE_MODEL_FAMILY,
   /**
-   * Extra
+   * Timeout
    */
-  vlMode: MIDSCENE_VL_MODE,
+  timeout: MIDSCENE_MODEL_TIMEOUT,
 } as const;
 
 // read from process.env
 export const DEFAULT_MODEL_CONFIG_KEYS_LEGACY: IModelConfigKeys = {
   modelName: MIDSCENE_MODEL_NAME,
   /**
-   * proxy
+   * proxy - Uses legacy MIDSCENE_OPENAI_* variables for backward compatibility
    */
   socksProxy: MIDSCENE_OPENAI_SOCKS_PROXY,
   httpProxy: MIDSCENE_OPENAI_HTTP_PROXY,
   /**
-   * OpenAI
+   * Model API - Uses legacy OPENAI_* variables for backward compatibility
    */
   openaiBaseURL: OPENAI_BASE_URL,
   openaiApiKey: OPENAI_API_KEY,
   openaiExtraConfig: MIDSCENE_OPENAI_INIT_CONFIG_JSON,
   /**
-   * Azure
-   */
-  openaiUseAzureDeprecated: OPENAI_USE_AZURE,
-  useAzureOpenai: MIDSCENE_USE_AZURE_OPENAI,
-  azureOpenaiScope: MIDSCENE_AZURE_OPENAI_SCOPE,
-  azureOpenaiKey: AZURE_OPENAI_KEY,
-  azureOpenaiEndpoint: AZURE_OPENAI_ENDPOINT,
-  azureOpenaiApiVersion: AZURE_OPENAI_API_VERSION,
-  azureOpenaiDeployment: AZURE_OPENAI_DEPLOYMENT,
-  azureExtraConfig: MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
-  /**
-   * Anthropic
+   * Extra
    */
-  useAnthropicSdk: MIDSCENE_USE_ANTHROPIC_SDK,
-  anthropicApiKey: ANTHROPIC_API_KEY,
+  modelFamily: 'DEFAULT_MODEL_CONFIG_KEYS has no modelFamily key',
   /**
-   * Extra
+   * Timeout - use the new key for legacy mode too
    */
-  vlMode: 'DEFAULT_MODEL_CONFIG_KEYS has no vlMode key',
+  timeout: MIDSCENE_MODEL_TIMEOUT,
 } as const;