@midscene/shared 0.30.10 → 1.0.0

package/src/mcp/types.ts

@@ -0,0 +1,108 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { z } from 'zod';
+
+// Avoid circular dependency: don't import from @midscene/core
+// Instead, use generic types that will be provided by implementation
+
+/**
+ * Default timeout constants for app loading verification
+ */
+export const defaultAppLoadingTimeoutMs = 10000;
+export const defaultAppLoadingCheckIntervalMs = 2000;
+
+/**
+ * Content item types for tool results (MCP compatible)
+ */
+export type ToolResultContent =
+  | { type: 'text'; text: string }
+  | { type: 'image'; data: string; mimeType: string }
+  | { type: 'audio'; data: string; mimeType: string }
+  | {
+      type: 'resource';
+      resource:
+        | { text: string; uri: string; mimeType?: string }
+        | { uri: string; blob: string; mimeType?: string };
+    };
+
+/**
+ * Result type for tool execution (MCP compatible)
+ */
+export interface ToolResult {
+  [x: string]: unknown;
+  content: ToolResultContent[];
+  isError?: boolean;
+  _meta?: Record<string, unknown>;
+}
+
+/**
+ * Tool handler function type
+ * Takes parsed arguments and returns a tool result
+ */
+export type ToolHandler<T = Record<string, unknown>> = (
+  args: T,
+) => Promise<ToolResult>;
+
+/**
+ * Tool schema type using Zod
+ */
+export type ToolSchema = Record<string, z.ZodTypeAny>;
+
+/**
+ * Tool definition for MCP server
+ */
+export interface ToolDefinition<T = Record<string, unknown>> {
+  name: string;
+  description: string;
+  schema: ToolSchema;
+  handler: ToolHandler<T>;
+  autoDestroy?: boolean;
+}
+
+/**
+ * Action space item definition
+ * Note: Intentionally no index signature to maintain compatibility with DeviceAction
+ */
+export interface ActionSpaceItem {
+  name: string;
+  description?: string;
+  args?: Record<string, unknown>;
+  paramSchema?: z.ZodTypeAny;
+}
+
+/**
+ * Base agent interface
+ * Represents a platform-specific agent (Android, iOS, Web)
+ * Note: Return types use `unknown` for compatibility with platform-specific implementations
+ */
+export interface BaseAgent {
+  getActionSpace(): Promise<ActionSpaceItem[]>;
+  destroy?(): Promise<void>;
+  page?: {
+    screenshotBase64(): Promise<string>;
+  };
+  aiAction?: (
+    description: string,
+    params?: Record<string, unknown>,
+  ) => Promise<unknown>;
+  aiWaitFor?: (
+    assertion: string,
+    options: Record<string, unknown>,
+  ) => Promise<unknown>;
+}
+
+/**
+ * Base device interface for temporary device instances
+ */
+export interface BaseDevice {
+  actionSpace(): ActionSpaceItem[];
+  destroy?(): Promise<void>;
+}
+
+/**
+ * Interface for platform-specific MCP tools manager
+ */
+export interface IMidsceneTools {
+  attachToServer(server: McpServer): void;
+  initTools(): Promise<void>;
+  closeBrowser?(): Promise<void>;
+}

package/src/node/fs.ts

@@ -75,7 +75,7 @@ export async function getExtraReturnLogic(tree = false) {
     return null;
   }
 
-  const elementInfosScriptContent = `${getElementInfosScriptContent()}midscene_element_inspector.setNodeHashCacheListOnWindow();`;
+  const elementInfosScriptContent = `${getElementInfosScriptContent()};`;
 
   if (tree) {
     return `${elementInfosScriptContent}midscene_element_inspector.webExtractNodeTree()`;

package/src/types/index.ts

@@ -17,7 +17,7 @@ export type Rect = Point & Size & { zoom?: number };
 export abstract class BaseElement {
   abstract id: string;
 
-  abstract indexId?: number; // markerId for web
+  // abstract indexId?: number; // markerId for web
 
   abstract attributes: {
     nodeType: NodeType;
@@ -30,7 +30,7 @@ export abstract class BaseElement {
 
   abstract center: [number, number];
 
-  abstract xpaths?: string[];
+  // abstract xpaths?: string[];
 
   abstract isVisible: boolean;
 }
@@ -45,3 +45,9 @@ export interface ElementTreeNode<
 export interface WebElementInfo extends ElementInfo {
   zoom: number;
 }
+
+export type LocateResultElement = {
+  description: string; // the description of the element
+  center: [number, number];
+  rect: Rect;
+};

package/src/utils.ts

@@ -123,5 +123,5 @@ export const antiEscapeScriptTag = (html: string) => {
 
 export function replaceIllegalPathCharsAndSpace(str: string) {
   // Only replace characters that are illegal in filenames, but preserve path separators
-  return str.replace(/[:*?"<>| ]/g, '-');
+  return str.replace(/[:*?"<>|# ]/g, '-');
 }

package/src/zod-schema-utils.ts

@@ -0,0 +1,133 @@
+import type { z } from 'zod';
+
+/**
+ * Recursively unwrap optional, nullable, default, and effects wrapper types
+ * to get the actual inner Zod type
+ */
+export function unwrapZodField(field: unknown): unknown {
+  const f = field as {
+    _def?: { typeName?: string; innerType?: unknown; schema?: unknown };
+  };
+  if (!f._def) return f;
+
+  const typeName = f._def.typeName;
+
+  // Handle wrapper types that have innerType
+  if (
+    typeName === 'ZodOptional' ||
+    typeName === 'ZodNullable' ||
+    typeName === 'ZodDefault'
+  ) {
+    return unwrapZodField(f._def.innerType);
+  }
+
+  // Handle ZodEffects (transformations, refinements, preprocessors)
+  if (typeName === 'ZodEffects') {
+    if (f._def.schema) {
+      return unwrapZodField(f._def.schema);
+    }
+  }
+
+  return f;
+}
+
+/**
+ * Check if a field is a Midscene locator field
+ * Checks for either:
+ * 1. midscene_location_field_flag in shape (result schema)
+ * 2. prompt field in shape (input schema)
+ */
+export function isMidsceneLocatorField(field: unknown): boolean {
+  const actualField = unwrapZodField(field) as {
+    _def?: { typeName?: string; shape?: () => Record<string, unknown> };
+  };
+
+  if (actualField._def?.typeName === 'ZodObject') {
+    const shape = actualField._def.shape?.();
+    if (shape) {
+      // Method 1: Check for the location field flag (for result schema)
+      if ('midscene_location_field_flag' in shape) {
+        return true;
+      }
+      // Method 2: Check if it's the input schema by checking for 'prompt' field
+      if ('prompt' in shape && shape.prompt) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+/**
+ * Get type name string from a Zod schema field
+ * @param field - Zod schema field
+ * @param locatorTypeDescription - Optional description for MidsceneLocation fields (used by core)
+ */
+export function getZodTypeName(
+  field: unknown,
+  locatorTypeDescription?: string,
+): string {
+  const actualField = unwrapZodField(field) as {
+    _def?: { typeName?: string; values?: unknown[]; options?: unknown[] };
+  };
+  const fieldTypeName = actualField._def?.typeName;
+
+  if (fieldTypeName === 'ZodString') return 'string';
+  if (fieldTypeName === 'ZodNumber') return 'number';
+  if (fieldTypeName === 'ZodBoolean') return 'boolean';
+  if (fieldTypeName === 'ZodArray') return 'array';
+  if (fieldTypeName === 'ZodObject') {
+    // Check if this is a Midscene locator field
+    if (isMidsceneLocatorField(actualField)) {
+      return locatorTypeDescription || 'object';
+    }
+    return 'object';
+  }
+  if (fieldTypeName === 'ZodEnum') {
+    const values =
+      (actualField._def?.values as unknown[] | undefined)
+        ?.map((option: unknown) => String(`'${option}'`))
+        .join(', ') ?? 'enum';
+    return `enum(${values})`;
+  }
+  // Handle ZodUnion by listing all option types
+  if (fieldTypeName === 'ZodUnion') {
+    const options = actualField._def?.options as unknown[] | undefined;
+    if (options && options.length > 0) {
+      const types = options.map((opt: unknown) =>
+        getZodTypeName(opt, locatorTypeDescription),
+      );
+      return types.join(' | ');
+    }
+    return 'union';
+  }
+
+  return 'unknown';
+}
+
+/**
+ * Get description from a Zod schema field
+ */
+export function getZodDescription(field: z.ZodTypeAny): string | null {
+  // Check for direct description on the original field (wrapper may have description)
+  if ('description' in field) {
+    return (field as { description?: string }).description || null;
+  }
+
+  const actualField = unwrapZodField(field) as {
+    description?: string;
+    _def?: { typeName?: string; shape?: () => Record<string, unknown> };
+  };
+
+  // Check for description on the unwrapped field
+  if ('description' in actualField) {
+    return actualField.description || null;
+  }
+
+  // Check for MidsceneLocation fields and add description
+  if (isMidsceneLocatorField(actualField)) {
+    return 'Location information for the target element';
+  }
+
+  return null;
+}

package/dist/es/env/decide-model-config.mjs

@@ -1,172 +0,0 @@
-import { DEFAULT_MODEL_CONFIG_KEYS, DEFAULT_MODEL_CONFIG_KEYS_LEGACY, GROUNDING_MODEL_CONFIG_KEYS, PLANNING_MODEL_CONFIG_KEYS, VQA_MODEL_CONFIG_KEYS } from "./constants.mjs";
-import { getDebug } from "../logger.mjs";
-import { assert } from "../utils.mjs";
-import { createAssert, maskConfig, parseJson } from "./helper.mjs";
-import { initDebugConfig } from "./init-debug.mjs";
-import { parseVlModeAndUiTarsFromGlobalConfig, parseVlModeAndUiTarsModelVersionFromRawValue } from "./parse.mjs";
-const KEYS_MAP = {
-    VQA: VQA_MODEL_CONFIG_KEYS,
-    grounding: GROUNDING_MODEL_CONFIG_KEYS,
-    planning: PLANNING_MODEL_CONFIG_KEYS,
-    default: DEFAULT_MODEL_CONFIG_KEYS
-};
-const decideOpenaiSdkConfig = ({ keys, provider, valueAssert })=>{
-    initDebugConfig();
-    const debugLog = getDebug('ai:config');
-    const socksProxy = provider[keys.socksProxy];
-    const httpProxy = provider[keys.httpProxy];
-    const vlMode = provider[keys.vlMode];
-    debugLog('enter decideOpenaiSdkConfig with keys:', keys);
-    if (provider[keys.openaiUseAzureDeprecated]) {
-        debugLog(`provider has ${keys.openaiUseAzureDeprecated} with value${provider[keys.openaiUseAzureDeprecated]}`);
-        const openaiBaseURL = provider[keys.openaiBaseURL];
-        const openaiApiKey = provider[keys.openaiApiKey];
-        const openaiExtraConfig = parseJson(keys.openaiExtraConfig, provider[keys.openaiExtraConfig]);
-        valueAssert(openaiApiKey, keys.openaiApiKey, keys.openaiUseAzureDeprecated);
-        return {
-            socksProxy,
-            httpProxy,
-            vlModeRaw: vlMode,
-            openaiUseAzureDeprecated: true,
-            openaiApiKey,
-            openaiBaseURL,
-            openaiExtraConfig
-        };
-    }
-    if (provider[keys.useAzureOpenai]) {
-        debugLog(`provider has ${keys.useAzureOpenai} with value ${provider[keys.useAzureOpenai]}`);
-        const azureOpenaiScope = provider[keys.azureOpenaiScope];
-        const azureOpenaiKey = provider[keys.azureOpenaiKey];
-        const azureOpenaiEndpoint = provider[keys.azureOpenaiEndpoint];
-        const azureOpenaiDeployment = provider[keys.azureOpenaiDeployment];
-        const azureOpenaiApiVersion = provider[keys.azureOpenaiApiVersion];
-        const azureExtraConfig = parseJson(keys.azureExtraConfig, provider[keys.azureExtraConfig]);
-        const openaiExtraConfig = parseJson(keys.openaiExtraConfig, provider[keys.openaiExtraConfig]);
-        valueAssert(azureOpenaiKey, keys.azureOpenaiKey, keys.useAzureOpenai);
-        return {
-            socksProxy,
-            httpProxy,
-            vlModeRaw: vlMode,
-            useAzureOpenai: true,
-            azureOpenaiScope,
-            azureOpenaiKey,
-            azureOpenaiEndpoint,
-            azureOpenaiDeployment,
-            azureOpenaiApiVersion,
-            azureExtraConfig,
-            openaiExtraConfig
-        };
-    }
-    if (provider[keys.useAnthropicSdk]) {
-        debugLog(`provider has ${keys.useAnthropicSdk} with value ${provider[keys.useAnthropicSdk]}`);
-        const anthropicApiKey = provider[keys.anthropicApiKey];
-        valueAssert(anthropicApiKey, keys.anthropicApiKey, keys.useAnthropicSdk);
-        return {
-            socksProxy,
-            httpProxy,
-            useAnthropicSdk: true,
-            anthropicApiKey
-        };
-    }
-    {
-        debugLog('provider has no specific model SDK declared');
-        const openaiBaseURL = provider[keys.openaiBaseURL];
-        const openaiApiKey = provider[keys.openaiApiKey];
-        const openaiExtraConfig = parseJson(keys.openaiExtraConfig, provider[keys.openaiExtraConfig]);
-        valueAssert(openaiApiKey, keys.openaiApiKey);
-        return {
-            socksProxy,
-            httpProxy,
-            vlModeRaw: vlMode,
-            openaiBaseURL,
-            openaiApiKey,
-            openaiExtraConfig
-        };
-    }
-};
-const getModelDescription = (vlMode, uiTarsVersion)=>{
-    if (vlMode) if (uiTarsVersion) return `UI-TARS=${uiTarsVersion}`;
-    else return `${vlMode} mode`;
-    return '';
-};
-const decideModelConfigFromIntentConfig = (intent, intentConfig)=>{
-    const debugLog = getDebug('ai:config');
-    debugLog('decideModelConfig base on agent.modelConfig()');
-    const keysForFn = KEYS_MAP[intent];
-    const candidateModelNameFromConfig = intentConfig[keysForFn.modelName];
-    debugLog('Got modelName from modelConfigFn', candidateModelNameFromConfig);
-    const chosenKeys = (()=>{
-        if (candidateModelNameFromConfig) {
-            debugLog('query modelConfig from fn by intent got corresponding modelName, will get other corresponding keys');
-            return keysForFn;
-        }
-        debugLog('query modelConfig from fn by intent got no corresponding modelName, will get other keys by default');
-        assert(intentConfig[DEFAULT_MODEL_CONFIG_KEYS.modelName], `The return value of agent.modelConfig do not have a valid value with key ${DEFAULT_MODEL_CONFIG_KEYS.modelName}.`);
-        return DEFAULT_MODEL_CONFIG_KEYS;
-    })();
-    const result = decideOpenaiSdkConfig({
-        keys: chosenKeys,
-        provider: intentConfig,
-        valueAssert: createAssert(chosenKeys.modelName, 'modelConfig', candidateModelNameFromConfig)
-    });
-    const { vlMode, uiTarsVersion } = parseVlModeAndUiTarsModelVersionFromRawValue(result.vlModeRaw);
-    const modelDescription = getModelDescription(vlMode, uiTarsVersion);
-    const finalResult = {
-        ...result,
-        modelName: intentConfig[chosenKeys.modelName],
-        vlMode,
-        uiTarsModelVersion: uiTarsVersion,
-        modelDescription,
-        from: 'modelConfig',
-        intent
-    };
-    debugLog(`decideModelConfig result by agent.modelConfig() with intent ${intent}:`, maskConfig(finalResult));
-    return finalResult;
-};
-const decideModelConfigFromEnv = (intent, allEnvConfig)=>{
-    initDebugConfig();
-    const debugLog = getDebug('ai:config');
-    const keysForEnv = 'default' === intent ? DEFAULT_MODEL_CONFIG_KEYS_LEGACY : KEYS_MAP[intent];
-    if ('default' !== intent && allEnvConfig[keysForEnv.modelName]) {
-        const modelName = allEnvConfig[keysForEnv.modelName];
-        debugLog(`Got intent ${intent} corresponding modelName ${modelName} by key ${keysForEnv.modelName} from globalConfig, will get other config by intent.`);
-        const result = decideOpenaiSdkConfig({
-            keys: keysForEnv,
-            provider: allEnvConfig,
-            valueAssert: createAssert(keysForEnv.modelName, 'process.env', modelName)
-        });
-        const { vlMode, uiTarsVersion } = parseVlModeAndUiTarsModelVersionFromRawValue(result.vlModeRaw);
-        const modelDescription = getModelDescription(vlMode, uiTarsVersion);
-        const finalResult = {
-            ...result,
-            modelName,
-            vlMode,
-            uiTarsModelVersion: uiTarsVersion,
-            modelDescription,
-            from: 'env',
-            intent
-        };
-        debugLog(`decideModelConfig result by process.env with intent ${intent}:`, maskConfig(finalResult));
-        return finalResult;
-    }
-    debugLog(`decideModelConfig as legacy logic with intent ${intent}.`);
-    const result = decideOpenaiSdkConfig({
-        keys: DEFAULT_MODEL_CONFIG_KEYS_LEGACY,
-        provider: allEnvConfig,
-        valueAssert: createAssert(DEFAULT_MODEL_CONFIG_KEYS_LEGACY.modelName, 'process.env')
-    });
-    const { vlMode, uiTarsVersion } = parseVlModeAndUiTarsFromGlobalConfig(allEnvConfig);
-    const modelDescription = getModelDescription(vlMode, uiTarsVersion);
-    const finalResult = {
-        ...result,
-        modelName: allEnvConfig[DEFAULT_MODEL_CONFIG_KEYS_LEGACY.modelName] || 'gpt-4o',
-        vlMode,
-        uiTarsModelVersion: uiTarsVersion,
-        modelDescription,
-        from: 'legacy-env',
-        intent
-    };
-    debugLog(`decideModelConfig result by legacy logic with intent ${intent}:`, maskConfig(finalResult));
-    return finalResult;
-};
-export { decideModelConfigFromEnv, decideModelConfigFromIntentConfig, decideOpenaiSdkConfig };

package/dist/es/env/parse.mjs

@@ -1,69 +0,0 @@
-import { MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN3_VL, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, UITarsModelVersion, VL_MODE_RAW_VALID_VALUES } from "./types.mjs";
-const parseVlModeAndUiTarsModelVersionFromRawValue = (vlModeRaw)=>{
-    if (!vlModeRaw) return {
-        vlMode: void 0,
-        uiTarsVersion: void 0
-    };
-    if (!VL_MODE_RAW_VALID_VALUES.includes(vlModeRaw)) throw new Error(`the value ${vlModeRaw} is not a valid VL_MODE value, must be one of ${VL_MODE_RAW_VALID_VALUES}`);
-    const raw = vlModeRaw;
-    if ('vlm-ui-tars' === raw) return {
-        vlMode: 'vlm-ui-tars',
-        uiTarsVersion: UITarsModelVersion.V1_0
-    };
-    if ('vlm-ui-tars-doubao' === raw || 'vlm-ui-tars-doubao-1.5' === raw) return {
-        vlMode: 'vlm-ui-tars',
-        uiTarsVersion: UITarsModelVersion.DOUBAO_1_5_20B
-    };
-    return {
-        vlMode: raw,
-        uiTarsVersion: void 0
-    };
-};
-const parseVlModeAndUiTarsFromGlobalConfig = (provider)=>{
-    const isDoubao = provider[MIDSCENE_USE_DOUBAO_VISION];
-    const isQwen = provider[MIDSCENE_USE_QWEN_VL];
-    const isQwen3 = provider[MIDSCENE_USE_QWEN3_VL];
-    const isUiTars = provider[MIDSCENE_USE_VLM_UI_TARS];
-    const isGemini = provider[MIDSCENE_USE_GEMINI];
-    const enabledModes = [
-        isDoubao && MIDSCENE_USE_DOUBAO_VISION,
-        isQwen && MIDSCENE_USE_QWEN_VL,
-        isQwen3 && MIDSCENE_USE_QWEN3_VL,
-        isUiTars && MIDSCENE_USE_VLM_UI_TARS,
-        isGemini && MIDSCENE_USE_GEMINI
-    ].filter(Boolean);
-    if (enabledModes.length > 1) throw new Error(`Only one vision mode can be enabled at a time. Currently enabled modes: ${enabledModes.join(', ')}. Please disable all but one mode.`);
-    if (isQwen3) return {
-        vlMode: 'qwen3-vl',
-        uiTarsVersion: void 0
-    };
-    if (isQwen) return {
-        vlMode: 'qwen-vl',
-        uiTarsVersion: void 0
-    };
-    if (isDoubao) return {
-        vlMode: 'doubao-vision',
-        uiTarsVersion: void 0
-    };
-    if (isGemini) return {
-        vlMode: 'gemini',
-        uiTarsVersion: void 0
-    };
-    if (isUiTars) if ('1' === isUiTars) return {
-        vlMode: 'vlm-ui-tars',
-        uiTarsVersion: UITarsModelVersion.V1_0
-    };
-    else if ('DOUBAO' === isUiTars || 'DOUBAO-1.5' === isUiTars) return {
-        vlMode: 'vlm-ui-tars',
-        uiTarsVersion: UITarsModelVersion.DOUBAO_1_5_20B
-    };
-    else return {
-        vlMode: 'vlm-ui-tars',
-        uiTarsVersion: `${isUiTars}`
-    };
-    return {
-        vlMode: void 0,
-        uiTarsVersion: void 0
-    };
-};
-export { parseVlModeAndUiTarsFromGlobalConfig, parseVlModeAndUiTarsModelVersionFromRawValue };