@aiscene/shared 8.0.2 → 8.0.3

package/src/env/parse-model-config.ts

@@ -24,10 +24,26 @@ import {
 } from './types';
 
 import { getDebug } from '../logger';
+
 import { assert } from '../utils';
 import { maskConfig, parseJson } from './helper';
 import { initDebugConfig } from './init-debug';
 
+declare const __VERSION__: string | undefined;
+
+const MODEL_CONFIG_DOC_URL = 'https://midscenejs.com/model-common-config.html';
+
+const getCurrentVersion = (): string => {
+  if (typeof __VERSION__ !== 'undefined' && __VERSION__) {
+    return __VERSION__;
+  }
+
+  return 'unknown';
+};
+
+const getInvalidModelFamilyMessage = (modelFamily: TModelFamily): string =>
+  `Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}. Current version v${getCurrentVersion()} accepts the following model families: ${MODEL_FAMILY_VALUES.join(', ')}. You can also visit ${MODEL_CONFIG_DOC_URL} for the latest configuration information.`;
+
 type TModelConfigKeys =
   | typeof INSIGHT_MODEL_CONFIG_KEYS
   | typeof PLANNING_MODEL_CONFIG_KEYS
@@ -70,7 +86,7 @@ export const getUITarsModelVersion = (
  */
 export const validateModelFamily = (modelFamily?: TModelFamily): void => {
   if (modelFamily && !MODEL_FAMILY_VALUES.includes(modelFamily as any)) {
-    throw new Error(`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}`);
+    throw new Error(getInvalidModelFamilyMessage(modelFamily));
   }
 };
 
@@ -228,6 +244,7 @@ export const parseOpenaiSdkConfig = ({
     modelName: modelName!,
     modelDescription,
     intent: '-' as any,
+    slot: '-' as any,
     timeout: provider[keys.timeout]
       ? Number(provider[keys.timeout])
       : undefined,
@@ -292,6 +309,7 @@ export const decideModelConfigFromIntentConfig = (
     useLegacyLogic: intent === 'default',
   });
   finalResult.intent = intent;
+  finalResult.slot = intent;
 
   debugLog(
     'decideModelConfig result by agent.modelConfig() with intent',

package/src/env/types.ts

@@ -69,6 +69,8 @@ export const MIDSCENE_ANDROID_IME_STRATEGY = 'MIDSCENE_ANDROID_IME_STRATEGY';
 
 export const MIDSCENE_IOS_DEVICE_UDID = 'MIDSCENE_IOS_DEVICE_UDID';
 export const MIDSCENE_IOS_SIMULATOR_UDID = 'MIDSCENE_IOS_SIMULATOR_UDID';
+export const MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE =
+  'MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE';
 
 export const MIDSCENE_CACHE = 'MIDSCENE_CACHE';
 export const MIDSCENE_USE_VLM_UI_TARS = 'MIDSCENE_USE_VLM_UI_TARS';
@@ -425,7 +427,7 @@ export enum UITarsModelVersion {
 
 /**
  * Callback to create custom OpenAI client instance
- * @param config - Resolved model configuration including apiKey, baseURL, modelName, intent, etc.
+ * @param config - Resolved model configuration including apiKey, baseURL, modelName, intent, slot, etc.
  * @returns OpenAI client instance (can be wrapped with langsmith, langfuse, etc.)
  *
  * Note: Wrapper functions like langsmith's wrapOpenAI() return the same OpenAI instance
@@ -519,9 +521,16 @@ export interface IModelConfig {
   uiTarsModelVersion?: UITarsModelVersion;
   modelDescription: string;
   /**
-   * original intent from the config
+   * The semantic intent this config is requested for.
+   * For example, getModelConfig('planning') always returns intent === 'planning'.
    */
   intent: TIntent;
+  /**
+   * The model-config slot this config was resolved from.
+   * For example, getModelConfig('planning') may resolve from slot === 'default'
+   * when MIDSCENE_PLANNING_MODEL_NAME is not configured.
+   */
+  slot: TIntent;
   /**
    * Custom OpenAI client factory function
    *

package/src/env/utils.ts

@@ -13,56 +13,6 @@ export const globalConfigManager = new GlobalConfigManager();
 globalConfigManager.registerModelConfigManager(globalModelConfigManager);
 globalModelConfigManager.registerGlobalConfigManager(globalConfigManager);
 
-/**
- * Interface for devices that support getTimestamp method.
- * This is a minimal interface to avoid circular dependencies with @midscene/core.
- */
-export interface DeviceWithTimestamp {
-  getTimestamp?: () => Promise<number>;
-}
-
-/**
- * Get the current timestamp, optionally from the target device.
- *
- * When useDeviceTimestamp is enabled and a device with getTimestamp is provided,
- * this function will return the device's time. Otherwise, it returns the system time.
- *
- * This is useful when:
- * - Testing on devices with different time zones
- * - Debugging time-sensitive features
- * - The system clock and device clock are not synchronized
- *
- * @param device Optional device interface that supports getTimestamp
- * @param useDeviceTimestamp Whether to use device timestamp (from agent config)
- * @returns Timestamp in milliseconds
- *
- * @example
- * // Without device - always returns system time
- * const systemTime = await getCurrentTime();
- *
- * @example
- * // With device and config enabled - returns device time
- * const deviceTime = await getCurrentTime(androidDevice, true);
- */
-export async function getCurrentTime(
-  device?: DeviceWithTimestamp,
-  useDeviceTimestamp?: boolean,
-): Promise<number> {
-  if (useDeviceTimestamp && device?.getTimestamp) {
-    try {
-      return await device.getTimestamp();
-    } catch (error) {
-      // Fall back to system time if device time retrieval fails
-      console.warn(
-        `Failed to get device time, falling back to system time: ${error}`,
-      );
-      return Date.now();
-    }
-  }
-
-  return Date.now();
-}
-
 export const getPreferredLanguage = () => {
   const prefer = globalConfigManager.getEnvConfigValue(
     MIDSCENE_PREFERRED_LANGUAGE,

package/src/extractor/index.ts

@@ -43,6 +43,6 @@ export {
   getElementXpath,
 } from './locator';
 
-export { generateElementByRect } from './dom-util';
+export { generateElementByPoint, generateElementByRect } from './dom-util';
 
 export { isNotContainerElement } from './dom-util';

package/src/img/index.ts

@@ -3,6 +3,8 @@ export {
   isValidPNGImageBuffer,
   isValidJPEGImageBuffer,
   isValidImageBuffer,
+  validateScreenshotBuffer,
+  type ValidateScreenshotBufferOptions,
 } from './info';
 export {
   resizeAndConvertImgBuffer,

package/src/img/info.ts

@@ -14,52 +14,16 @@ export interface ImageInfo extends Size {}
 export async function imageInfoOfBase64(
   imageBase64: string,
 ): Promise<ImageInfo> {
-  // Validate input before passing to Photon
-  if (!imageBase64 || typeof imageBase64 !== 'string') {
-    throw new Error('Invalid image base64: data is empty or not a string');
-  }
-
+  const { PhotonImage } = await getPhoton();
   const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
-
-  // Check if base64 data is valid after stripping data URL prefix
-  if (!base64Data || base64Data.length === 0) {
-    throw new Error('Invalid image base64: no data after stripping prefix');
-  }
-
-  // Basic validation: base64 should only contain valid characters
-  // (A-Z, a-z, 0-9, +, /, and = for padding)
-  if (!/^[A-Za-z0-9+/]*={0,2}$/.test(base64Data)) {
-    throw new Error(
-      'Invalid image base64: contains invalid characters',
-    );
-  }
-
-  // Check minimum length for a valid image (at least a few bytes)
-  if (base64Data.length < 4) {
-    throw new Error(
-      `Invalid image base64: data too short (${base64Data.length} characters)`,
-    );
-  }
-
-  try {
-    const { PhotonImage } = await getPhoton();
-    // Support both sync (Photon) and async (Canvas fallback) versions
-    const result = PhotonImage.new_from_base64(base64Data);
-    const image = result instanceof Promise ? await result : result;
-    const width = image.get_width();
-    const height = image.get_height();
-    image.free();
-    assert(width && height, 'Invalid image: cannot get width or height');
-    return { width, height };
-  } catch (error) {
-    // Provide more helpful error message for Photon failures
-    if (error instanceof Error && error.message.includes('unreachable')) {
-      throw new Error(
-        `Failed to decode image: invalid or corrupted image data. Original error: ${error.message}`,
-      );
-    }
-    throw error;
-  }
+  // Support both sync (Photon) and async (Canvas fallback) versions
+  const result = PhotonImage.new_from_base64(base64Data);
+  const image = result instanceof Promise ? await result : result;
+  const width = image.get_width();
+  const height = image.get_height();
+  image.free();
+  assert(width && height, 'Invalid image: cannot get width or height');
+  return { width, height };
 }
 
 /**
@@ -109,3 +73,30 @@ export function isValidJPEGImageBuffer(buffer: Buffer): boolean {
 export function isValidImageBuffer(buffer: Buffer): boolean {
   return isValidPNGImageBuffer(buffer) || isValidJPEGImageBuffer(buffer);
 }
+
+export interface ValidateScreenshotBufferOptions {
+  label: string;
+  minBufferSize?: number;
+}
+
+export function validateScreenshotBuffer(
+  screenshotBuffer: Buffer | undefined,
+  { label, minBufferSize = 0 }: ValidateScreenshotBufferOptions,
+): asserts screenshotBuffer is Buffer {
+  const bufferSize = screenshotBuffer?.length ?? 0;
+  if (!screenshotBuffer || bufferSize === 0) {
+    throw new Error(
+      `${label} validation failed: buffer size ${bufferSize} bytes`,
+    );
+  }
+
+  if (!isValidImageBuffer(screenshotBuffer)) {
+    throw new Error(`${label} buffer has invalid image format`);
+  }
+
+  if (minBufferSize > 0 && bufferSize < minBufferSize) {
+    throw new Error(
+      `${label} validation failed: buffer size ${bufferSize} bytes (minimum: ${minBufferSize})`,
+    );
+  }
+}

package/src/key-alias-utils.ts

@@ -0,0 +1,23 @@
+/**
+ * Internal-only helpers for CLI/MCP argument key aliasing.
+ * Not re-exported from the package entry point — keep consumers within
+ * `cli/` and `mcp/`.
+ */
+
+export function kebabToCamel(str: string): string {
+  return str.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase());
+}
+
+export function camelToKebab(str: string): string {
+  return str
+    .replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`)
+    .replace(/^-/, '');
+}
+
+export function getKeyAliases(key: string): string[] {
+  return [...new Set([key, kebabToCamel(key), camelToKebab(key)])];
+}
+
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}

package/src/mcp/base-server.ts

@@ -1,5 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import type { ParseArgsConfig } from 'node:util';
+import { setIsMcp } from '@aiscene/shared/utils';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
@@ -8,7 +9,7 @@ import express, {
   type Request,
   type Response,
 } from 'express';
-import { setIsMcp } from '../utils';
+import { getErrorMessage } from './error-formatter';
 import type { IMidsceneTools } from './types';
 
 export interface BaseMCPServerConfig {
@@ -119,7 +120,7 @@ export abstract class BaseMCPServer {
     try {
       await this.toolsManager.initTools();
     } catch (error: unknown) {
-      const message = error instanceof Error ? error.message : String(error);
+      const message = getErrorMessage(error);
       console.error(`Failed to initialize tools: ${message}`);
       console.error('Tools will be initialized on first use');
     }
@@ -160,7 +161,7 @@ export abstract class BaseMCPServer {
     try {
       await this.mcpServer.connect(transport);
     } catch (error: unknown) {
-      const message = error instanceof Error ? error.message : String(error);
+      const message = getErrorMessage(error);
       console.error(`Failed to connect MCP stdio transport: ${message}`);
       throw new Error(`Failed to initialize MCP stdio transport: ${message}`);
     }
@@ -282,7 +283,7 @@ export abstract class BaseMCPServer {
             .json({ error: 'Invalid session or GET without session' });
         }
       } catch (error: unknown) {
-        const message = error instanceof Error ? error.message : String(error);
+        const message = getErrorMessage(error);
         const duration = Date.now() - startTime;
         console.error(
           `[${new Date().toISOString()}] [${requestId}] MCP request error after ${duration}ms: ${message}`,
@@ -336,8 +337,7 @@ export abstract class BaseMCPServer {
           try {
             await session.transport.close();
           } catch (error: unknown) {
-            const message =
-              error instanceof Error ? error.message : String(error);
+            const message = getErrorMessage(error);
             console.error(
               `Failed to close session ${session.transport.sessionId}: ${message}`,
             );
@@ -390,7 +390,7 @@ export abstract class BaseMCPServer {
     try {
       await this.mcpServer.connect(transport);
     } catch (error: unknown) {
-      const message = error instanceof Error ? error.message : String(error);
+      const message = getErrorMessage(error);
       console.error(
         `[${new Date().toISOString()}] Failed to connect MCP transport: ${message}`,
       );
@@ -425,8 +425,7 @@ export abstract class BaseMCPServer {
               `[${new Date().toISOString()}] Session ${sid} cleaned up due to inactivity (remaining: ${sessions.size})`,
             );
           } catch (error: unknown) {
-            const message =
-              error instanceof Error ? error.message : String(error);
+            const message = getErrorMessage(error);
             console.error(
               `[${new Date().toISOString()}] Failed to close session ${sid} during cleanup: ${message}`,
             );
@@ -455,8 +454,7 @@ export abstract class BaseMCPServer {
         try {
           session.transport.close();
         } catch (error: unknown) {
-          const message =
-            error instanceof Error ? error.message : String(error);
+          const message = getErrorMessage(error);
           console.error(`Error closing session during shutdown: ${message}`);
         }
       }
@@ -475,7 +473,7 @@ export abstract class BaseMCPServer {
           this.performCleanup().finally(() => process.exit(1));
         }, 5000);
       } catch (error: unknown) {
-        const message = error instanceof Error ? error.message : String(error);
+        const message = getErrorMessage(error);
         console.error(`Error closing HTTP server: ${message}`);
         this.performCleanup().finally(() => process.exit(1));
       }

package/src/mcp/base-tools.ts

@@ -1,6 +1,19 @@
+import { parseBase64 } from '@aiscene/shared/img';
+import { getDebug } from '@aiscene/shared/logger';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { parseBase64 } from '../img';
-import { getDebug } from '../logger';
+import type { z } from 'zod';
+import { camelToKebab, getKeyAliases } from '../key-alias-utils';
+import {
+  type CliReportSession,
+  generateCliReportSession,
+  readCliReportSession,
+  writeCliReportSession,
+} from './cli-report-session';
+import {
+  createNamespacedInitArgSchema,
+  extractNamespacedArgs,
+  sanitizeNamespacedArgs,
+} from './init-arg-utils';
 import {
   generateCommonTools,
   generateToolsFromActionSpace,
@@ -10,22 +23,68 @@ import type {
   BaseAgent,
   BaseDevice,
   IMidsceneTools,
+  ToolCliMetadata,
   ToolDefinition,
+  ToolSchema,
 } from './types';
 
 const debug = getDebug('mcp:base-tools');
 
 /**
- * Base class for platform-specific MCP tools
- * Generic type TAgent allows subclasses to use their specific agent types
+ * Declarative description of a platform's agent init args.
+ * Collapses the `extractAgentInitParam` / `sanitizeToolArgs` /
+ * `getAgentInitArgSchema` trio into a single data declaration.
+ */
+export interface InitArgSpec<TInitParam> {
+  /** Arg namespace, e.g. `android`, `ios`. */
+  namespace: string;
+  /** Zod shape describing the init args. Field names drive the MCP schema. */
+  shape: Record<string, z.ZodTypeAny>;
+  /**
+   * Optional CLI presentation hints. These affect `--help` output for
+   * single-platform CLIs but do not alter MCP/YAML protocol keys.
+   */
+  cli?: {
+    /** Prefer bare `--device-id`-style options in platform CLI help output. */
+    preferBareKeys?: boolean;
+    /** Override the displayed option name for specific init arg fields. */
+    preferredNames?: Record<string, string>;
+  };
+  /**
+   * Adapt extracted namespaced args into the concrete `TInitParam` passed to
+   * `ensureAgent`. Defaults to returning the raw extracted record.
+   */
+  adapt?: (
+    extracted: Record<string, unknown> | undefined,
+  ) => TInitParam | undefined;
+}
+
+/**
+ * Base class for platform-specific MCP tools.
+ * @typeParam TAgent - Platform-specific agent type.
+ * @typeParam TInitParam - Platform-specific init parameter consumed by
+ *   `ensureAgent`. Defaults to `undefined` for platforms that take no args.
  */
-export abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent>
-  implements IMidsceneTools
+export abstract class BaseMidsceneTools<
+  TAgent extends BaseAgent = BaseAgent,
+  TInitParam = unknown,
+> implements IMidsceneTools
 {
   protected mcpServer?: McpServer;
   protected agent?: TAgent;
   protected toolDefinitions: ToolDefinition[] = [];
 
+  /**
+   * Declarative init-arg spec. Subclasses that accept CLI/MCP init args should
+   * set this once and get `extractAgentInitParam` / `sanitizeToolArgs` /
+   * `getAgentInitArgSchema` auto-implemented.
+   *
+   * Declared with `declare` so that TS doesn't emit an `Object.defineProperty`
+   * for this field on the base constructor, which would otherwise overwrite
+   * a subclass field initializer under `useDefineForClassFields`.
+   */
+  protected declare readonly initArgSpec?: InitArgSpec<TInitParam>;
+
   /**
    * Ensure agent is initialized and ready for use.
    * Must be implemented by subclasses to create platform-specific agent.
@@ -33,7 +92,102 @@ export abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent>
    * @returns Promise resolving to initialized agent instance
    * @throws Error if agent initialization fails
    */
-  protected abstract ensureAgent(initParam?: string): Promise<TAgent>;
+  protected abstract ensureAgent(initParam?: TInitParam): Promise<TAgent>;
+
+  private getInitArgKeys(): readonly string[] {
+    return this.initArgSpec ? Object.keys(this.initArgSpec.shape) : [];
+  }
+
+  /**
+   * Extract a platform-specific agent init parameter from CLI/MCP tool args.
+   */
+  protected extractAgentInitParam(
+    args: Record<string, unknown>,
+  ): TInitParam | undefined {
+    if (!this.initArgSpec) {
+      return undefined;
+    }
+    const extracted = extractNamespacedArgs(
+      args,
+      this.initArgSpec.namespace,
+      this.getInitArgKeys(),
+    );
+    if (this.initArgSpec.adapt) {
+      return this.initArgSpec.adapt(extracted);
+    }
+    return extracted as TInitParam | undefined;
+  }
+
+  /**
+   * Remove platform-specific init args before dispatching a tool payload to the action itself.
+   */
+  protected sanitizeToolArgs(
+    args: Record<string, unknown>,
+  ): Record<string, unknown> {
+    if (!this.initArgSpec) {
+      return args;
+    }
+    return sanitizeNamespacedArgs(
+      args,
+      this.initArgSpec.namespace,
+      this.getInitArgKeys(),
+    );
+  }
+
+  /**
+   * Expose platform-specific init args on action/common tool schemas.
+   */
+  protected getAgentInitArgSchema(): ToolSchema {
+    if (!this.initArgSpec) {
+      return {};
+    }
+    return createNamespacedInitArgSchema(
+      this.initArgSpec.namespace,
+      this.initArgSpec.shape,
+    );
+  }
+
+  /**
+   * Expose CLI-only metadata for platform init args so single-platform help can
+   * show ergonomic bare flags while the underlying schema stays namespaced.
+   * When `preferBareKeys` is enabled, single-platform CLIs only accept the
+   * bare spellings; namespaced dotted spellings remain available through the
+   * MCP/YAML schema instead of the platform CLI surface.
+   */
+  protected getAgentInitArgCliMetadata(): ToolCliMetadata | undefined {
+    if (!this.initArgSpec?.cli) {
+      return undefined;
+    }
+
+    const options = Object.fromEntries(
+      this.getInitArgKeys().map((key) => {
+        const canonicalKey = `${this.initArgSpec!.namespace}.${key}`;
+        const preferredName =
+          this.initArgSpec!.cli?.preferredNames?.[key] ??
+          (this.initArgSpec!.cli?.preferBareKeys
+            ? camelToKebab(key)
+            : canonicalKey);
+
+        const acceptedNames = new Set<string>([
+          preferredName,
+          ...(this.initArgSpec!.cli?.preferBareKeys
+            ? getKeyAliases(key)
+            : getKeyAliases(canonicalKey)),
+        ]);
+        acceptedNames.delete(preferredName);
+
+        return [
+          canonicalKey,
+          {
+            preferredName,
+            aliases: [...acceptedNames],
+          },
+        ];
+      }),
+    );
+
+    return { options };
+  }
 
   /**
    * Optional: prepare platform-specific tools (e.g., device connection)
@@ -42,6 +196,52 @@ export abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent>
     return [];
   }
 
+  protected getCliReportSessionName(): string | undefined {
+    return undefined;
+  }
+
+  protected createNewCliReportSession(
+    targetIdentity?: string,
+  ): CliReportSession | undefined {
+    const sessionName = this.getCliReportSessionName();
+    if (!sessionName) {
+      return undefined;
+    }
+    return generateCliReportSession(sessionName, targetIdentity);
+  }
+
+  protected commitCliReportSession(session?: CliReportSession): void {
+    if (session) {
+      writeCliReportSession(session);
+    }
+  }
+
+  protected readCliReportFileName(): string | undefined {
+    const sessionName = this.getCliReportSessionName();
+    if (!sessionName) {
+      return undefined;
+    }
+    return readCliReportSession(sessionName)?.reportFileName;
+  }
+
+  protected readCliReportAgentOptions():
+    | {
+        reportFileName: string;
+        reportAttributes: Record<string, string>;
+      }
+    | undefined {
+    const reportFileName = this.readCliReportFileName();
+    if (!reportFileName) {
+      return undefined;
+    }
+    return {
+      reportFileName,
+      reportAttributes: {
+        'data-group-id': reportFileName,
+      },
+    };
+  }
+
   /**
    * Must be implemented by subclasses to create a temporary device instance
    * This allows getting real actionSpace without connecting to device
@@ -83,13 +283,20 @@ export abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent>
     }
 
     // 3. Generate tools from action space (core innovation)
-    const actionTools = generateToolsFromActionSpace(actionSpace, () =>
-      this.ensureAgent(),
+    const actionTools = generateToolsFromActionSpace(
+      actionSpace,
+      (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
+      (args = {}) => this.sanitizeToolArgs(args),
+      this.getAgentInitArgSchema(),
+      this.getAgentInitArgCliMetadata(),
     );
 
     // 4. Add common tools (screenshot, waitFor)
-    const commonTools = generateCommonTools(() => this.ensureAgent());
-
+    const commonTools = generateCommonTools(
+      (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
+      this.getAgentInitArgSchema(),
+      this.getAgentInitArgCliMetadata(),
+    );
     this.toolDefinitions.push(...actionTools, ...commonTools);
 
     debug('Total tools prepared:', this.toolDefinitions.length);