@aiscene/shared 8.0.2 → 8.0.4

package/src/img/transform.ts

@@ -4,11 +4,12 @@ import { readFileSync } from 'node:fs';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon-node';
-import type { Rect } from 'src/types';
+import type { Rect, Size } from '../types';
 import { getDebug } from '../logger';
 import { ifInNode } from '../utils';
 import getPhoton from './get-photon';
 import getSharp from './get-sharp';
+import { encodedImageInfoOfBuffer } from './info';
 
 const imgDebug = getDebug('img');
 
@@ -154,10 +155,268 @@ export async function resizeAndConvertImgBuffer(
   };
 }
 
+const rawBase64BodyPattern = /^[A-Za-z0-9+/]+={0,2}$/;
+const base64ImageDataUrlPattern =
+  /^data:image\/(?:png|jpe?g);base64,[A-Za-z0-9+/=\s]+$/i;
+const supportedScreenshotDataUriPattern =
+  /^data:image\/(png|jpe?g);base64,([A-Za-z0-9+/=\s]+)$/i;
+
+function normalizeBase64Body(body: string): string {
+  return body.replace(/\s/g, '');
+}
+
+function detectImageMimeTypeFromBuffer(buffer: Buffer): string | undefined {
+  if (
+    buffer.length >= 8 &&
+    buffer[0] === 0x89 &&
+    buffer[1] === 0x50 &&
+    buffer[2] === 0x4e &&
+    buffer[3] === 0x47 &&
+    buffer[4] === 0x0d &&
+    buffer[5] === 0x0a &&
+    buffer[6] === 0x1a &&
+    buffer[7] === 0x0a
+  ) {
+    return 'image/png';
+  }
+  if (
+    buffer.length >= 3 &&
+    buffer[0] === 0xff &&
+    buffer[1] === 0xd8 &&
+    buffer[2] === 0xff
+  ) {
+    return 'image/jpeg';
+  }
+  return undefined;
+}
+
+export function inferBase64ImageFormat(base64Body: string): string {
+  const buffer = Buffer.from(normalizeBase64Body(base64Body), 'base64');
+  const mimeType = detectImageMimeTypeFromBuffer(buffer);
+  if (mimeType === 'image/jpeg') return 'jpeg';
+  if (mimeType === 'image/png') return 'png';
+  return 'png';
+}
+
 export const createImgBase64ByFormat = (format: string, body: string) => {
-  return `data:image/${format};base64,${body}`;
+  return `data:image/${format};base64,${normalizeBase64Body(body)}`;
 };
 
+export interface NormalizeScreenshotBase64Options {
+  label?: string;
+}
+
+export const normalizeScreenshotBase64 = (
+  base64: string,
+  options?: NormalizeScreenshotBase64Options,
+) => {
+  const label = options?.label ?? 'screenshot base64';
+  const trimmedBase64 = base64.trim();
+  if (!trimmedBase64) {
+    throw new Error(`${label} cannot be empty`);
+  }
+
+  const dataUriMatch = trimmedBase64.match(supportedScreenshotDataUriPattern);
+  if (dataUriMatch) {
+    const imageFormat =
+      dataUriMatch[1].toLowerCase() === 'jpg'
+        ? 'jpeg'
+        : dataUriMatch[1].toLowerCase();
+    const body = normalizeBase64Body(dataUriMatch[2]);
+    if (!body) {
+      throw new Error(`${label} cannot be empty`);
+    }
+    return createImgBase64ByFormat(imageFormat, body);
+  }
+
+  if (trimmedBase64.startsWith('data:')) {
+    throw new Error(
+      `${label} must be a PNG/JPEG data URI or raw PNG base64 string`,
+    );
+  }
+
+  if (!rawBase64BodyPattern.test(normalizeBase64Body(trimmedBase64))) {
+    throw new Error(
+      `${label} must be a PNG/JPEG data URI or raw PNG base64 string`,
+    );
+  }
+
+  return createImgBase64ByFormat('png', trimmedBase64);
+};
+
+export const normalizeBase64Image = (base64: string) => {
+  const trimmedBase64 = base64.trim();
+  if (base64ImageDataUrlPattern.test(trimmedBase64)) {
+    return trimmedBase64.replace(/\s/g, '');
+  }
+
+  const base64Body = normalizeBase64Body(trimmedBase64);
+  assert(base64Body, 'base64 image must include image data');
+  return createImgBase64ByFormat(
+    inferBase64ImageFormat(base64Body),
+    base64Body,
+  );
+};
+
+export type JpegBase64DataUrl = `data:image/jpeg;base64,${string}`;
+
+export interface ResizeBase64ImageToJpegOptions {
+  sourceSize?: Size;
+  targetSize: Size;
+  jpegQuality?: number;
+}
+
+export interface ConstrainBase64ImageToMaxSizeOptions {
+  maxSize: number;
+  jpegQuality?: number;
+}
+
+function assertValidJpegQuality(jpegQuality: number): void {
+  if (!Number.isInteger(jpegQuality) || jpegQuality < 1 || jpegQuality > 100) {
+    throw new Error(
+      `jpegQuality must be an integer between 1 and 100. Received: ${jpegQuality}`,
+    );
+  }
+}
+
+function assertValidImageSize(size: Size, label: string): void {
+  if (
+    !Number.isInteger(size.width) ||
+    !Number.isInteger(size.height) ||
+    size.width <= 0 ||
+    size.height <= 0
+  ) {
+    throw new Error(
+      `${label} width and height must be positive integers. Received width: ${size.width}, height: ${size.height}`,
+    );
+  }
+}
+
+export async function convertImgBufferToJpeg(
+  inputData: Buffer,
+  jpegQuality = 90,
+): Promise<Buffer> {
+  assertValidJpegQuality(jpegQuality);
+
+  if (ifInNode) {
+    try {
+      const Sharp = await getSharp();
+      return await Sharp(inputData).jpeg({ quality: jpegQuality }).toBuffer();
+    } catch (error) {
+      imgDebug('Sharp failed, falling back to Photon:', error);
+    }
+  }
+
+  const { PhotonImage } = await getPhoton();
+  const inputBytes = new Uint8Array(inputData);
+  const bytesliceResult = PhotonImage.new_from_byteslice(inputBytes);
+  const image =
+    bytesliceResult instanceof Promise
+      ? await bytesliceResult
+      : bytesliceResult;
+  try {
+    return Buffer.from(image.get_bytes_jpeg(jpegQuality));
+  } finally {
+    image.free();
+  }
+}
+
+export async function convertBase64ImageToJpeg(
+  inputBase64: string,
+  jpegQuality = 90,
+): Promise<JpegBase64DataUrl> {
+  assertValidJpegQuality(jpegQuality);
+  const normalized = normalizeBase64Image(inputBase64);
+  const { body } = parseBase64(normalized);
+  const imageBuffer = Buffer.from(body, 'base64');
+  const detectedMimeType = detectImageMimeTypeFromBuffer(imageBuffer);
+  if (detectedMimeType === 'image/jpeg') {
+    return createImgBase64ByFormat('jpeg', body) as JpegBase64DataUrl;
+  }
+  if (detectedMimeType !== 'image/png') {
+    throw new Error(
+      `inputBase64 must contain a PNG or JPEG image. Detected: ${detectedMimeType ?? 'unsupported format'}`,
+    );
+  }
+
+  const jpegBuffer = await convertImgBufferToJpeg(imageBuffer, jpegQuality);
+  return createImgBase64ByFormat(
+    'jpeg',
+    jpegBuffer.toString('base64'),
+  ) as JpegBase64DataUrl;
+}
+
+export async function resizeBase64ImageToJpeg(
+  inputBase64: string,
+  options: ResizeBase64ImageToJpegOptions,
+): Promise<JpegBase64DataUrl> {
+  const jpegQuality = options.jpegQuality ?? 90;
+  assertValidJpegQuality(jpegQuality);
+  assertValidImageSize(options.targetSize, 'targetSize');
+
+  const normalized = normalizeBase64Image(inputBase64);
+  const { body, mimeType } = parseBase64(normalized);
+  const imageBuffer = Buffer.from(body, 'base64');
+  const sourceSize = encodedImageInfoOfBuffer(imageBuffer);
+  if (options.sourceSize) {
+    assertValidImageSize(options.sourceSize, 'sourceSize');
+    if (
+      options.sourceSize.width !== sourceSize.width ||
+      options.sourceSize.height !== sourceSize.height
+    ) {
+      throw new Error(
+        `sourceSize ${options.sourceSize.width}x${options.sourceSize.height} does not match encoded image dimensions ${sourceSize.width}x${sourceSize.height}`,
+      );
+    }
+  }
+
+  const dimensionsUnchanged =
+    sourceSize.width === options.targetSize.width &&
+    sourceSize.height === options.targetSize.height;
+  if (dimensionsUnchanged) {
+    return convertBase64ImageToJpeg(normalized, jpegQuality);
+  }
+
+  const { buffer } = await resizeAndConvertImgBuffer(
+    mimeType.split('/')[1],
+    imageBuffer,
+    options.targetSize,
+  );
+  return createImgBase64ByFormat(
+    'jpeg',
+    buffer.toString('base64'),
+  ) as JpegBase64DataUrl;
+}
+
+export async function constrainBase64ImageToMaxSize(
+  inputBase64: string,
+  options: ConstrainBase64ImageToMaxSizeOptions,
+): Promise<string> {
+  if (!Number.isInteger(options.maxSize) || options.maxSize <= 0) {
+    throw new Error(
+      `maxSize must be a positive integer. Received: ${options.maxSize}`,
+    );
+  }
+
+  const normalized = normalizeBase64Image(inputBase64);
+  const { body } = parseBase64(normalized);
+  const sourceSize = encodedImageInfoOfBuffer(Buffer.from(body, 'base64'));
+  const largestDimension = Math.max(sourceSize.width, sourceSize.height);
+  if (largestDimension <= options.maxSize) {
+    return normalized;
+  }
+
+  const scale = options.maxSize / largestDimension;
+  return resizeBase64ImageToJpeg(normalized, {
+    targetSize: {
+      width: Math.max(1, Math.round(sourceSize.width * scale)),
+      height: Math.max(1, Math.round(sourceSize.height * scale)),
+    },
+    sourceSize,
+    jpegQuality: options.jpegQuality,
+  });
+}
+
 export async function resizeImgBase64(
   inputBase64: string,
   newSize: {

package/src/key-alias-utils.ts

@@ -0,0 +1,23 @@
+/**
+ * Internal-only helpers for CLI/MCP argument key aliasing.
+ * Not re-exported from the package entry point — keep consumers within
+ * `cli/` and `mcp/`.
+ */
+
+export function kebabToCamel(str: string): string {
+  return str.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase());
+}
+
+export function camelToKebab(str: string): string {
+  return str
+    .replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`)
+    .replace(/^-/, '');
+}
+
+export function getKeyAliases(key: string): string[] {
+  return [...new Set([key, kebabToCamel(key), camelToKebab(key)])];
+}
+
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}

package/src/mcp/base-server.ts

@@ -1,5 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import type { ParseArgsConfig } from 'node:util';
+import { setIsMcp } from '@aiscene/shared/utils';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
@@ -8,7 +9,7 @@ import express, {
   type Request,
   type Response,
 } from 'express';
-import { setIsMcp } from '../utils';
+import { getErrorMessage } from './error-formatter';
 import type { IMidsceneTools } from './types';
 
 export interface BaseMCPServerConfig {
@@ -119,7 +120,7 @@ export abstract class BaseMCPServer {
     try {
       await this.toolsManager.initTools();
     } catch (error: unknown) {
-      const message = error instanceof Error ? error.message : String(error);
+      const message = getErrorMessage(error);
       console.error(`Failed to initialize tools: ${message}`);
       console.error('Tools will be initialized on first use');
     }
@@ -160,7 +161,7 @@ export abstract class BaseMCPServer {
     try {
       await this.mcpServer.connect(transport);
     } catch (error: unknown) {
-      const message = error instanceof Error ? error.message : String(error);
+      const message = getErrorMessage(error);
       console.error(`Failed to connect MCP stdio transport: ${message}`);
       throw new Error(`Failed to initialize MCP stdio transport: ${message}`);
     }
@@ -282,7 +283,7 @@ export abstract class BaseMCPServer {
             .json({ error: 'Invalid session or GET without session' });
         }
       } catch (error: unknown) {
-        const message = error instanceof Error ? error.message : String(error);
+        const message = getErrorMessage(error);
         const duration = Date.now() - startTime;
         console.error(
           `[${new Date().toISOString()}] [${requestId}] MCP request error after ${duration}ms: ${message}`,
@@ -336,8 +337,7 @@ export abstract class BaseMCPServer {
           try {
             await session.transport.close();
           } catch (error: unknown) {
-            const message =
-              error instanceof Error ? error.message : String(error);
+            const message = getErrorMessage(error);
             console.error(
               `Failed to close session ${session.transport.sessionId}: ${message}`,
             );
@@ -390,7 +390,7 @@ export abstract class BaseMCPServer {
     try {
       await this.mcpServer.connect(transport);
     } catch (error: unknown) {
-      const message = error instanceof Error ? error.message : String(error);
+      const message = getErrorMessage(error);
       console.error(
         `[${new Date().toISOString()}] Failed to connect MCP transport: ${message}`,
       );
@@ -425,8 +425,7 @@ export abstract class BaseMCPServer {
               `[${new Date().toISOString()}] Session ${sid} cleaned up due to inactivity (remaining: ${sessions.size})`,
             );
           } catch (error: unknown) {
-            const message =
-              error instanceof Error ? error.message : String(error);
+            const message = getErrorMessage(error);
             console.error(
               `[${new Date().toISOString()}] Failed to close session ${sid} during cleanup: ${message}`,
             );
@@ -455,8 +454,7 @@ export abstract class BaseMCPServer {
         try {
           session.transport.close();
         } catch (error: unknown) {
-          const message =
-            error instanceof Error ? error.message : String(error);
+          const message = getErrorMessage(error);
           console.error(`Error closing session during shutdown: ${message}`);
         }
       }
@@ -475,7 +473,7 @@ export abstract class BaseMCPServer {
           this.performCleanup().finally(() => process.exit(1));
         }, 5000);
       } catch (error: unknown) {
-        const message = error instanceof Error ? error.message : String(error);
+        const message = getErrorMessage(error);
         console.error(`Error closing HTTP server: ${message}`);
         this.performCleanup().finally(() => process.exit(1));
       }

package/src/mcp/base-tools.ts

@@ -1,6 +1,19 @@
+import { parseBase64 } from '@aiscene/shared/img';
+import { getDebug } from '@aiscene/shared/logger';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { parseBase64 } from '../img';
-import { getDebug } from '../logger';
+import type { z } from 'zod';
+import { camelToKebab, getKeyAliases } from '../key-alias-utils';
+import {
+  type CliReportSession,
+  generateCliReportSession,
+  readCliReportSession,
+  writeCliReportSession,
+} from './cli-report-session';
+import {
+  createNamespacedInitArgSchema,
+  extractNamespacedArgs,
+  sanitizeNamespacedArgs,
+} from './init-arg-utils';
 import {
   generateCommonTools,
   generateToolsFromActionSpace,
@@ -10,22 +23,68 @@ import type {
   BaseAgent,
   BaseDevice,
   IMidsceneTools,
+  ToolCliMetadata,
   ToolDefinition,
+  ToolSchema,
 } from './types';
 
 const debug = getDebug('mcp:base-tools');
 
 /**
- * Base class for platform-specific MCP tools
- * Generic type TAgent allows subclasses to use their specific agent types
+ * Declarative description of a platform's agent init args.
+ * Collapses the `extractAgentInitParam` / `sanitizeToolArgs` /
+ * `getAgentInitArgSchema` trio into a single data declaration.
+ */
+export interface InitArgSpec<TInitParam> {
+  /** Arg namespace, e.g. `android`, `ios`. */
+  namespace: string;
+  /** Zod shape describing the init args. Field names drive the MCP schema. */
+  shape: Record<string, z.ZodTypeAny>;
+  /**
+   * Optional CLI presentation hints. These affect `--help` output for
+   * single-platform CLIs but do not alter MCP/YAML protocol keys.
+   */
+  cli?: {
+    /** Prefer bare `--device-id`-style options in platform CLI help output. */
+    preferBareKeys?: boolean;
+    /** Override the displayed option name for specific init arg fields. */
+    preferredNames?: Record<string, string>;
+  };
+  /**
+   * Adapt extracted namespaced args into the concrete `TInitParam` passed to
+   * `ensureAgent`. Defaults to returning the raw extracted record.
+   */
+  adapt?: (
+    extracted: Record<string, unknown> | undefined,
+  ) => TInitParam | undefined;
+}
+
+/**
+ * Base class for platform-specific MCP tools.
+ * @typeParam TAgent - Platform-specific agent type.
+ * @typeParam TInitParam - Platform-specific init parameter consumed by
+ *   `ensureAgent`. Defaults to `undefined` for platforms that take no args.
  */
-export abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent>
-  implements IMidsceneTools
+export abstract class BaseMidsceneTools<
+  TAgent extends BaseAgent = BaseAgent,
+  TInitParam = unknown,
+> implements IMidsceneTools
 {
   protected mcpServer?: McpServer;
   protected agent?: TAgent;
   protected toolDefinitions: ToolDefinition[] = [];
 
+  /**
+   * Declarative init-arg spec. Subclasses that accept CLI/MCP init args should
+   * set this once and get `extractAgentInitParam` / `sanitizeToolArgs` /
+   * `getAgentInitArgSchema` auto-implemented.
+   *
+   * Declared with `declare` so that TS doesn't emit an `Object.defineProperty`
+   * for this field on the base constructor, which would otherwise overwrite
+   * a subclass field initializer under `useDefineForClassFields`.
+   */
+  protected declare readonly initArgSpec?: InitArgSpec<TInitParam>;
+
   /**
    * Ensure agent is initialized and ready for use.
    * Must be implemented by subclasses to create platform-specific agent.
@@ -33,7 +92,102 @@ export abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent>
    * @returns Promise resolving to initialized agent instance
    * @throws Error if agent initialization fails
    */
-  protected abstract ensureAgent(initParam?: string): Promise<TAgent>;
+  protected abstract ensureAgent(initParam?: TInitParam): Promise<TAgent>;
+
+  private getInitArgKeys(): readonly string[] {
+    return this.initArgSpec ? Object.keys(this.initArgSpec.shape) : [];
+  }
+
+  /**
+   * Extract a platform-specific agent init parameter from CLI/MCP tool args.
+   */
+  protected extractAgentInitParam(
+    args: Record<string, unknown>,
+  ): TInitParam | undefined {
+    if (!this.initArgSpec) {
+      return undefined;
+    }
+    const extracted = extractNamespacedArgs(
+      args,
+      this.initArgSpec.namespace,
+      this.getInitArgKeys(),
+    );
+    if (this.initArgSpec.adapt) {
+      return this.initArgSpec.adapt(extracted);
+    }
+    return extracted as TInitParam | undefined;
+  }
+
+  /**
+   * Remove platform-specific init args before dispatching a tool payload to the action itself.
+   */
+  protected sanitizeToolArgs(
+    args: Record<string, unknown>,
+  ): Record<string, unknown> {
+    if (!this.initArgSpec) {
+      return args;
+    }
+    return sanitizeNamespacedArgs(
+      args,
+      this.initArgSpec.namespace,
+      this.getInitArgKeys(),
+    );
+  }
+
+  /**
+   * Expose platform-specific init args on action/common tool schemas.
+   */
+  protected getAgentInitArgSchema(): ToolSchema {
+    if (!this.initArgSpec) {
+      return {};
+    }
+    return createNamespacedInitArgSchema(
+      this.initArgSpec.namespace,
+      this.initArgSpec.shape,
+    );
+  }
+
+  /**
+   * Expose CLI-only metadata for platform init args so single-platform help can
+   * show ergonomic bare flags while the underlying schema stays namespaced.
+   * When `preferBareKeys` is enabled, single-platform CLIs only accept the
+   * bare spellings; namespaced dotted spellings remain available through the
+   * MCP/YAML schema instead of the platform CLI surface.
+   */
+  protected getAgentInitArgCliMetadata(): ToolCliMetadata | undefined {
+    if (!this.initArgSpec?.cli) {
+      return undefined;
+    }
+
+    const options = Object.fromEntries(
+      this.getInitArgKeys().map((key) => {
+        const canonicalKey = `${this.initArgSpec!.namespace}.${key}`;
+        const preferredName =
+          this.initArgSpec!.cli?.preferredNames?.[key] ??
+          (this.initArgSpec!.cli?.preferBareKeys
+            ? camelToKebab(key)
+            : canonicalKey);
+
+        const acceptedNames = new Set<string>([
+          preferredName,
+          ...(this.initArgSpec!.cli?.preferBareKeys
+            ? getKeyAliases(key)
+            : getKeyAliases(canonicalKey)),
+        ]);
+        acceptedNames.delete(preferredName);
+
+        return [
+          canonicalKey,
+          {
+            preferredName,
+            aliases: [...acceptedNames],
+          },
+        ];
+      }),
+    );
+
+    return { options };
+  }
 
   /**
    * Optional: prepare platform-specific tools (e.g., device connection)
@@ -42,6 +196,52 @@ export abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent>
     return [];
   }
 
+  protected getCliReportSessionName(): string | undefined {
+    return undefined;
+  }
+
+  protected createNewCliReportSession(
+    targetIdentity?: string,
+  ): CliReportSession | undefined {
+    const sessionName = this.getCliReportSessionName();
+    if (!sessionName) {
+      return undefined;
+    }
+    return generateCliReportSession(sessionName, targetIdentity);
+  }
+
+  protected commitCliReportSession(session?: CliReportSession): void {
+    if (session) {
+      writeCliReportSession(session);
+    }
+  }
+
+  protected readCliReportFileName(): string | undefined {
+    const sessionName = this.getCliReportSessionName();
+    if (!sessionName) {
+      return undefined;
+    }
+    return readCliReportSession(sessionName)?.reportFileName;
+  }
+
+  protected readCliReportAgentOptions():
+    | {
+        reportFileName: string;
+        reportAttributes: Record<string, string>;
+      }
+    | undefined {
+    const reportFileName = this.readCliReportFileName();
+    if (!reportFileName) {
+      return undefined;
+    }
+    return {
+      reportFileName,
+      reportAttributes: {
+        'data-group-id': reportFileName,
+      },
+    };
+  }
+
   /**
    * Must be implemented by subclasses to create a temporary device instance
    * This allows getting real actionSpace without connecting to device
@@ -83,13 +283,20 @@ export abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent>
     }
 
     // 3. Generate tools from action space (core innovation)
-    const actionTools = generateToolsFromActionSpace(actionSpace, () =>
-      this.ensureAgent(),
+    const actionTools = generateToolsFromActionSpace(
+      actionSpace,
+      (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
+      (args = {}) => this.sanitizeToolArgs(args),
+      this.getAgentInitArgSchema(),
+      this.getAgentInitArgCliMetadata(),
     );
 
     // 4. Add common tools (screenshot, waitFor)
-    const commonTools = generateCommonTools(() => this.ensureAgent());
-
+    const commonTools = generateCommonTools(
+      (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
+      this.getAgentInitArgSchema(),
+      this.getAgentInitArgCliMetadata(),
+    );
     this.toolDefinitions.push(...actionTools, ...commonTools);
 
     debug('Total tools prepared:', this.toolDefinitions.length);