@aiscene/shared 8.0.2 → 8.0.4

package/dist/types/img/info.d.ts

@@ -1,6 +1,8 @@
+import { Buffer } from 'node:buffer';
 import type { Size } from '../types';
 export interface ImageInfo extends Size {
 }
+export declare function encodedImageInfoOfBuffer(imageBuffer: Buffer): ImageInfo;
 /**
  * Retrieves the dimensions of an image from a base64-encoded string
  *
@@ -27,3 +29,8 @@ export declare function isValidJPEGImageBuffer(buffer: Buffer): boolean;
  * @returns true if the Buffer is a valid PNG or JPEG image, otherwise false
  */
 export declare function isValidImageBuffer(buffer: Buffer): boolean;
+export interface ValidateScreenshotBufferOptions {
+    label: string;
+    minBufferSize?: number;
+}
+export declare function validateScreenshotBuffer(screenshotBuffer: Buffer | undefined, { label, minBufferSize }: ValidateScreenshotBufferOptions): asserts screenshotBuffer is Buffer;

package/dist/types/img/photon-loader.d.ts

@@ -0,0 +1,2 @@
+export type PhotonModule = typeof import('@silvia-odwyer/photon');
+export declare function loadPhotonModule(): Promise<PhotonModule>;

package/dist/types/key-alias-utils.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Internal-only helpers for CLI/MCP argument key aliasing.
+ * Not re-exported from the package entry point — keep consumers within
+ * `cli/` and `mcp/`.
+ */
+export declare function kebabToCamel(str: string): string;
+export declare function camelToKebab(str: string): string;
+export declare function getKeyAliases(key: string): string[];
+export declare function isRecord(value: unknown): value is Record<string, unknown>;

package/dist/types/mcp/base-tools.d.ts

@@ -1,13 +1,53 @@
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import type { BaseAgent, BaseDevice, IMidsceneTools, ToolDefinition } from './types';
+import type { z } from 'zod';
+import { type CliReportSession } from './cli-report-session';
+import type { BaseAgent, BaseDevice, IMidsceneTools, ToolCliMetadata, ToolDefinition, ToolSchema } from './types';
 /**
- * Base class for platform-specific MCP tools
- * Generic type TAgent allows subclasses to use their specific agent types
+ * Declarative description of a platform's agent init args.
+ * Collapses the `extractAgentInitParam` / `sanitizeToolArgs` /
+ * `getAgentInitArgSchema` trio into a single data declaration.
  */
-export declare abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent> implements IMidsceneTools {
+export interface InitArgSpec<TInitParam> {
+    /** Arg namespace, e.g. `android`, `ios`. */
+    namespace: string;
+    /** Zod shape describing the init args. Field names drive the MCP schema. */
+    shape: Record<string, z.ZodTypeAny>;
+    /**
+     * Optional CLI presentation hints. These affect `--help` output for
+     * single-platform CLIs but do not alter MCP/YAML protocol keys.
+     */
+    cli?: {
+        /** Prefer bare `--device-id`-style options in platform CLI help output. */
+        preferBareKeys?: boolean;
+        /** Override the displayed option name for specific init arg fields. */
+        preferredNames?: Record<string, string>;
+    };
+    /**
+     * Adapt extracted namespaced args into the concrete `TInitParam` passed to
+     * `ensureAgent`. Defaults to returning the raw extracted record.
+     */
+    adapt?: (extracted: Record<string, unknown> | undefined) => TInitParam | undefined;
+}
+/**
+ * Base class for platform-specific MCP tools.
+ * @typeParam TAgent - Platform-specific agent type.
+ * @typeParam TInitParam - Platform-specific init parameter consumed by
+ *   `ensureAgent`. Defaults to `undefined` for platforms that take no args.
+ */
+export declare abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent, TInitParam = unknown> implements IMidsceneTools {
     protected mcpServer?: McpServer;
     protected agent?: TAgent;
     protected toolDefinitions: ToolDefinition[];
+    /**
+     * Declarative init-arg spec. Subclasses that accept CLI/MCP init args should
+     * set this once and get `extractAgentInitParam` / `sanitizeToolArgs` /
+     * `getAgentInitArgSchema` auto-implemented.
+     *
+     * Declared with `declare` so that TS doesn't emit an `Object.defineProperty`
+     * for this field on the base constructor, which would otherwise overwrite
+     * a subclass field initializer under `useDefineForClassFields`.
+     */
+    protected readonly initArgSpec?: InitArgSpec<TInitParam>;
     /**
      * Ensure agent is initialized and ready for use.
      * Must be implemented by subclasses to create platform-specific agent.
@@ -15,11 +55,40 @@ export declare abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseA
      * @returns Promise resolving to initialized agent instance
      * @throws Error if agent initialization fails
      */
-    protected abstract ensureAgent(initParam?: string): Promise<TAgent>;
+    protected abstract ensureAgent(initParam?: TInitParam): Promise<TAgent>;
+    private getInitArgKeys;
+    /**
+     * Extract a platform-specific agent init parameter from CLI/MCP tool args.
+     */
+    protected extractAgentInitParam(args: Record<string, unknown>): TInitParam | undefined;
+    /**
+     * Remove platform-specific init args before dispatching a tool payload to the action itself.
+     */
+    protected sanitizeToolArgs(args: Record<string, unknown>): Record<string, unknown>;
+    /**
+     * Expose platform-specific init args on action/common tool schemas.
+     */
+    protected getAgentInitArgSchema(): ToolSchema;
+    /**
+     * Expose CLI-only metadata for platform init args so single-platform help can
+     * show ergonomic bare flags while the underlying schema stays namespaced.
+     * When `preferBareKeys` is enabled, single-platform CLIs only accept the
+     * bare spellings; namespaced dotted spellings remain available through the
+     * MCP/YAML schema instead of the platform CLI surface.
+     */
+    protected getAgentInitArgCliMetadata(): ToolCliMetadata | undefined;
     /**
      * Optional: prepare platform-specific tools (e.g., device connection)
      */
     protected preparePlatformTools(): ToolDefinition[];
+    protected getCliReportSessionName(): string | undefined;
+    protected createNewCliReportSession(targetIdentity?: string): CliReportSession | undefined;
+    protected commitCliReportSession(session?: CliReportSession): void;
+    protected readCliReportFileName(): string | undefined;
+    protected readCliReportAgentOptions(): {
+        reportFileName: string;
+        reportAttributes: Record<string, string>;
+    } | undefined;
     /**
      * Must be implemented by subclasses to create a temporary device instance
      * This allows getting real actionSpace without connecting to device

package/dist/types/mcp/cli-report-session.d.ts

@@ -0,0 +1,12 @@
+export interface CliReportSession {
+    version: 1;
+    sessionName: string;
+    targetIdentity?: string;
+    reportFileName: string;
+    reportPath: string;
+    createdAt: number;
+}
+export declare function generateCliReportSession(sessionName: string, targetIdentity?: string): CliReportSession;
+export declare function writeCliReportSession(session: CliReportSession): void;
+export declare function createCliReportSession(sessionName: string, targetIdentity?: string): CliReportSession;
+export declare function readCliReportSession(sessionName: string): CliReportSession | undefined;

package/dist/types/mcp/error-formatter.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Extract a human-readable message from an unknown thrown value.
+ *
+ * Many SDK/transport layers reject with structured objects (e.g.
+ * `{ code, message }`, `{ error: { message } }`, `{ cause: { message } }`)
+ * rather than `Error` instances. `String(obj)` collapses those to
+ * `"[object Object]"`, which is useless for diagnostics. This helper walks
+ * the common shapes, falls back to `JSON.stringify`, and finally to
+ * `Object.prototype.toString.call` so that callers always get something
+ * actionable in logs and surfaced tool results.
+ */
+export declare function getErrorMessage(error: unknown): string;

package/dist/types/mcp/index.d.ts

@@ -1,5 +1,7 @@
 export * from './base-server';
 export * from './base-tools';
+export * from './init-arg-utils';
+export * from './error-formatter';
 export * from './tool-generator';
 export * from './types';
 export * from './inject-report-html-plugin';

package/dist/types/mcp/init-arg-utils.d.ts

@@ -0,0 +1,13 @@
+import type { z } from 'zod';
+import type { ToolSchema } from './types';
+export declare function extractNamespacedArgs<TFieldName extends string, TArgs extends Record<string, unknown> = Record<string, unknown>>(args: Record<string, unknown>, namespace: string, keys: readonly TFieldName[]): TArgs | undefined;
+export declare function sanitizeNamespacedArgs(args: Record<string, unknown>, namespace: string, keys: readonly string[]): Record<string, unknown>;
+/**
+ * Build a flat MCP tool schema whose keys are dotted `"<namespace>.<field>"`.
+ *
+ * We intentionally stay flat (rather than `{ namespace: z.object({...}) }`) so
+ * that CLI (`--android.device-id`), MCP clients, and `--help` output all share
+ * the same spelling. `readNamespacedArg` understands all three input shapes:
+ * nested namespace object, dotted flat key, and bare key fallback.
+ */
+export declare function createNamespacedInitArgSchema(namespace: string, shape: Record<string, z.ZodTypeAny>): ToolSchema;

package/dist/types/mcp/inject-report-html-plugin.d.ts

@@ -2,12 +2,12 @@ interface RslibPluginApi {
     onAfterBuild: (callback: () => void) => void;
 }
 /**
- * Rslib plugin to inject report HTML from @midscene/core dist into MCP bundle.
+ * Rslib plugin to inject report HTML from @aiscene/core dist into MCP bundle.
  * This runs after build and reads the already-injected HTML from core.
  *
  * Prerequisites:
  * - @midscene/report must be in devDependencies to ensure correct build order
- * - @midscene/core dist must exist with injected HTML
+ * - @aiscene/core dist must exist with injected HTML
  *
  * @param packageDir - The directory of the MCP package (use __dirname)
  */

package/dist/types/mcp/launcher-helper.d.ts

@@ -9,7 +9,7 @@ export interface LaunchMCPServerOptions extends HttpLaunchOptions {
     verbose?: boolean;
 }
 /**
- * Generic agent type (avoid importing from @midscene/core to prevent circular deps)
+ * Generic agent type (avoid importing from @aiscene/core to prevent circular deps)
  */
 export interface GenericAgent<TDevice = any> {
     interface: TDevice;
@@ -58,8 +58,8 @@ export interface MCPServerLauncherConfig<AgentType extends GenericAgent = Generi
  *
  * @example
  * ```typescript
- * import { createMCPServerLauncher } from '@midscene/shared/mcp';
- * import { Agent } from '@midscene/core/agent';
+ * import { createMCPServerLauncher } from '@aiscene/shared/mcp';
+ * import { Agent } from '@aiscene/core/agent';
  * import { WebMidsceneTools } from './web-tools';
  * import { WebMCPServer } from './server';
  *

package/dist/types/mcp/tool-generator.d.ts

@@ -1,10 +1,10 @@
-import type { ActionSpaceItem, BaseAgent, ToolDefinition } from './types';
+import type { ActionSpaceItem, BaseAgent, ToolCliMetadata, ToolDefinition, ToolSchema } from './types';
 /**
  * Converts DeviceAction from actionSpace into MCP ToolDefinition
  * This is the core logic that removes need for hardcoded tool definitions
  */
-export declare function generateToolsFromActionSpace(actionSpace: ActionSpaceItem[], getAgent: () => Promise<BaseAgent>): ToolDefinition[];
+export declare function generateToolsFromActionSpace(actionSpace: ActionSpaceItem[], getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, sanitizeArgs?: (args: Record<string, unknown>) => Record<string, unknown>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata): ToolDefinition[];
 /**
  * Generate common tools (screenshot, act)
  */
-export declare function generateCommonTools(getAgent: () => Promise<BaseAgent>): ToolDefinition[];
+export declare function generateCommonTools(getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata): ToolDefinition[];

package/dist/types/mcp/types.d.ts

@@ -49,6 +49,13 @@ export type ToolHandler<T = Record<string, unknown>> = (args: T) => Promise<Tool
  * Tool schema type using Zod
  */
 export type ToolSchema = Record<string, z.ZodTypeAny>;
+export interface ToolCliOption {
+    preferredName?: string;
+    aliases?: string[];
+}
+export interface ToolCliMetadata {
+    options?: Record<string, ToolCliOption>;
+}
 /**
  * Tool definition for MCP server
  */
@@ -57,6 +64,7 @@ export interface ToolDefinition<T = Record<string, unknown>> {
     description: string;
     schema: ToolSchema;
     handler: ToolHandler<T>;
+    cli?: ToolCliMetadata;
 }
 /**
  * Tool type for mcpKitForAgent return value
@@ -86,6 +94,8 @@ export interface BaseAgent {
     callActionInActionSpace?: (actionName: string, params?: unknown) => Promise<unknown>;
     aiAction?: (description: string, params?: Record<string, unknown>) => Promise<unknown>;
     aiWaitFor?: (assertion: string, options: Record<string, unknown>) => Promise<unknown>;
+    aiAssert?: (assertion: string, msg?: string, options?: Record<string, unknown>) => Promise<unknown>;
+    aiQuery?: (demand: any, opt?: any) => Promise<any>;
 }
 /**
  * Base device interface for temporary device instances

package/dist/types/recorder.d.ts

@@ -0,0 +1,113 @@
+export type MidsceneRecorderEventType = 'click' | 'drag' | 'scroll' | 'input' | 'navigation' | 'setViewport' | 'keydown';
+export type MidsceneRecorderSourceKind = 'studio-preview' | 'unsupported' | (string & {});
+export type MidsceneRecorderPlatformId = 'web' | 'android' | 'ios' | 'computer' | 'harmony' | (string & {});
+export interface MidsceneRecorderElementRect {
+    left?: number;
+    top?: number;
+    width?: number;
+    height?: number;
+    x?: number;
+    y?: number;
+}
+export interface MidsceneRecorderPageInfo {
+    width: number;
+    height: number;
+}
+/**
+ * A screenshot stored outside the recording event payload.
+ *
+ * Recorder events are persisted in the Studio renderer. Keeping full data
+ * URLs there makes long recordings retain every screenshot in the renderer
+ * heap, so screenshot bytes live in the Playground run directory instead.
+ */
+export interface MidsceneRecorderScreenshotAssetRef {
+    id: string;
+    mimeType: string;
+    bytes: number;
+}
+export interface MidsceneRecorderEvent {
+    type: MidsceneRecorderEventType;
+    source?: MidsceneRecorderSourceKind;
+    actionType?: string;
+    rawPayload?: Record<string, unknown>;
+    url?: string;
+    title?: string;
+    value?: string;
+    elementRect?: MidsceneRecorderElementRect;
+    pageInfo: MidsceneRecorderPageInfo;
+    screenshotBefore?: string;
+    screenshotAfter?: string;
+    /** The single screenshot retained for AI description and Markdown export. */
+    screenshotAsset?: MidsceneRecorderScreenshotAssetRef;
+    semantic?: MidsceneRecorderSemantic;
+    elementDescription?: string;
+    descriptionLoading?: boolean;
+    screenshotWithBox?: string;
+    timestamp: number;
+    hashId: string;
+    mergedHashIds?: string[];
+}
+export type MidsceneRecorderSemanticSource = 'aiDescribe' | 'recorderAI' | 'heuristic';
+export type MidsceneRecorderSemanticStatus = 'pending' | 'ready' | 'failed';
+export type MidsceneRecorderSemanticConfidence = 'high' | 'medium' | 'low';
+export interface MidsceneRecorderSemanticAiDescribe {
+    verifyPrompt: boolean;
+    verifyPassed?: boolean;
+    deepLocate?: boolean;
+    centerDistance?: number;
+    expectedCenter?: [number, number];
+    actualCenter?: [number, number];
+    annotatedScreenshotPath?: string;
+}
+export interface MidsceneRecorderSemantic {
+    source: MidsceneRecorderSemanticSource;
+    status: MidsceneRecorderSemanticStatus;
+    elementDescription?: string;
+    replayInstruction?: string;
+    actionSummary?: string;
+    confidence?: MidsceneRecorderSemanticConfidence;
+    error?: string;
+    aiDescribe?: MidsceneRecorderSemanticAiDescribe;
+    fallbackFrom?: MidsceneRecorderSemantic;
+}
+export interface MidsceneRecorderSemanticAction {
+    type: MidsceneRecorderEventType;
+    actionType?: string;
+    value?: string;
+    url?: string;
+    scrollDestinationDescription?: string;
+}
+export interface MidsceneRecorderTarget {
+    platformId: MidsceneRecorderPlatformId;
+    deviceId?: string;
+    label?: string;
+    values: Record<string, string | number | boolean>;
+}
+export interface MidsceneRecorderGeneratedCode {
+    markdown?: string;
+    yaml?: string;
+    playwright?: string;
+    updatedAt?: number;
+}
+export interface MidsceneRecorderMarkdownScreenshotAsset {
+    eventIndex: number;
+    eventHashId: string;
+    eventType: MidsceneRecorderEventType;
+    relativePath: string;
+    dataUrl: string;
+    base64Data: string;
+    mimeType: string;
+}
+export interface MidsceneRecorderMarkdownScreenshotOptions {
+    baseDir?: string;
+    maxScreenshots?: number;
+}
+export declare const DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS = 20;
+export declare function getMidsceneRecorderSemantic(event: Pick<MidsceneRecorderEvent, 'semantic'>): MidsceneRecorderSemantic | undefined;
+export declare function buildMidsceneRecorderReplayInstruction(event: MidsceneRecorderSemanticAction, elementDescription: string): string;
+export declare function buildMidsceneRecorderActionSummary(event: MidsceneRecorderSemanticAction, elementDescription: string): string;
+export declare function getMidsceneRecorderEventDescription(event: MidsceneRecorderEvent): string;
+export declare function getMidsceneRecorderScreenshotsForLLM(events: MidsceneRecorderEvent[], maxScreenshots?: number): string[];
+export declare function sanitizeMidsceneRecorderFileName(value: string): string;
+export declare function createMidsceneRecorderMarkdownScreenshotAssets(events: MidsceneRecorderEvent[], options?: MidsceneRecorderMarkdownScreenshotOptions): MidsceneRecorderMarkdownScreenshotAsset[];
+export declare function stringifyMidsceneRecorderTargetBlock(target: MidsceneRecorderTarget): string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiscene/shared",
-  "version": "8.0.2",
+  "version": "8.0.4",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",

package/src/agent-tools/agent-behavior-init-args.ts

@@ -0,0 +1,109 @@
+import { z } from 'zod';
+
+export interface AgentBehaviorInitArgs {
+  aiActContext?: string;
+  aiActionContext?: string;
+  replanningCycleLimit?: number;
+  waitAfterAction?: number;
+  screenshotShrinkFactor?: number;
+}
+
+type ExposedAgentBehaviorInitArgKey = Exclude<
+  keyof AgentBehaviorInitArgs,
+  'aiActionContext'
+>;
+
+export const agentBehaviorInitArgShape = {
+  aiActContext: z
+    .string()
+    .optional()
+    .describe(
+      'Background knowledge passed to aiAct. Default: no extra context.',
+    ),
+  replanningCycleLimit: z
+    .number()
+    .int()
+    .nonnegative()
+    .optional()
+    .describe(
+      'Maximum number of replanning cycles for aiAct. Default: model adapter default.',
+    ),
+  waitAfterAction: z
+    .number()
+    .nonnegative()
+    .optional()
+    .describe(
+      'Wait time in milliseconds after each action execution. Default: 300ms.',
+    ),
+  screenshotShrinkFactor: z
+    .number()
+    .min(1)
+    .optional()
+    .describe(
+      'Screenshot shrink factor before sending images to AI. Default: 1; high values may reduce recognition quality, especially on mobile.',
+    ),
+} satisfies Record<ExposedAgentBehaviorInitArgKey, z.ZodTypeAny>;
+
+export function extractAgentBehaviorInitArgs(
+  extracted: Partial<AgentBehaviorInitArgs> | undefined,
+): AgentBehaviorInitArgs | undefined {
+  if (!extracted) {
+    return undefined;
+  }
+
+  const agentOptions: AgentBehaviorInitArgs = {
+    ...(typeof extracted.aiActContext === 'string'
+      ? { aiActContext: extracted.aiActContext }
+      : {}),
+    ...(typeof extracted.aiActionContext === 'string'
+      ? { aiActionContext: extracted.aiActionContext }
+      : {}),
+    ...(typeof extracted.replanningCycleLimit === 'number'
+      ? { replanningCycleLimit: extracted.replanningCycleLimit }
+      : {}),
+    ...(typeof extracted.waitAfterAction === 'number'
+      ? { waitAfterAction: extracted.waitAfterAction }
+      : {}),
+    ...(typeof extracted.screenshotShrinkFactor === 'number'
+      ? { screenshotShrinkFactor: extracted.screenshotShrinkFactor }
+      : {}),
+  };
+
+  return Object.keys(agentOptions).length > 0 ? agentOptions : undefined;
+}
+
+function stableJsonValue(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map(stableJsonValue);
+  }
+
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([key, nestedValue]) => [key, stableJsonValue(nestedValue)]),
+    );
+  }
+
+  return value;
+}
+
+export function getAgentInitArgsSignature(
+  initArgs: object | undefined,
+): string | undefined {
+  if (!initArgs || Object.keys(initArgs).length === 0) {
+    return undefined;
+  }
+
+  return JSON.stringify(stableJsonValue(initArgs));
+}
+
+export function shouldRebuildAgentForInitArgs(
+  currentSignature: string | undefined,
+  nextSignature: string | undefined,
+): boolean {
+  return (
+    currentSignature !== nextSignature &&
+    (currentSignature !== undefined || nextSignature !== undefined)
+  );
+}