@aiscene/shared 8.0.3 → 8.0.4

package/dist/types/agent-tools/tool-generator.d.ts

@@ -0,0 +1,13 @@
+import type { ToolDefaults } from './tool-defaults';
+import type { ActionSpaceItem, BaseAgent, ToolCliMetadata, ToolDefinition, ToolSchema } from './types';
+import { composeUserPrompt } from './user-prompt';
+export { composeUserPrompt };
+/**
+ * Converts DeviceAction from actionSpace into ToolDefinition.
+ * This is the core logic that removes need for hardcoded tool definitions
+ */
+export declare function generateToolsFromActionSpace(actionSpace: ActionSpaceItem[], getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, sanitizeArgs?: (args: Record<string, unknown>) => Record<string, unknown>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata, toolDefaults?: ToolDefaults): ToolDefinition[];
+/**
+ * Generate common tools (screenshot, act)
+ */
+export declare function generateCommonTools(getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata, toolDefaults?: ToolDefaults): ToolDefinition[];

package/dist/types/agent-tools/types.d.ts

@@ -0,0 +1,213 @@
+import type { z } from 'zod';
+import type { ToolDefaults } from './tool-defaults';
+/**
+ * Default timeout constants for app loading verification
+ */
+export declare const defaultAppLoadingTimeoutMs = 10000;
+export declare const defaultAppLoadingCheckIntervalMs = 2000;
+/**
+ * Content item types for tool results.
+ */
+export type ToolResultContent = {
+    type: 'text';
+    text: string;
+} | {
+    type: 'image';
+    data: string;
+    mimeType: string;
+} | {
+    type: 'audio';
+    data: string;
+    mimeType: string;
+} | {
+    type: 'resource';
+    resource: {
+        text: string;
+        uri: string;
+        mimeType?: string;
+    } | {
+        uri: string;
+        blob: string;
+        mimeType?: string;
+    };
+};
+/**
+ * Result type for tool execution.
+ */
+export interface ToolResult {
+    [x: string]: unknown;
+    content: ToolResultContent[];
+    isError?: boolean;
+    _meta?: Record<string, unknown>;
+}
+/**
+ * Tool handler function type
+ * Takes parsed arguments and returns a tool result
+ */
+export type ToolHandler<T = Record<string, unknown>> = (args: T) => Promise<ToolResult>;
+/**
+ * Tool schema type using Zod
+ */
+export type ToolSchema = Record<string, z.ZodTypeAny>;
+export interface ToolCliOption {
+    preferredName?: string;
+    aliases?: string[];
+    hidden?: boolean;
+}
+export interface ToolCliMetadata {
+    options?: Record<string, ToolCliOption>;
+    /** Schema keys populated from leading positional CLI arguments. */
+    positionals?: string[];
+}
+/**
+ * Tool definition for Midscene CLI and Skill surfaces.
+ */
+export interface ToolDefinition<T = Record<string, unknown>> {
+    name: string;
+    description: string;
+    schema: ToolSchema;
+    handler: ToolHandler<T>;
+    cli?: ToolCliMetadata;
+}
+export type Tool = ToolDefinition;
+/**
+ * Action space item definition
+ * Note: Intentionally no index signature to maintain compatibility with DeviceAction
+ */
+export interface ActionSpaceItem {
+    name: string;
+    description?: string;
+    args?: Record<string, unknown>;
+    paramSchema?: z.ZodTypeAny;
+}
+/**
+ * Structural shape compatible with @aiscene/core `TUserPrompt`.
+ * Declared locally to avoid a circular dep on `@aiscene/core` from `@aiscene/shared`.
+ *
+ * Consumed by the `assert` and `act` tools in `generateCommonTools`, both of
+ * which forward reference images to core (`aiAssert` / `aiAct`). `aiWaitFor`
+ * stays string-only at the CLI surface because the tools generator does not
+ * yet expose a multimodal entry point for it.
+ */
+export type UserPromptLike = string | {
+    prompt: string;
+    images?: Array<{
+        name: string;
+        url: string;
+    }>;
+    convertHttpImage2Base64?: boolean;
+};
+export interface RecordToReportScreenshot {
+    /**
+     * PNG/JPEG data URI, or raw PNG base64 body.
+     */
+    base64: string;
+    description?: string;
+}
+export interface RecordToReportOptions {
+    content?: string;
+    /**
+     * @deprecated Use `screenshots: [{ base64 }]` instead.
+     */
+    screenshotBase64?: string;
+    screenshots?: RecordToReportScreenshot[];
+}
+/** Generic progress-bus envelope as seen by tool consumers (all untrusted). */
+export interface BaseAgentProgressEvent {
+    scope?: unknown;
+    phase?: unknown;
+    sequence?: unknown;
+    data?: unknown;
+}
+/** A single frame in a portable UI observation record. */
+export interface UIObservationFrame {
+    /** Path to the captured image. Serialized manifests use relative paths. */
+    path: string;
+    /** MIME type of the captured image file. */
+    mimeType: 'image/png' | 'image/jpeg';
+    /** Capture timestamp in milliseconds. */
+    capturedAt: number;
+}
+/**
+ * Serializable observation window that can be persisted and loaded later.
+ * Frames are ordered from earliest to latest; the final frame represents the
+ * UI state at the end of the observation window.
+ */
+export interface UIObservationRecord {
+    type: 'midscene_ui_observation';
+    version: 1;
+    /** Time when observation sampling started. */
+    startedAt: number;
+    /** Time when the final representative frame was captured. */
+    endedAt: number;
+    frames: UIObservationFrame[];
+    shotSize: {
+        width: number;
+        height: number;
+    };
+    shrunkShotToLogicalRatio: number;
+}
+/** Fixed observation window consumed by shared CLI surfaces. */
+export interface BaseUIObservation {
+    readonly frameCount: number;
+    readonly startedAt: number;
+    readonly endedAt: number;
+    aiAssert(assertion: UserPromptLike, message?: string, options?: Record<string, unknown>): Promise<unknown>;
+    /** Release any resources owned by this observation. */
+    dispose?(): Promise<void>;
+}
+/** Minimal UI observation lifecycle required by shared tool surfaces. */
+export interface BaseUIObserver {
+    readonly bufferedFrameCount: number;
+    /** Stop sampling and return the fixed observed window. */
+    stop(): Promise<BaseUIObservation>;
+    /** Release temporary backing files if observation does not complete. */
+    dispose?(): Promise<void>;
+}
+/** Options for {@link BaseAgent.startObserving}. */
+export interface BaseUIObserverOptions {
+    /** Sampling interval in milliseconds. Defaults to 1000; minimum 200. */
+    intervalMs?: number;
+    /** Maximum number of buffered frames. Defaults to 30; minimum 2. */
+    maxFrames?: number;
+    /** Auto-stop timeout in milliseconds. Defaults to 300000; 0 disables it. */
+    watchdogMs?: number;
+}
+/**
+ * Base agent interface
+ * Represents a platform-specific agent (Android, iOS, Web)
+ * Note: Return types use `unknown` for compatibility with platform-specific implementations
+ */
+export interface BaseAgent {
+    getActionSpace(): Promise<ActionSpaceItem[]>;
+    destroy?(): Promise<void>;
+    reportFile?: string | null;
+    page?: {
+        screenshotBase64(): Promise<string>;
+    };
+    addDumpUpdateListener?(listener: (dump: string, executionDump?: unknown) => void): () => void;
+    addProgressListener?(listener: (event: BaseAgentProgressEvent) => void): () => void;
+    recordToReport?(title?: string, opt?: RecordToReportOptions): Promise<void>;
+    callActionInActionSpace?(actionName: string, params?: unknown): Promise<unknown>;
+    aiAction?(description: UserPromptLike, params?: Record<string, unknown>): Promise<unknown>;
+    aiWaitFor?(assertion: string, options: Record<string, unknown>): Promise<unknown>;
+    aiAssert?(assertion: UserPromptLike, msg?: string, options?: Record<string, unknown>): Promise<unknown>;
+    /** Start a UI observation window and capture its baseline frame. */
+    startObserving?(options?: BaseUIObserverOptions): Promise<BaseUIObserver>;
+}
+/**
+ * Base device interface for temporary device instances
+ */
+export interface BaseDevice {
+    actionSpace(): ActionSpaceItem[];
+    destroy?(): Promise<void>;
+}
+/**
+ * Interface for platform-specific tools manager.
+ */
+export interface IMidsceneTools {
+    initTools(): Promise<void>;
+    destroy?(): Promise<void>;
+    getCliToolDefinitions?(): ToolDefinition[];
+    setToolDefaults?(toolDefaults: ToolDefaults): void;
+}

package/dist/types/agent-tools/user-prompt.d.ts

@@ -0,0 +1,13 @@
+import { z } from 'zod';
+import type { UserPromptLike } from './types';
+export declare function composeUserPrompt(input: {
+    prompt: string;
+    image?: unknown;
+    imageName?: unknown;
+    convertHttpImage2Base64?: unknown;
+}): UserPromptLike;
+export declare const promptInputExtraSchema: {
+    image: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
+    imageName: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
+    convertHttpImage2Base64: z.ZodOptional<z.ZodUnion<[z.ZodBoolean, z.ZodString]>>;
+};

package/dist/types/cli/interrupt.d.ts

@@ -0,0 +1,49 @@
+export type CliInterruptReason = 'sigint' | 'sigterm' | 'sighup' | 'watchdog';
+type CliInterruptSignal = 'SIGINT' | 'SIGTERM' | 'SIGHUP';
+export interface CliInterruptSource {
+    on(event: CliInterruptSignal, listener: () => void): unknown;
+    removeListener(event: CliInterruptSignal, listener: () => void): unknown;
+}
+export interface CliInterruptInputSource {
+    readonly isTTY?: boolean;
+    readonly isRaw?: boolean;
+    readonly readableFlowing?: boolean | null;
+    on(event: 'data', listener: (chunk: unknown) => void): unknown;
+    removeListener(event: 'data', listener: (chunk: unknown) => void): unknown;
+    setRawMode?(enabled: boolean): unknown;
+    pause?(): unknown;
+}
+export interface CliInterruptWaiter {
+    /** Resolves on the first stop signal or watchdog timeout. */
+    readonly result: Promise<CliInterruptReason>;
+    /** Release signal handlers after asynchronous finalization has completed. */
+    dispose(): void;
+}
+export interface CliInterruptWaiterOptions {
+    source?: CliInterruptSource;
+    input?: CliInterruptInputSource;
+    /** Called after restoring the terminal when Ctrl+C is pressed again. */
+    forceExit?: (exitCode: number) => void;
+}
+/** Whether a foreground CLI command is currently waiting for this source. */
+export declare function hasActiveCliInterruptWaiter(source?: CliInterruptSource): boolean;
+/**
+ * Keep graceful-stop handlers installed until the caller has finished saving.
+ *
+ * Package runners such as pnpm can deliver SIGINT to the foreground child,
+ * immediately follow it with SIGTERM, then cause SIGHUP when the runner exits
+ * and its pseudo-terminal closes. Resolving on the first signal is not enough:
+ * removing any handler at that point lets a subsequent signal kill the child
+ * during asynchronous artifact finalization.
+ *
+ * On a TTY, Ctrl+C is captured as raw input so the package runner itself stays
+ * alive until the child has saved and restored the terminal. Signal handlers
+ * remain as the graceful-stop path for externally delivered termination.
+ */
+export declare function createCliInterruptWaiter(watchdogMs: number, options?: CliInterruptWaiterOptions): CliInterruptWaiter;
+/**
+ * Wait for one stop request and release the handlers immediately afterwards.
+ * Long-running finalizers should use {@link createCliInterruptWaiter} instead.
+ */
+export declare function waitForCliInterrupt(watchdogMs: number, source?: CliInterruptSource, input?: CliInterruptInputSource | undefined): Promise<CliInterruptReason>;
+export {};

package/dist/types/cli/record-command.d.ts

@@ -0,0 +1,3 @@
+import type { BaseAgent, ToolCliMetadata, ToolDefinition, ToolSchema } from '../agent-tools/types';
+/** Create the foreground recording command exposed only by Midscene CLIs. */
+export declare function createRecordCliCommand(getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata): ToolDefinition;

package/dist/types/cli/screenshot-file.d.ts

@@ -0,0 +1,10 @@
+export interface WriteCliScreenshotFileOptions {
+    id?: unknown;
+    mimeType?: unknown;
+    extension?: unknown;
+    directoryPath?: string;
+    directoryName?: string;
+    filenamePrefix?: string;
+    overwrite?: boolean;
+}
+export declare function writeCliScreenshotFile(rawBase64: string, options?: WriteCliScreenshotFileOptions): string;

package/dist/types/cli/verbose-ai-act.d.ts

@@ -0,0 +1,44 @@
+import { type CliVerboseScreenshotCollectOptions } from './verbose-screenshot';
+export interface CliVerboseLine {
+    key: string;
+    text: string;
+}
+/** Generic progress-bus envelope as seen by the CLI (all fields untrusted). */
+export interface CliAgentProgressEvent {
+    scope?: unknown;
+    phase?: unknown;
+    sequence?: unknown;
+    data?: unknown;
+}
+/** Flattened, render-ready aiAct payload derived from the envelope. */
+export interface CliAiActProgressPayload {
+    phase?: unknown;
+    sequence?: unknown;
+    prompt?: unknown;
+    planIndex?: unknown;
+    planLimit?: unknown;
+    action?: unknown;
+    thought?: unknown;
+    log?: unknown;
+    output?: unknown;
+    durationMs?: unknown;
+    error?: unknown;
+    screenshots?: Array<Record<string, unknown>>;
+    screenshotPath?: string;
+    [key: string]: unknown;
+}
+/** aiAct scope tag on the progress bus (mirrors core's `aiActProgressScope`). */
+export declare const cliAiActProgressScope = "aiAct";
+export declare function normalizeAiActProgressEventForCli(event: unknown, screenshotOptions?: CliVerboseScreenshotCollectOptions): CliAiActProgressPayload | undefined;
+export declare function buildAiActProgressEventLines(event: CliAiActProgressPayload): CliVerboseLine[];
+/**
+ * A producer-specific renderer for the generic CLI progress dispatcher: turn a
+ * raw progress envelope into a normalized payload, then that payload into
+ * verbose lines. New producers register their own renderer; the dispatcher core
+ * stays untouched.
+ */
+export interface CliProgressRenderer {
+    normalize(event: unknown, screenshotOptions: CliVerboseScreenshotCollectOptions): Record<string, unknown> | undefined;
+    buildLines(payload: Record<string, unknown>): CliVerboseLine[];
+}
+export declare const aiActCliProgressRenderer: CliProgressRenderer;

package/dist/types/cli/verbose-screenshot.d.ts

@@ -0,0 +1,10 @@
+export type CliVerboseScreenshotExportMode = 'none' | 'tmp' | 'report';
+export interface CliVerboseScreenshotCollectOptions {
+    reportFile?: unknown;
+    exportMode?: CliVerboseScreenshotExportMode;
+    cache?: Map<string, string>;
+}
+export declare function collectScreenshotRefs(value: unknown, options?: CliVerboseScreenshotCollectOptions): Array<Record<string, unknown>>;
+export declare function pathForReportScreenshot(path: string, reportFile?: unknown): string;
+export declare function latestScreenshotPathForAiAct(value: unknown, options?: CliVerboseScreenshotCollectOptions): string;
+export declare function renderScreenshotList(screenshots: unknown): string;

package/dist/types/cli/verbose.d.ts

@@ -0,0 +1,40 @@
+export declare const cliVerboseFlag = "verbose";
+export type CliVerboseFormat = 'text' | 'jsonl';
+export interface CliVerboseContext {
+    enabled: boolean;
+    format?: CliVerboseFormat;
+    scriptName?: string;
+    commandName?: string;
+    startedAt?: number;
+    renderedLineKeys?: Set<string>;
+}
+export interface CliVerboseEvent {
+    event: string;
+    scriptName?: string;
+    command?: string;
+    status?: 'ok' | 'error';
+    durationMs?: number;
+    error?: string;
+    [key: string]: unknown;
+}
+type DumpUpdateAgent = {
+    addDumpUpdateListener?: (listener: (dump: string, executionDump?: unknown) => void) => () => void;
+    addProgressListener?: (listener: (event: unknown) => void) => () => void;
+    reportFile?: string | null;
+};
+export declare function stripVerboseFlag(argv: readonly string[]): {
+    rawArgs: string[];
+    verbose: boolean;
+    format: CliVerboseFormat;
+};
+export declare function withCliVerboseContext<T>(context: CliVerboseContext, fn: () => Promise<T>): Promise<T>;
+export declare function getCliVerboseContext(): CliVerboseContext;
+export declare function isCliVerboseEnabled(): boolean;
+export declare function emitCliVerboseEvent(event: CliVerboseEvent): void;
+declare function errorMessageOf(error: unknown): string;
+export declare function compactCliVerboseValue(value: unknown): unknown;
+export declare function compactCliVerboseArgs(args: Record<string, unknown>): Record<string, unknown>;
+export declare function attachCliVerboseDumpListener(agent: DumpUpdateAgent, options?: {
+    toolName?: string;
+}): () => void;
+export { errorMessageOf as cliVerboseErrorMessage };

package/dist/types/env/types.d.ts

@@ -6,6 +6,7 @@ export declare const MIDSCENE_DEBUG_MODEL_RESPONSE = "MIDSCENE_DEBUG_MODEL_RESPO
 export declare const MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG = "MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG";
 export declare const MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
 export declare const MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
+export declare const MIDSCENE_CHROME_PATH = "MIDSCENE_CHROME_PATH";
 export declare const MIDSCENE_MCP_CHROME_PATH = "MIDSCENE_MCP_CHROME_PATH";
 export declare const MIDSCENE_MCP_ANDROID_MODE = "MIDSCENE_MCP_ANDROID_MODE";
 export declare const DOCKER_CONTAINER = "DOCKER_CONTAINER";
@@ -110,20 +111,20 @@ export declare const UNUSED_ENV_KEYS: string[];
 export declare const BASIC_ENV_KEYS: readonly ["MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR"];
 export declare const BOOLEAN_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET"];
 export declare const NUMBER_ENV_KEYS: readonly ["MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT"];
-export declare const STRING_ENV_KEYS: readonly ["MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
+export declare const STRING_ENV_KEYS: readonly ["MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
 /**
  * Non model related env keys, used for globally controlling the behavior of midscene
  * Can not be override by agent.modelConfig but can be override by overrideAIConfig
  * Can be access at any time
  */
-export declare const GLOBAL_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
+export declare const GLOBAL_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
 /**
  * Model related eve keys, used for declare which model to use.
  * Can be override by both agent.modelConfig and overrideAIConfig
  * Can only be access after agent.constructor
  */
 export declare const MODEL_ENV_KEYS: readonly ["MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
-export declare const ALL_ENV_KEYS: readonly [...string[], "MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER", "MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
+export declare const ALL_ENV_KEYS: readonly [...string[], "MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER", "MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
 export type TEnvKeys = (typeof ALL_ENV_KEYS)[number];
 export type TGlobalConfig = Record<TEnvKeys, string | undefined>;
 /**

package/dist/types/img/info.d.ts

@@ -1,6 +1,8 @@
+import { Buffer } from 'node:buffer';
 import type { Size } from '../types';
 export interface ImageInfo extends Size {
 }
+export declare function encodedImageInfoOfBuffer(imageBuffer: Buffer): ImageInfo;
 /**
  * Retrieves the dimensions of an image from a base64-encoded string
  *

package/dist/types/img/photon-loader.d.ts

@@ -0,0 +1,2 @@
+export type PhotonModule = typeof import('@silvia-odwyer/photon');
+export declare function loadPhotonModule(): Promise<PhotonModule>;

package/dist/types/mcp/types.d.ts

@@ -95,6 +95,7 @@ export interface BaseAgent {
     aiAction?: (description: string, params?: Record<string, unknown>) => Promise<unknown>;
     aiWaitFor?: (assertion: string, options: Record<string, unknown>) => Promise<unknown>;
     aiAssert?: (assertion: string, msg?: string, options?: Record<string, unknown>) => Promise<unknown>;
+    aiQuery?: (demand: any, opt?: any) => Promise<any>;
 }
 /**
  * Base device interface for temporary device instances

package/dist/types/recorder.d.ts

@@ -0,0 +1,113 @@
+export type MidsceneRecorderEventType = 'click' | 'drag' | 'scroll' | 'input' | 'navigation' | 'setViewport' | 'keydown';
+export type MidsceneRecorderSourceKind = 'studio-preview' | 'unsupported' | (string & {});
+export type MidsceneRecorderPlatformId = 'web' | 'android' | 'ios' | 'computer' | 'harmony' | (string & {});
+export interface MidsceneRecorderElementRect {
+    left?: number;
+    top?: number;
+    width?: number;
+    height?: number;
+    x?: number;
+    y?: number;
+}
+export interface MidsceneRecorderPageInfo {
+    width: number;
+    height: number;
+}
+/**
+ * A screenshot stored outside the recording event payload.
+ *
+ * Recorder events are persisted in the Studio renderer. Keeping full data
+ * URLs there makes long recordings retain every screenshot in the renderer
+ * heap, so screenshot bytes live in the Playground run directory instead.
+ */
+export interface MidsceneRecorderScreenshotAssetRef {
+    id: string;
+    mimeType: string;
+    bytes: number;
+}
+export interface MidsceneRecorderEvent {
+    type: MidsceneRecorderEventType;
+    source?: MidsceneRecorderSourceKind;
+    actionType?: string;
+    rawPayload?: Record<string, unknown>;
+    url?: string;
+    title?: string;
+    value?: string;
+    elementRect?: MidsceneRecorderElementRect;
+    pageInfo: MidsceneRecorderPageInfo;
+    screenshotBefore?: string;
+    screenshotAfter?: string;
+    /** The single screenshot retained for AI description and Markdown export. */
+    screenshotAsset?: MidsceneRecorderScreenshotAssetRef;
+    semantic?: MidsceneRecorderSemantic;
+    elementDescription?: string;
+    descriptionLoading?: boolean;
+    screenshotWithBox?: string;
+    timestamp: number;
+    hashId: string;
+    mergedHashIds?: string[];
+}
+export type MidsceneRecorderSemanticSource = 'aiDescribe' | 'recorderAI' | 'heuristic';
+export type MidsceneRecorderSemanticStatus = 'pending' | 'ready' | 'failed';
+export type MidsceneRecorderSemanticConfidence = 'high' | 'medium' | 'low';
+export interface MidsceneRecorderSemanticAiDescribe {
+    verifyPrompt: boolean;
+    verifyPassed?: boolean;
+    deepLocate?: boolean;
+    centerDistance?: number;
+    expectedCenter?: [number, number];
+    actualCenter?: [number, number];
+    annotatedScreenshotPath?: string;
+}
+export interface MidsceneRecorderSemantic {
+    source: MidsceneRecorderSemanticSource;
+    status: MidsceneRecorderSemanticStatus;
+    elementDescription?: string;
+    replayInstruction?: string;
+    actionSummary?: string;
+    confidence?: MidsceneRecorderSemanticConfidence;
+    error?: string;
+    aiDescribe?: MidsceneRecorderSemanticAiDescribe;
+    fallbackFrom?: MidsceneRecorderSemantic;
+}
+export interface MidsceneRecorderSemanticAction {
+    type: MidsceneRecorderEventType;
+    actionType?: string;
+    value?: string;
+    url?: string;
+    scrollDestinationDescription?: string;
+}
+export interface MidsceneRecorderTarget {
+    platformId: MidsceneRecorderPlatformId;
+    deviceId?: string;
+    label?: string;
+    values: Record<string, string | number | boolean>;
+}
+export interface MidsceneRecorderGeneratedCode {
+    markdown?: string;
+    yaml?: string;
+    playwright?: string;
+    updatedAt?: number;
+}
+export interface MidsceneRecorderMarkdownScreenshotAsset {
+    eventIndex: number;
+    eventHashId: string;
+    eventType: MidsceneRecorderEventType;
+    relativePath: string;
+    dataUrl: string;
+    base64Data: string;
+    mimeType: string;
+}
+export interface MidsceneRecorderMarkdownScreenshotOptions {
+    baseDir?: string;
+    maxScreenshots?: number;
+}
+export declare const DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS = 20;
+export declare function getMidsceneRecorderSemantic(event: Pick<MidsceneRecorderEvent, 'semantic'>): MidsceneRecorderSemantic | undefined;
+export declare function buildMidsceneRecorderReplayInstruction(event: MidsceneRecorderSemanticAction, elementDescription: string): string;
+export declare function buildMidsceneRecorderActionSummary(event: MidsceneRecorderSemanticAction, elementDescription: string): string;
+export declare function getMidsceneRecorderEventDescription(event: MidsceneRecorderEvent): string;
+export declare function getMidsceneRecorderScreenshotsForLLM(events: MidsceneRecorderEvent[], maxScreenshots?: number): string[];
+export declare function sanitizeMidsceneRecorderFileName(value: string): string;
+export declare function createMidsceneRecorderMarkdownScreenshotAssets(events: MidsceneRecorderEvent[], options?: MidsceneRecorderMarkdownScreenshotOptions): MidsceneRecorderMarkdownScreenshotAsset[];
+export declare function stringifyMidsceneRecorderTargetBlock(target: MidsceneRecorderTarget): string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiscene/shared",
-  "version": "8.0.3",
+  "version": "8.0.4",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",