@aiscene/shared 8.0.2 → 8.0.4

package/dist/types/agent-tools/tool-generator.d.ts

@@ -0,0 +1,13 @@
+import type { ToolDefaults } from './tool-defaults';
+import type { ActionSpaceItem, BaseAgent, ToolCliMetadata, ToolDefinition, ToolSchema } from './types';
+import { composeUserPrompt } from './user-prompt';
+export { composeUserPrompt };
+/**
+ * Converts DeviceAction from actionSpace into ToolDefinition.
+ * This is the core logic that removes need for hardcoded tool definitions
+ */
+export declare function generateToolsFromActionSpace(actionSpace: ActionSpaceItem[], getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, sanitizeArgs?: (args: Record<string, unknown>) => Record<string, unknown>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata, toolDefaults?: ToolDefaults): ToolDefinition[];
+/**
+ * Generate common tools (screenshot, act)
+ */
+export declare function generateCommonTools(getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata, toolDefaults?: ToolDefaults): ToolDefinition[];

package/dist/types/agent-tools/types.d.ts

@@ -0,0 +1,213 @@
+import type { z } from 'zod';
+import type { ToolDefaults } from './tool-defaults';
+/**
+ * Default timeout constants for app loading verification
+ */
+export declare const defaultAppLoadingTimeoutMs = 10000;
+export declare const defaultAppLoadingCheckIntervalMs = 2000;
+/**
+ * Content item types for tool results.
+ */
+export type ToolResultContent = {
+    type: 'text';
+    text: string;
+} | {
+    type: 'image';
+    data: string;
+    mimeType: string;
+} | {
+    type: 'audio';
+    data: string;
+    mimeType: string;
+} | {
+    type: 'resource';
+    resource: {
+        text: string;
+        uri: string;
+        mimeType?: string;
+    } | {
+        uri: string;
+        blob: string;
+        mimeType?: string;
+    };
+};
+/**
+ * Result type for tool execution.
+ */
+export interface ToolResult {
+    [x: string]: unknown;
+    content: ToolResultContent[];
+    isError?: boolean;
+    _meta?: Record<string, unknown>;
+}
+/**
+ * Tool handler function type
+ * Takes parsed arguments and returns a tool result
+ */
+export type ToolHandler<T = Record<string, unknown>> = (args: T) => Promise<ToolResult>;
+/**
+ * Tool schema type using Zod
+ */
+export type ToolSchema = Record<string, z.ZodTypeAny>;
+export interface ToolCliOption {
+    preferredName?: string;
+    aliases?: string[];
+    hidden?: boolean;
+}
+export interface ToolCliMetadata {
+    options?: Record<string, ToolCliOption>;
+    /** Schema keys populated from leading positional CLI arguments. */
+    positionals?: string[];
+}
+/**
+ * Tool definition for Midscene CLI and Skill surfaces.
+ */
+export interface ToolDefinition<T = Record<string, unknown>> {
+    name: string;
+    description: string;
+    schema: ToolSchema;
+    handler: ToolHandler<T>;
+    cli?: ToolCliMetadata;
+}
+export type Tool = ToolDefinition;
+/**
+ * Action space item definition
+ * Note: Intentionally no index signature to maintain compatibility with DeviceAction
+ */
+export interface ActionSpaceItem {
+    name: string;
+    description?: string;
+    args?: Record<string, unknown>;
+    paramSchema?: z.ZodTypeAny;
+}
+/**
+ * Structural shape compatible with @aiscene/core `TUserPrompt`.
+ * Declared locally to avoid a circular dep on `@aiscene/core` from `@aiscene/shared`.
+ *
+ * Consumed by the `assert` and `act` tools in `generateCommonTools`, both of
+ * which forward reference images to core (`aiAssert` / `aiAct`). `aiWaitFor`
+ * stays string-only at the CLI surface because the tools generator does not
+ * yet expose a multimodal entry point for it.
+ */
+export type UserPromptLike = string | {
+    prompt: string;
+    images?: Array<{
+        name: string;
+        url: string;
+    }>;
+    convertHttpImage2Base64?: boolean;
+};
+export interface RecordToReportScreenshot {
+    /**
+     * PNG/JPEG data URI, or raw PNG base64 body.
+     */
+    base64: string;
+    description?: string;
+}
+export interface RecordToReportOptions {
+    content?: string;
+    /**
+     * @deprecated Use `screenshots: [{ base64 }]` instead.
+     */
+    screenshotBase64?: string;
+    screenshots?: RecordToReportScreenshot[];
+}
+/** Generic progress-bus envelope as seen by tool consumers (all untrusted). */
+export interface BaseAgentProgressEvent {
+    scope?: unknown;
+    phase?: unknown;
+    sequence?: unknown;
+    data?: unknown;
+}
+/** A single frame in a portable UI observation record. */
+export interface UIObservationFrame {
+    /** Path to the captured image. Serialized manifests use relative paths. */
+    path: string;
+    /** MIME type of the captured image file. */
+    mimeType: 'image/png' | 'image/jpeg';
+    /** Capture timestamp in milliseconds. */
+    capturedAt: number;
+}
+/**
+ * Serializable observation window that can be persisted and loaded later.
+ * Frames are ordered from earliest to latest; the final frame represents the
+ * UI state at the end of the observation window.
+ */
+export interface UIObservationRecord {
+    type: 'midscene_ui_observation';
+    version: 1;
+    /** Time when observation sampling started. */
+    startedAt: number;
+    /** Time when the final representative frame was captured. */
+    endedAt: number;
+    frames: UIObservationFrame[];
+    shotSize: {
+        width: number;
+        height: number;
+    };
+    shrunkShotToLogicalRatio: number;
+}
+/** Fixed observation window consumed by shared CLI surfaces. */
+export interface BaseUIObservation {
+    readonly frameCount: number;
+    readonly startedAt: number;
+    readonly endedAt: number;
+    aiAssert(assertion: UserPromptLike, message?: string, options?: Record<string, unknown>): Promise<unknown>;
+    /** Release any resources owned by this observation. */
+    dispose?(): Promise<void>;
+}
+/** Minimal UI observation lifecycle required by shared tool surfaces. */
+export interface BaseUIObserver {
+    readonly bufferedFrameCount: number;
+    /** Stop sampling and return the fixed observed window. */
+    stop(): Promise<BaseUIObservation>;
+    /** Release temporary backing files if observation does not complete. */
+    dispose?(): Promise<void>;
+}
+/** Options for {@link BaseAgent.startObserving}. */
+export interface BaseUIObserverOptions {
+    /** Sampling interval in milliseconds. Defaults to 1000; minimum 200. */
+    intervalMs?: number;
+    /** Maximum number of buffered frames. Defaults to 30; minimum 2. */
+    maxFrames?: number;
+    /** Auto-stop timeout in milliseconds. Defaults to 300000; 0 disables it. */
+    watchdogMs?: number;
+}
+/**
+ * Base agent interface
+ * Represents a platform-specific agent (Android, iOS, Web)
+ * Note: Return types use `unknown` for compatibility with platform-specific implementations
+ */
+export interface BaseAgent {
+    getActionSpace(): Promise<ActionSpaceItem[]>;
+    destroy?(): Promise<void>;
+    reportFile?: string | null;
+    page?: {
+        screenshotBase64(): Promise<string>;
+    };
+    addDumpUpdateListener?(listener: (dump: string, executionDump?: unknown) => void): () => void;
+    addProgressListener?(listener: (event: BaseAgentProgressEvent) => void): () => void;
+    recordToReport?(title?: string, opt?: RecordToReportOptions): Promise<void>;
+    callActionInActionSpace?(actionName: string, params?: unknown): Promise<unknown>;
+    aiAction?(description: UserPromptLike, params?: Record<string, unknown>): Promise<unknown>;
+    aiWaitFor?(assertion: string, options: Record<string, unknown>): Promise<unknown>;
+    aiAssert?(assertion: UserPromptLike, msg?: string, options?: Record<string, unknown>): Promise<unknown>;
+    /** Start a UI observation window and capture its baseline frame. */
+    startObserving?(options?: BaseUIObserverOptions): Promise<BaseUIObserver>;
+}
+/**
+ * Base device interface for temporary device instances
+ */
+export interface BaseDevice {
+    actionSpace(): ActionSpaceItem[];
+    destroy?(): Promise<void>;
+}
+/**
+ * Interface for platform-specific tools manager.
+ */
+export interface IMidsceneTools {
+    initTools(): Promise<void>;
+    destroy?(): Promise<void>;
+    getCliToolDefinitions?(): ToolDefinition[];
+    setToolDefaults?(toolDefaults: ToolDefaults): void;
+}

package/dist/types/agent-tools/user-prompt.d.ts

@@ -0,0 +1,13 @@
+import { z } from 'zod';
+import type { UserPromptLike } from './types';
+export declare function composeUserPrompt(input: {
+    prompt: string;
+    image?: unknown;
+    imageName?: unknown;
+    convertHttpImage2Base64?: unknown;
+}): UserPromptLike;
+export declare const promptInputExtraSchema: {
+    image: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
+    imageName: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
+    convertHttpImage2Base64: z.ZodOptional<z.ZodUnion<[z.ZodBoolean, z.ZodString]>>;
+};

package/dist/types/cli/cli-args.d.ts

@@ -0,0 +1,8 @@
+import type { ToolCliOption, ToolDefinition } from '../mcp/types';
+export declare function parseValue(raw: string): unknown;
+export declare function parseCliArgs(args: string[]): Record<string, unknown>;
+export declare function getCliOptionDisplay(key: string, cliOption?: ToolCliOption): {
+    label: string;
+    aliases: string[];
+};
+export declare function formatCliValidationError(scriptName: string, commandName: string, def: ToolDefinition, rawArgs: Record<string, unknown>): string | undefined;

package/dist/types/cli/cli-error.d.ts

@@ -0,0 +1,5 @@
+export declare class CLIError extends Error {
+    exitCode: number;
+    constructor(message: string, exitCode?: number);
+}
+export declare function reportCLIError(error: unknown, log?: (message?: unknown, ...optionalParams: unknown[]) => void): number;

package/dist/types/cli/cli-runner.d.ts

@@ -12,11 +12,8 @@ export interface CLIRunnerOptions {
     version?: string;
     extraCommands?: CLIExtraCommand[];
 }
-export declare class CLIError extends Error {
-    exitCode: number;
-    constructor(message: string, exitCode?: number);
-}
-export declare function parseValue(raw: string): unknown;
-export declare function parseCliArgs(args: string[]): Record<string, unknown>;
+export { parseCliArgs, parseValue } from './cli-args';
+export { CLIError, reportCLIError } from './cli-error';
 export declare function removePrefix(name: string, prefix?: string): string;
-export declare function runToolsCLI(tools: BaseMidsceneTools, scriptName: string, options?: CLIRunnerOptions): Promise<void>;
+type AnyMidsceneTools = BaseMidsceneTools<any, any>;
+export declare function runToolsCLI(tools: AnyMidsceneTools, scriptName: string, options?: CLIRunnerOptions): Promise<void>;

package/dist/types/cli/index.d.ts

@@ -1,2 +1,4 @@
-export { runToolsCLI, CLIError, parseValue, parseCliArgs, removePrefix, } from './cli-runner';
+export { CLIError, reportCLIError } from './cli-error';
+export { parseCliArgs, parseValue } from './cli-args';
+export { runToolsCLI, removePrefix } from './cli-runner';
 export type { CLIRunnerOptions, CLIExtraCommand } from './cli-runner';

package/dist/types/cli/interrupt.d.ts

@@ -0,0 +1,49 @@
+export type CliInterruptReason = 'sigint' | 'sigterm' | 'sighup' | 'watchdog';
+type CliInterruptSignal = 'SIGINT' | 'SIGTERM' | 'SIGHUP';
+export interface CliInterruptSource {
+    on(event: CliInterruptSignal, listener: () => void): unknown;
+    removeListener(event: CliInterruptSignal, listener: () => void): unknown;
+}
+export interface CliInterruptInputSource {
+    readonly isTTY?: boolean;
+    readonly isRaw?: boolean;
+    readonly readableFlowing?: boolean | null;
+    on(event: 'data', listener: (chunk: unknown) => void): unknown;
+    removeListener(event: 'data', listener: (chunk: unknown) => void): unknown;
+    setRawMode?(enabled: boolean): unknown;
+    pause?(): unknown;
+}
+export interface CliInterruptWaiter {
+    /** Resolves on the first stop signal or watchdog timeout. */
+    readonly result: Promise<CliInterruptReason>;
+    /** Release signal handlers after asynchronous finalization has completed. */
+    dispose(): void;
+}
+export interface CliInterruptWaiterOptions {
+    source?: CliInterruptSource;
+    input?: CliInterruptInputSource;
+    /** Called after restoring the terminal when Ctrl+C is pressed again. */
+    forceExit?: (exitCode: number) => void;
+}
+/** Whether a foreground CLI command is currently waiting for this source. */
+export declare function hasActiveCliInterruptWaiter(source?: CliInterruptSource): boolean;
+/**
+ * Keep graceful-stop handlers installed until the caller has finished saving.
+ *
+ * Package runners such as pnpm can deliver SIGINT to the foreground child,
+ * immediately follow it with SIGTERM, then cause SIGHUP when the runner exits
+ * and its pseudo-terminal closes. Resolving on the first signal is not enough:
+ * removing any handler at that point lets a subsequent signal kill the child
+ * during asynchronous artifact finalization.
+ *
+ * On a TTY, Ctrl+C is captured as raw input so the package runner itself stays
+ * alive until the child has saved and restored the terminal. Signal handlers
+ * remain as the graceful-stop path for externally delivered termination.
+ */
+export declare function createCliInterruptWaiter(watchdogMs: number, options?: CliInterruptWaiterOptions): CliInterruptWaiter;
+/**
+ * Wait for one stop request and release the handlers immediately afterwards.
+ * Long-running finalizers should use {@link createCliInterruptWaiter} instead.
+ */
+export declare function waitForCliInterrupt(watchdogMs: number, source?: CliInterruptSource, input?: CliInterruptInputSource | undefined): Promise<CliInterruptReason>;
+export {};

package/dist/types/cli/record-command.d.ts

@@ -0,0 +1,3 @@
+import type { BaseAgent, ToolCliMetadata, ToolDefinition, ToolSchema } from '../agent-tools/types';
+/** Create the foreground recording command exposed only by Midscene CLIs. */
+export declare function createRecordCliCommand(getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata): ToolDefinition;

package/dist/types/cli/screenshot-file.d.ts

@@ -0,0 +1,10 @@
+export interface WriteCliScreenshotFileOptions {
+    id?: unknown;
+    mimeType?: unknown;
+    extension?: unknown;
+    directoryPath?: string;
+    directoryName?: string;
+    filenamePrefix?: string;
+    overwrite?: boolean;
+}
+export declare function writeCliScreenshotFile(rawBase64: string, options?: WriteCliScreenshotFileOptions): string;

package/dist/types/cli/verbose-ai-act.d.ts

@@ -0,0 +1,44 @@
+import { type CliVerboseScreenshotCollectOptions } from './verbose-screenshot';
+export interface CliVerboseLine {
+    key: string;
+    text: string;
+}
+/** Generic progress-bus envelope as seen by the CLI (all fields untrusted). */
+export interface CliAgentProgressEvent {
+    scope?: unknown;
+    phase?: unknown;
+    sequence?: unknown;
+    data?: unknown;
+}
+/** Flattened, render-ready aiAct payload derived from the envelope. */
+export interface CliAiActProgressPayload {
+    phase?: unknown;
+    sequence?: unknown;
+    prompt?: unknown;
+    planIndex?: unknown;
+    planLimit?: unknown;
+    action?: unknown;
+    thought?: unknown;
+    log?: unknown;
+    output?: unknown;
+    durationMs?: unknown;
+    error?: unknown;
+    screenshots?: Array<Record<string, unknown>>;
+    screenshotPath?: string;
+    [key: string]: unknown;
+}
+/** aiAct scope tag on the progress bus (mirrors core's `aiActProgressScope`). */
+export declare const cliAiActProgressScope = "aiAct";
+export declare function normalizeAiActProgressEventForCli(event: unknown, screenshotOptions?: CliVerboseScreenshotCollectOptions): CliAiActProgressPayload | undefined;
+export declare function buildAiActProgressEventLines(event: CliAiActProgressPayload): CliVerboseLine[];
+/**
+ * A producer-specific renderer for the generic CLI progress dispatcher: turn a
+ * raw progress envelope into a normalized payload, then that payload into
+ * verbose lines. New producers register their own renderer; the dispatcher core
+ * stays untouched.
+ */
+export interface CliProgressRenderer {
+    normalize(event: unknown, screenshotOptions: CliVerboseScreenshotCollectOptions): Record<string, unknown> | undefined;
+    buildLines(payload: Record<string, unknown>): CliVerboseLine[];
+}
+export declare const aiActCliProgressRenderer: CliProgressRenderer;

package/dist/types/cli/verbose-screenshot.d.ts

@@ -0,0 +1,10 @@
+export type CliVerboseScreenshotExportMode = 'none' | 'tmp' | 'report';
+export interface CliVerboseScreenshotCollectOptions {
+    reportFile?: unknown;
+    exportMode?: CliVerboseScreenshotExportMode;
+    cache?: Map<string, string>;
+}
+export declare function collectScreenshotRefs(value: unknown, options?: CliVerboseScreenshotCollectOptions): Array<Record<string, unknown>>;
+export declare function pathForReportScreenshot(path: string, reportFile?: unknown): string;
+export declare function latestScreenshotPathForAiAct(value: unknown, options?: CliVerboseScreenshotCollectOptions): string;
+export declare function renderScreenshotList(screenshots: unknown): string;

package/dist/types/cli/verbose.d.ts

@@ -0,0 +1,40 @@
+export declare const cliVerboseFlag = "verbose";
+export type CliVerboseFormat = 'text' | 'jsonl';
+export interface CliVerboseContext {
+    enabled: boolean;
+    format?: CliVerboseFormat;
+    scriptName?: string;
+    commandName?: string;
+    startedAt?: number;
+    renderedLineKeys?: Set<string>;
+}
+export interface CliVerboseEvent {
+    event: string;
+    scriptName?: string;
+    command?: string;
+    status?: 'ok' | 'error';
+    durationMs?: number;
+    error?: string;
+    [key: string]: unknown;
+}
+type DumpUpdateAgent = {
+    addDumpUpdateListener?: (listener: (dump: string, executionDump?: unknown) => void) => () => void;
+    addProgressListener?: (listener: (event: unknown) => void) => () => void;
+    reportFile?: string | null;
+};
+export declare function stripVerboseFlag(argv: readonly string[]): {
+    rawArgs: string[];
+    verbose: boolean;
+    format: CliVerboseFormat;
+};
+export declare function withCliVerboseContext<T>(context: CliVerboseContext, fn: () => Promise<T>): Promise<T>;
+export declare function getCliVerboseContext(): CliVerboseContext;
+export declare function isCliVerboseEnabled(): boolean;
+export declare function emitCliVerboseEvent(event: CliVerboseEvent): void;
+declare function errorMessageOf(error: unknown): string;
+export declare function compactCliVerboseValue(value: unknown): unknown;
+export declare function compactCliVerboseArgs(args: Record<string, unknown>): Record<string, unknown>;
+export declare function attachCliVerboseDumpListener(agent: DumpUpdateAgent, options?: {
+    toolName?: string;
+}): () => void;
+export { errorMessageOf as cliVerboseErrorMessage };

package/dist/types/constants/example-code.d.ts

@@ -1,2 +1,2 @@
-export declare const PLAYWRIGHT_EXAMPLE_CODE = "\n// Reference the following code to generate Midscene test cases\n// The following is test code for Midscene AI, for reference\n// The following is Playwright syntax, you can use Playwright to assist in test generation\nIMPORTANT: Follow these exact type signatures for AI functions:\n\n// Type signatures for AI functions:\naiAct(prompt: string, options?: { cacheable?: boolean, deepThink?: 'unset' | true | false }): Promise<void>\naiInput(text: string, locate: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiTap(locate: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiHover(locate: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiDoubleClick(locate: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiKeyboardPress(key: string, locate?: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiScroll(locate: string | undefined, options: {\n  direction?: 'up' | 'down' | 'left' | 'right',\n  scrollType?: 'singleAction' | 'scrollToBottom' | 'scrollToTop' | 'scrollToRight' | 'scrollToLeft',\n  distance?: number | null,\n  deepLocate?: boolean,\n  xpath?: string,\n  cacheable?: boolean\n}): Promise<void>\naiAssert(assertion: string, options?: { errorMessage?: string }): Promise<void>\naiWaitFor(prompt: string, options?: { timeout?: number }): Promise<void>\naiQuery<T>(queryObject: Record<string, string>): Promise<T> // Extracts data from page based on descriptions\n\n// examples:\n// Reference the following code to generate Midscene test cases\n// The following is test code for Midscene AI, for reference\n// The following is Playwright syntax, you can use Playwright to assist in test generation\nimport { test as base } from '@playwright/test';\nimport type { PlayWrightAiFixtureType } from '@midscene/web/playwright';\nimport { PlaywrightAiFixture } from '@midscene/web/playwright';\n\nconst test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture({\n  waitForNetworkIdleTimeout: 2000, // optional, the timeout for waiting for network idle between each action, default is 2000ms\n}));\n\n\ntest.beforeEach(async ({ page }) => {\n  await page.goto('https://www.xxx.com/');\n  await page.setViewportSize({ width: 1920, height: 1080 });\n});\n\ntest('ai shop', async ({\n  aiAct,\n  aiInput,\n  aiAssert,\n  aiQuery,\n  aiKeyboardPress,\n  aiHover,\n  aiTap,\n  aiWaitFor,\n  agentForPage,\n  page,\n}) => {\n  // login\n  await aiAssert('The page shows the login interface');\n  await aiInput('user_name', 'in user name input');\n  await aiInput('password', 'in password input');\n  await aiKeyboardPress('Enter', 'Login Button');\n\n  // check the login success\n  await aiWaitFor('The page shows that the loading is complete');\n  await aiAssert('The current page shows the product detail page');\n\n  // check the product info\n  const dataA = await aiQuery({\n    userInfo: 'User information in the format {name: string}',\n    theFirstProductInfo: 'The first product info in the format {name: string, price: number}',\n  });\n  expect(dataA.theFirstProductInfo.name).toBe('xxx');\n  expect(dataA.theFirstProductInfo.price).toBe(100);\n\n\n  // add to cart\n  await aiTap('click add to cart button');\n  \n  await aiTap('click right top cart icon');\n  await aiAssert('The cart icon shows the number 1');\n});\n";
+export declare const PLAYWRIGHT_EXAMPLE_CODE = "\n// Reference the following code to generate Midscene test cases\n// The following is test code for Midscene AI, for reference\n// The following is Playwright syntax, you can use Playwright to assist in test generation\nIMPORTANT: Follow these exact type signatures for AI functions:\n\n// Type signatures for AI functions:\naiAct(prompt: string, options?: { cacheable?: boolean, deepThink?: 'unset' | true | false }): Promise<void>\naiInput(text: string, locate: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiTap(locate: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiHover(locate: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiDoubleClick(locate: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiKeyboardPress(key: string, locate?: string, options?: { deepLocate?: boolean, xpath?: string, cacheable?: boolean }): Promise<void>\naiScroll(locate: string | undefined, options: {\n  direction?: 'up' | 'down' | 'left' | 'right',\n  scrollType?: 'singleAction' | 'scrollToBottom' | 'scrollToTop' | 'scrollToRight' | 'scrollToLeft',\n  distance?: number | null,\n  deepLocate?: boolean,\n  xpath?: string,\n  cacheable?: boolean\n}): Promise<void>\naiAssert(assertion: string, options?: { errorMessage?: string }): Promise<void>\naiWaitFor(prompt: string, options?: { timeout?: number }): Promise<void>\naiQuery<T>(queryObject: Record<string, string>): Promise<T> // Extracts data from page based on descriptions\n\n// examples:\n// Reference the following code to generate Midscene test cases\n// The following is test code for Midscene AI, for reference\n// The following is Playwright syntax, you can use Playwright to assist in test generation\nimport { test as base } from '@playwright/test';\nimport type { PlayWrightAiFixtureType } from '@aiscene/web/playwright';\nimport { PlaywrightAiFixture } from '@aiscene/web/playwright';\n\nconst test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture({\n  waitForNetworkIdleTimeout: 2000, // optional, the timeout for waiting for network idle between each action, default is 2000ms\n}));\n\n\ntest.beforeEach(async ({ page }) => {\n  await page.goto('https://www.xxx.com/');\n  await page.setViewportSize({ width: 1920, height: 1080 });\n});\n\ntest('ai shop', async ({\n  aiAct,\n  aiInput,\n  aiAssert,\n  aiQuery,\n  aiKeyboardPress,\n  aiHover,\n  aiTap,\n  aiWaitFor,\n  agentForPage,\n  page,\n}) => {\n  // login\n  await aiAssert('The page shows the login interface');\n  await aiInput('user_name', 'in user name input');\n  await aiInput('password', 'in password input');\n  await aiKeyboardPress('Enter', 'Login Button');\n\n  // check the login success\n  await aiWaitFor('The page shows that the loading is complete');\n  await aiAssert('The current page shows the product detail page');\n\n  // check the product info\n  const dataA = await aiQuery({\n    userInfo: 'User information in the format {name: string}',\n    theFirstProductInfo: 'The first product info in the format {name: string, price: number}',\n  });\n  expect(dataA.theFirstProductInfo.name).toBe('xxx');\n  expect(dataA.theFirstProductInfo.price).toBe(100);\n\n\n  // add to cart\n  await aiTap('click add to cart button');\n  \n  await aiTap('click right top cart icon');\n  await aiAssert('The cart icon shows the number 1');\n});\n";
 export declare const YAML_EXAMPLE_CODE = "\nCRITICAL - YAML Indentation Rules:\nFor actions with additional parameters (aiScroll, aiInput, aiKeyboardPress), the parameters must be SIBLING keys at the SAME indentation level as the action key, NOT nested children indented further.\nCORRECT (parameters align with the action key):\n      - aiScroll:\n        direction: 'down'\n        scrollType: 'singleAction'\n        distance: 500\n        locate: \"main content area\"\n      - aiInput: 'text value'\n        locate: 'input field description'\nWRONG (parameters are indented further than the action key, DO NOT do this):\n      - aiScroll:\n          direction: 'down'\n          scrollType: 'singleAction'\n      - aiInput: 'text value'\n          locate: 'input field description'\n\n1. Format:\n\nweb:\n  url: \"starting_url\"\n  viewportWidth: 1280\n  viewportHeight: 960\n\ntasks:\n  - name: \"descriptive task name\"\n    flow:\n      - aiTap: \"element description\"\n        xpath: '/html/body/div[1]/button[1]'\n      - aiInput: 'text value'\n        locate: 'input field description'\n        xpath: '/html/body/div[1]/input[1]'\n      - aiScroll:\n        direction: 'down'\n        scrollType: 'singleAction'\n        distance: 500\n        locate: \"scrollable area description\"\n        xpath: '/html/body/div[1]/main[1]'\n      - aiAssert: \"expected state\"\n      - sleep: 1000\n\n2. Action Types:\n- aiTap: for clicks (natural language targeting)\n- aiInput: for text input with 'locate' field\n- aiScroll: with direction and scrollType\n- aiAssert: for validations\n- sleep: for delays (milliseconds)\n\n\n\nYAML type\ntasks:\n  - name: <name>\n    continueOnError: <boolean> # Optional, whether to continue to the next task on error, defaults to false.\n    flow:\n      # Auto Planning (.ai)\n      # ----------------\n\n      # Perform an interaction. `ai` is a shorthand for `aiAct`.\n      - ai: <prompt>\n        cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n      # This usage is the same as `ai`.\n      - aiAct: <prompt>\n        cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n      # Instant Action (.aiTap, .aiDoubleClick, .aiHover, .aiInput, .aiKeyboardPress, .aiScroll)\n      # ----------------\n\n      # Tap an element described by a prompt.\n      - aiTap: <prompt>\n        deepLocate: <boolean> # Optional, whether to use deepLocate to precisely locate the element. Defaults to False.\n        xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Midscene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n        cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n      # Double click an element described by a prompt.\n      - aiDoubleClick: <prompt>\n        deepLocate: <boolean> # Optional, whether to use deepLocate to precisely locate the element. Defaults to False.\n        xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Midscene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n        cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n      # Hover over an element described by a prompt.\n      - aiHover: <prompt>\n        deepLocate: <boolean> # Optional, whether to use deepLocate to precisely locate the element. Defaults to False.\n        xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Midscene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n        cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n      # Input text into an element described by a prompt.\n      - aiInput: <final text content of the input>\n        locate: <prompt>\n        deepLocate: <boolean> # Optional, whether to use deepLocate to precisely locate the element. Defaults to False.\n        xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Midscene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n        cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n      # Press a key (e.g., Enter, Tab, Escape) on an element described by a prompt.\n      - aiKeyboardPress: <key>\n        locate: <prompt>\n        deepLocate: <boolean> # Optional, whether to use deepLocate to precisely locate the element. Defaults to False.\n        xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Midscene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n        cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n      # Scroll globally or on an element described by a prompt.\n      - aiScroll:\n        direction: 'up' # or 'down' | 'left' | 'right'\n        scrollType: 'singleAction' # or 'scrollToTop' | 'scrollToBottom' | 'scrollToLeft' | 'scrollToRight'\n        distance: <number> # Optional, the scroll distance in pixels.\n        locate: <prompt> # Optional, the element to scroll on.\n        deepLocate: <boolean> # Optional, whether to use deepLocate to precisely locate the element. Defaults to False.\n        xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Midscene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n        cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n      # Record the current screenshot with a description in the report file.\n      - recordToReport: <title> # Optional, the title of the screenshot. If not provided, the title will be 'untitled'.\n        content: <content> # Optional, the description of the screenshot.\n\n      # Data Extraction\n      # ----------------\n\n      # Perform a query that returns a JSON object.\n      - aiQuery: <prompt> # Remember to describe the format of the result in the prompt.\n        name: <name> # The key for the query result in the JSON output.\n\n      # More APIs\n      # ----------------\n\n      # Wait for a condition to be met, with a timeout (in ms, optional, defaults to 30000).\n      - aiWaitFor: <prompt>\n        timeout: <ms>\n\n      # Perform an assertion.\n      - aiAssert: <prompt>\n        errorMessage: <error-message> # Optional, the error message to print if the assertion fails.\n\n      # Wait for a specified amount of time.\n      - sleep: <ms>\n\n      # Execute a piece of JavaScript code in the web page context.\n      - javascript: <javascript>\n        name: <name> # Optional, assign a name to the return value, which will be used as a key in the JSON output.\n\n  - name: <name>\n    flow:\n      # ...\n";

package/dist/types/constants/index.d.ts

@@ -13,6 +13,11 @@ export declare enum NodeType {
 }
 export declare const PLAYGROUND_SERVER_PORT = 5800;
 export declare const SCRCPY_SERVER_PORT = 5700;
+export declare const SCRCPY_ADB_CONNECT_TIMEOUT_MS = 10000;
+export declare const SCRCPY_PUSH_TIMEOUT_MS = 10000;
+export declare const SCRCPY_START_TIMEOUT_MS = 15000;
+export declare const SCRCPY_VIDEO_STREAM_TIMEOUT_MS = 15000;
+export declare const SCRCPY_PREVIEW_METADATA_TIMEOUT_MS: number;
 export declare const WEBDRIVER_ELEMENT_ID_KEY = "element-6066-11e4-a52e-4f735466cecf";
 export declare const DEFAULT_WDA_PORT = 8100;
 export declare const DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT = 5000;

package/dist/types/env/types.d.ts

@@ -6,6 +6,7 @@ export declare const MIDSCENE_DEBUG_MODEL_RESPONSE = "MIDSCENE_DEBUG_MODEL_RESPO
 export declare const MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG = "MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG";
 export declare const MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
 export declare const MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
+export declare const MIDSCENE_CHROME_PATH = "MIDSCENE_CHROME_PATH";
 export declare const MIDSCENE_MCP_CHROME_PATH = "MIDSCENE_MCP_CHROME_PATH";
 export declare const MIDSCENE_MCP_ANDROID_MODE = "MIDSCENE_MCP_ANDROID_MODE";
 export declare const DOCKER_CONTAINER = "DOCKER_CONTAINER";
@@ -53,6 +54,7 @@ export declare const MIDSCENE_ADB_REMOTE_PORT = "MIDSCENE_ADB_REMOTE_PORT";
 export declare const MIDSCENE_ANDROID_IME_STRATEGY = "MIDSCENE_ANDROID_IME_STRATEGY";
 export declare const MIDSCENE_IOS_DEVICE_UDID = "MIDSCENE_IOS_DEVICE_UDID";
 export declare const MIDSCENE_IOS_SIMULATOR_UDID = "MIDSCENE_IOS_SIMULATOR_UDID";
+export declare const MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE = "MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE";
 export declare const MIDSCENE_CACHE = "MIDSCENE_CACHE";
 export declare const MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
 export declare const MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -109,20 +111,20 @@ export declare const UNUSED_ENV_KEYS: string[];
 export declare const BASIC_ENV_KEYS: readonly ["MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR"];
 export declare const BOOLEAN_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET"];
 export declare const NUMBER_ENV_KEYS: readonly ["MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT"];
-export declare const STRING_ENV_KEYS: readonly ["MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
+export declare const STRING_ENV_KEYS: readonly ["MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
 /**
  * Non model related env keys, used for globally controlling the behavior of midscene
  * Can not be override by agent.modelConfig but can be override by overrideAIConfig
  * Can be access at any time
  */
-export declare const GLOBAL_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
+export declare const GLOBAL_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
 /**
  * Model related eve keys, used for declare which model to use.
  * Can be override by both agent.modelConfig and overrideAIConfig
  * Can only be access after agent.constructor
  */
 export declare const MODEL_ENV_KEYS: readonly ["MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
-export declare const ALL_ENV_KEYS: readonly [...string[], "MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER", "MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
+export declare const ALL_ENV_KEYS: readonly [...string[], "MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER", "MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
 export type TEnvKeys = (typeof ALL_ENV_KEYS)[number];
 export type TGlobalConfig = Record<TEnvKeys, string | undefined>;
 /**
@@ -203,7 +205,7 @@ export declare enum UITarsModelVersion {
 }
 /**
  * Callback to create custom OpenAI client instance
- * @param config - Resolved model configuration including apiKey, baseURL, modelName, intent, etc.
+ * @param config - Resolved model configuration including apiKey, baseURL, modelName, intent, slot, etc.
  * @returns OpenAI client instance (can be wrapped with langsmith, langfuse, etc.)
  *
  * Note: Wrapper functions like langsmith's wrapOpenAI() return the same OpenAI instance
@@ -293,9 +295,16 @@ export interface IModelConfig {
     uiTarsModelVersion?: UITarsModelVersion;
     modelDescription: string;
     /**
-     * original intent from the config
+     * The semantic intent this config is requested for.
+     * For example, getModelConfig('planning') always returns intent === 'planning'.
      */
     intent: TIntent;
+    /**
+     * The model-config slot this config was resolved from.
+     * For example, getModelConfig('planning') may resolve from slot === 'default'
+     * when MIDSCENE_PLANNING_MODEL_NAME is not configured.
+     */
+    slot: TIntent;
     /**
      * Custom OpenAI client factory function
      *

package/dist/types/env/utils.d.ts

@@ -3,36 +3,5 @@ import { ModelConfigManager } from './model-config-manager';
 import { type GLOBAL_ENV_KEYS, type MODEL_ENV_KEYS } from './types';
 export declare const globalModelConfigManager: ModelConfigManager;
 export declare const globalConfigManager: GlobalConfigManager;
-/**
- * Interface for devices that support getTimestamp method.
- * This is a minimal interface to avoid circular dependencies with @midscene/core.
- */
-export interface DeviceWithTimestamp {
-    getTimestamp?: () => Promise<number>;
-}
-/**
- * Get the current timestamp, optionally from the target device.
- *
- * When useDeviceTimestamp is enabled and a device with getTimestamp is provided,
- * this function will return the device's time. Otherwise, it returns the system time.
- *
- * This is useful when:
- * - Testing on devices with different time zones
- * - Debugging time-sensitive features
- * - The system clock and device clock are not synchronized
- *
- * @param device Optional device interface that supports getTimestamp
- * @param useDeviceTimestamp Whether to use device timestamp (from agent config)
- * @returns Timestamp in milliseconds
- *
- * @example
- * // Without device - always returns system time
- * const systemTime = await getCurrentTime();
- *
- * @example
- * // With device and config enabled - returns device time
- * const deviceTime = await getCurrentTime(androidDevice, true);
- */
-export declare function getCurrentTime(device?: DeviceWithTimestamp, useDeviceTimestamp?: boolean): Promise<number>;
 export declare const getPreferredLanguage: () => string;
 export declare const overrideAIConfig: (newConfig: Partial<Record<(typeof GLOBAL_ENV_KEYS)[number] | (typeof MODEL_ENV_KEYS)[number], string>>, extendMode?: boolean) => void;

package/dist/types/extractor/index.d.ts

@@ -28,5 +28,5 @@ export { extractTextWithPosition as webExtractTextWithPosition } from './web-ext
 export { extractTreeNode as webExtractNodeTree } from './web-extractor';
 export { extractTreeNodeAsString as webExtractNodeTreeAsString } from './web-extractor';
 export { getXpathsByPoint, getXpathsById, getNodeInfoByXpath, getElementInfoByXpath, getElementXpath, } from './locator';
-export { generateElementByRect } from './dom-util';
+export { generateElementByPoint, generateElementByRect } from './dom-util';
 export { isNotContainerElement } from './dom-util';

package/dist/types/img/index.d.ts

@@ -1,3 +1,3 @@
-export { imageInfoOfBase64, isValidPNGImageBuffer, isValidJPEGImageBuffer, isValidImageBuffer, } from './info';
+export { imageInfoOfBase64, isValidPNGImageBuffer, isValidJPEGImageBuffer, isValidImageBuffer, validateScreenshotBuffer, type ValidateScreenshotBufferOptions, } from './info';
 export { resizeAndConvertImgBuffer, resizeImgBase64, zoomForGPT4o, saveBase64Image, paddingToMatchBlockByBase64, cropByRect, scaleImage, localImg2Base64, httpImg2Base64, preProcessImageUrl, parseBase64, createImgBase64ByFormat, } from './transform';
 export { processImageElementInfo, compositeElementInfoImg, annotateRects, } from './box-select';