@aiscene/shared 8.0.3 → 8.0.5

package/dist/types/agent-tools/tool-generator.d.ts

@@ -0,0 +1,13 @@
+import type { ToolDefaults } from './tool-defaults';
+import type { ActionSpaceItem, BaseAgent, ToolCliMetadata, ToolDefinition, ToolSchema } from './types';
+import { composeUserPrompt } from './user-prompt';
+export { composeUserPrompt };
+/**
+ * Converts DeviceAction from actionSpace into ToolDefinition.
+ * This is the core logic that removes need for hardcoded tool definitions
+ */
+export declare function generateToolsFromActionSpace(actionSpace: ActionSpaceItem[], getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, sanitizeArgs?: (args: Record<string, unknown>) => Record<string, unknown>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata, toolDefaults?: ToolDefaults): ToolDefinition[];
+/**
+ * Generate common tools (screenshot, act)
+ */
+export declare function generateCommonTools(getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata, toolDefaults?: ToolDefaults): ToolDefinition[];

package/dist/types/agent-tools/types.d.ts

@@ -0,0 +1,213 @@
+import type { z } from 'zod';
+import type { ToolDefaults } from './tool-defaults';
+/**
+ * Default timeout constants for app loading verification
+ */
+export declare const defaultAppLoadingTimeoutMs = 10000;
+export declare const defaultAppLoadingCheckIntervalMs = 2000;
+/**
+ * Content item types for tool results.
+ */
+export type ToolResultContent = {
+    type: 'text';
+    text: string;
+} | {
+    type: 'image';
+    data: string;
+    mimeType: string;
+} | {
+    type: 'audio';
+    data: string;
+    mimeType: string;
+} | {
+    type: 'resource';
+    resource: {
+        text: string;
+        uri: string;
+        mimeType?: string;
+    } | {
+        uri: string;
+        blob: string;
+        mimeType?: string;
+    };
+};
+/**
+ * Result type for tool execution.
+ */
+export interface ToolResult {
+    [x: string]: unknown;
+    content: ToolResultContent[];
+    isError?: boolean;
+    _meta?: Record<string, unknown>;
+}
+/**
+ * Tool handler function type
+ * Takes parsed arguments and returns a tool result
+ */
+export type ToolHandler<T = Record<string, unknown>> = (args: T) => Promise<ToolResult>;
+/**
+ * Tool schema type using Zod
+ */
+export type ToolSchema = Record<string, z.ZodTypeAny>;
+export interface ToolCliOption {
+    preferredName?: string;
+    aliases?: string[];
+    hidden?: boolean;
+}
+export interface ToolCliMetadata {
+    options?: Record<string, ToolCliOption>;
+    /** Schema keys populated from leading positional CLI arguments. */
+    positionals?: string[];
+}
+/**
+ * Tool definition for Midscene CLI and Skill surfaces.
+ */
+export interface ToolDefinition<T = Record<string, unknown>> {
+    name: string;
+    description: string;
+    schema: ToolSchema;
+    handler: ToolHandler<T>;
+    cli?: ToolCliMetadata;
+}
+export type Tool = ToolDefinition;
+/**
+ * Action space item definition
+ * Note: Intentionally no index signature to maintain compatibility with DeviceAction
+ */
+export interface ActionSpaceItem {
+    name: string;
+    description?: string;
+    args?: Record<string, unknown>;
+    paramSchema?: z.ZodTypeAny;
+}
+/**
+ * Structural shape compatible with @aiscene/core `TUserPrompt`.
+ * Declared locally to avoid a circular dep on `@aiscene/core` from `@aiscene/shared`.
+ *
+ * Consumed by the `assert` and `act` tools in `generateCommonTools`, both of
+ * which forward reference images to core (`aiAssert` / `aiAct`). `aiWaitFor`
+ * stays string-only at the CLI surface because the tools generator does not
+ * yet expose a multimodal entry point for it.
+ */
+export type UserPromptLike = string | {
+    prompt: string;
+    images?: Array<{
+        name: string;
+        url: string;
+    }>;
+    convertHttpImage2Base64?: boolean;
+};
+export interface RecordToReportScreenshot {
+    /**
+     * PNG/JPEG data URI, or raw PNG base64 body.
+     */
+    base64: string;
+    description?: string;
+}
+export interface RecordToReportOptions {
+    content?: string;
+    /**
+     * @deprecated Use `screenshots: [{ base64 }]` instead.
+     */
+    screenshotBase64?: string;
+    screenshots?: RecordToReportScreenshot[];
+}
+/** Generic progress-bus envelope as seen by tool consumers (all untrusted). */
+export interface BaseAgentProgressEvent {
+    scope?: unknown;
+    phase?: unknown;
+    sequence?: unknown;
+    data?: unknown;
+}
+/** A single frame in a portable UI observation record. */
+export interface UIObservationFrame {
+    /** Path to the captured image. Serialized manifests use relative paths. */
+    path: string;
+    /** MIME type of the captured image file. */
+    mimeType: 'image/png' | 'image/jpeg';
+    /** Capture timestamp in milliseconds. */
+    capturedAt: number;
+}
+/**
+ * Serializable observation window that can be persisted and loaded later.
+ * Frames are ordered from earliest to latest; the final frame represents the
+ * UI state at the end of the observation window.
+ */
+export interface UIObservationRecord {
+    type: 'midscene_ui_observation';
+    version: 1;
+    /** Time when observation sampling started. */
+    startedAt: number;
+    /** Time when the final representative frame was captured. */
+    endedAt: number;
+    frames: UIObservationFrame[];
+    shotSize: {
+        width: number;
+        height: number;
+    };
+    shrunkShotToLogicalRatio: number;
+}
+/** Fixed observation window consumed by shared CLI surfaces. */
+export interface BaseUIObservation {
+    readonly frameCount: number;
+    readonly startedAt: number;
+    readonly endedAt: number;
+    aiAssert(assertion: UserPromptLike, message?: string, options?: Record<string, unknown>): Promise<unknown>;
+    /** Release any resources owned by this observation. */
+    dispose?(): Promise<void>;
+}
+/** Minimal UI observation lifecycle required by shared tool surfaces. */
+export interface BaseUIObserver {
+    readonly bufferedFrameCount: number;
+    /** Stop sampling and return the fixed observed window. */
+    stop(): Promise<BaseUIObservation>;
+    /** Release temporary backing files if observation does not complete. */
+    dispose?(): Promise<void>;
+}
+/** Options for {@link BaseAgent.startObserving}. */
+export interface BaseUIObserverOptions {
+    /** Sampling interval in milliseconds. Defaults to 1000; minimum 200. */
+    intervalMs?: number;
+    /** Maximum number of buffered frames. Defaults to 30; minimum 2. */
+    maxFrames?: number;
+    /** Auto-stop timeout in milliseconds. Defaults to 300000; 0 disables it. */
+    watchdogMs?: number;
+}
+/**
+ * Base agent interface
+ * Represents a platform-specific agent (Android, iOS, Web)
+ * Note: Return types use `unknown` for compatibility with platform-specific implementations
+ */
+export interface BaseAgent {
+    getActionSpace(): Promise<ActionSpaceItem[]>;
+    destroy?(): Promise<void>;
+    reportFile?: string | null;
+    page?: {
+        screenshotBase64(): Promise<string>;
+    };
+    addDumpUpdateListener?(listener: (dump: string, executionDump?: unknown) => void): () => void;
+    addProgressListener?(listener: (event: BaseAgentProgressEvent) => void): () => void;
+    recordToReport?(title?: string, opt?: RecordToReportOptions): Promise<void>;
+    callActionInActionSpace?(actionName: string, params?: unknown): Promise<unknown>;
+    aiAction?(description: UserPromptLike, params?: Record<string, unknown>): Promise<unknown>;
+    aiWaitFor?(assertion: string, options: Record<string, unknown>): Promise<unknown>;
+    aiAssert?(assertion: UserPromptLike, msg?: string, options?: Record<string, unknown>): Promise<unknown>;
+    /** Start a UI observation window and capture its baseline frame. */
+    startObserving?(options?: BaseUIObserverOptions): Promise<BaseUIObserver>;
+}
+/**
+ * Base device interface for temporary device instances
+ */
+export interface BaseDevice {
+    actionSpace(): ActionSpaceItem[];
+    destroy?(): Promise<void>;
+}
+/**
+ * Interface for platform-specific tools manager.
+ */
+export interface IMidsceneTools {
+    initTools(): Promise<void>;
+    destroy?(): Promise<void>;
+    getCliToolDefinitions?(): ToolDefinition[];
+    setToolDefaults?(toolDefaults: ToolDefaults): void;
+}

package/dist/types/agent-tools/user-prompt.d.ts

@@ -0,0 +1,13 @@
+import { z } from 'zod';
+import type { UserPromptLike } from './types';
+export declare function composeUserPrompt(input: {
+    prompt: string;
+    image?: unknown;
+    imageName?: unknown;
+    convertHttpImage2Base64?: unknown;
+}): UserPromptLike;
+export declare const promptInputExtraSchema: {
+    image: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
+    imageName: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
+    convertHttpImage2Base64: z.ZodOptional<z.ZodUnion<[z.ZodBoolean, z.ZodString]>>;
+};

package/dist/types/cli/interrupt.d.ts

@@ -0,0 +1,49 @@
+export type CliInterruptReason = 'sigint' | 'sigterm' | 'sighup' | 'watchdog';
+type CliInterruptSignal = 'SIGINT' | 'SIGTERM' | 'SIGHUP';
+export interface CliInterruptSource {
+    on(event: CliInterruptSignal, listener: () => void): unknown;
+    removeListener(event: CliInterruptSignal, listener: () => void): unknown;
+}
+export interface CliInterruptInputSource {
+    readonly isTTY?: boolean;
+    readonly isRaw?: boolean;
+    readonly readableFlowing?: boolean | null;
+    on(event: 'data', listener: (chunk: unknown) => void): unknown;
+    removeListener(event: 'data', listener: (chunk: unknown) => void): unknown;
+    setRawMode?(enabled: boolean): unknown;
+    pause?(): unknown;
+}
+export interface CliInterruptWaiter {
+    /** Resolves on the first stop signal or watchdog timeout. */
+    readonly result: Promise<CliInterruptReason>;
+    /** Release signal handlers after asynchronous finalization has completed. */
+    dispose(): void;
+}
+export interface CliInterruptWaiterOptions {
+    source?: CliInterruptSource;
+    input?: CliInterruptInputSource;
+    /** Called after restoring the terminal when Ctrl+C is pressed again. */
+    forceExit?: (exitCode: number) => void;
+}
+/** Whether a foreground CLI command is currently waiting for this source. */
+export declare function hasActiveCliInterruptWaiter(source?: CliInterruptSource): boolean;
+/**
+ * Keep graceful-stop handlers installed until the caller has finished saving.
+ *
+ * Package runners such as pnpm can deliver SIGINT to the foreground child,
+ * immediately follow it with SIGTERM, then cause SIGHUP when the runner exits
+ * and its pseudo-terminal closes. Resolving on the first signal is not enough:
+ * removing any handler at that point lets a subsequent signal kill the child
+ * during asynchronous artifact finalization.
+ *
+ * On a TTY, Ctrl+C is captured as raw input so the package runner itself stays
+ * alive until the child has saved and restored the terminal. Signal handlers
+ * remain as the graceful-stop path for externally delivered termination.
+ */
+export declare function createCliInterruptWaiter(watchdogMs: number, options?: CliInterruptWaiterOptions): CliInterruptWaiter;
+/**
+ * Wait for one stop request and release the handlers immediately afterwards.
+ * Long-running finalizers should use {@link createCliInterruptWaiter} instead.
+ */
+export declare function waitForCliInterrupt(watchdogMs: number, source?: CliInterruptSource, input?: CliInterruptInputSource | undefined): Promise<CliInterruptReason>;
+export {};

package/dist/types/cli/record-command.d.ts

@@ -0,0 +1,3 @@
+import type { BaseAgent, ToolCliMetadata, ToolDefinition, ToolSchema } from '../agent-tools/types';
+/** Create the foreground recording command exposed only by Midscene CLIs. */
+export declare function createRecordCliCommand(getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>, initArgSchema?: ToolSchema, initArgCliMetadata?: ToolCliMetadata): ToolDefinition;

package/dist/types/cli/screenshot-file.d.ts

@@ -0,0 +1,10 @@
+export interface WriteCliScreenshotFileOptions {
+    id?: unknown;
+    mimeType?: unknown;
+    extension?: unknown;
+    directoryPath?: string;
+    directoryName?: string;
+    filenamePrefix?: string;
+    overwrite?: boolean;
+}
+export declare function writeCliScreenshotFile(rawBase64: string, options?: WriteCliScreenshotFileOptions): string;

package/dist/types/cli/verbose-ai-act.d.ts

@@ -0,0 +1,44 @@
+import { type CliVerboseScreenshotCollectOptions } from './verbose-screenshot';
+export interface CliVerboseLine {
+    key: string;
+    text: string;
+}
+/** Generic progress-bus envelope as seen by the CLI (all fields untrusted). */
+export interface CliAgentProgressEvent {
+    scope?: unknown;
+    phase?: unknown;
+    sequence?: unknown;
+    data?: unknown;
+}
+/** Flattened, render-ready aiAct payload derived from the envelope. */
+export interface CliAiActProgressPayload {
+    phase?: unknown;
+    sequence?: unknown;
+    prompt?: unknown;
+    planIndex?: unknown;
+    planLimit?: unknown;
+    action?: unknown;
+    thought?: unknown;
+    log?: unknown;
+    output?: unknown;
+    durationMs?: unknown;
+    error?: unknown;
+    screenshots?: Array<Record<string, unknown>>;
+    screenshotPath?: string;
+    [key: string]: unknown;
+}
+/** aiAct scope tag on the progress bus (mirrors core's `aiActProgressScope`). */
+export declare const cliAiActProgressScope = "aiAct";
+export declare function normalizeAiActProgressEventForCli(event: unknown, screenshotOptions?: CliVerboseScreenshotCollectOptions): CliAiActProgressPayload | undefined;
+export declare function buildAiActProgressEventLines(event: CliAiActProgressPayload): CliVerboseLine[];
+/**
+ * A producer-specific renderer for the generic CLI progress dispatcher: turn a
+ * raw progress envelope into a normalized payload, then that payload into
+ * verbose lines. New producers register their own renderer; the dispatcher core
+ * stays untouched.
+ */
+export interface CliProgressRenderer {
+    normalize(event: unknown, screenshotOptions: CliVerboseScreenshotCollectOptions): Record<string, unknown> | undefined;
+    buildLines(payload: Record<string, unknown>): CliVerboseLine[];
+}
+export declare const aiActCliProgressRenderer: CliProgressRenderer;

package/dist/types/cli/verbose-screenshot.d.ts

@@ -0,0 +1,10 @@
+export type CliVerboseScreenshotExportMode = 'none' | 'tmp' | 'report';
+export interface CliVerboseScreenshotCollectOptions {
+    reportFile?: unknown;
+    exportMode?: CliVerboseScreenshotExportMode;
+    cache?: Map<string, string>;
+}
+export declare function collectScreenshotRefs(value: unknown, options?: CliVerboseScreenshotCollectOptions): Array<Record<string, unknown>>;
+export declare function pathForReportScreenshot(path: string, reportFile?: unknown): string;
+export declare function latestScreenshotPathForAiAct(value: unknown, options?: CliVerboseScreenshotCollectOptions): string;
+export declare function renderScreenshotList(screenshots: unknown): string;

package/dist/types/cli/verbose.d.ts

@@ -0,0 +1,40 @@
+export declare const cliVerboseFlag = "verbose";
+export type CliVerboseFormat = 'text' | 'jsonl';
+export interface CliVerboseContext {
+    enabled: boolean;
+    format?: CliVerboseFormat;
+    scriptName?: string;
+    commandName?: string;
+    startedAt?: number;
+    renderedLineKeys?: Set<string>;
+}
+export interface CliVerboseEvent {
+    event: string;
+    scriptName?: string;
+    command?: string;
+    status?: 'ok' | 'error';
+    durationMs?: number;
+    error?: string;
+    [key: string]: unknown;
+}
+type DumpUpdateAgent = {
+    addDumpUpdateListener?: (listener: (dump: string, executionDump?: unknown) => void) => () => void;
+    addProgressListener?: (listener: (event: unknown) => void) => () => void;
+    reportFile?: string | null;
+};
+export declare function stripVerboseFlag(argv: readonly string[]): {
+    rawArgs: string[];
+    verbose: boolean;
+    format: CliVerboseFormat;
+};
+export declare function withCliVerboseContext<T>(context: CliVerboseContext, fn: () => Promise<T>): Promise<T>;
+export declare function getCliVerboseContext(): CliVerboseContext;
+export declare function isCliVerboseEnabled(): boolean;
+export declare function emitCliVerboseEvent(event: CliVerboseEvent): void;
+declare function errorMessageOf(error: unknown): string;
+export declare function compactCliVerboseValue(value: unknown): unknown;
+export declare function compactCliVerboseArgs(args: Record<string, unknown>): Record<string, unknown>;
+export declare function attachCliVerboseDumpListener(agent: DumpUpdateAgent, options?: {
+    toolName?: string;
+}): () => void;
+export { errorMessageOf as cliVerboseErrorMessage };

package/dist/types/env/types.d.ts

@@ -6,6 +6,7 @@ export declare const MIDSCENE_DEBUG_MODEL_RESPONSE = "MIDSCENE_DEBUG_MODEL_RESPO
 export declare const MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG = "MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG";
 export declare const MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
 export declare const MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
+export declare const MIDSCENE_CHROME_PATH = "MIDSCENE_CHROME_PATH";
 export declare const MIDSCENE_MCP_CHROME_PATH = "MIDSCENE_MCP_CHROME_PATH";
 export declare const MIDSCENE_MCP_ANDROID_MODE = "MIDSCENE_MCP_ANDROID_MODE";
 export declare const DOCKER_CONTAINER = "DOCKER_CONTAINER";
@@ -23,6 +24,9 @@ export declare const MIDSCENE_MODEL_RETRY_INTERVAL = "MIDSCENE_MODEL_RETRY_INTER
 export declare const MIDSCENE_MODEL_REASONING_EFFORT = "MIDSCENE_MODEL_REASONING_EFFORT";
 export declare const MIDSCENE_MODEL_REASONING_ENABLED = "MIDSCENE_MODEL_REASONING_ENABLED";
 export declare const MIDSCENE_MODEL_REASONING_BUDGET = "MIDSCENE_MODEL_REASONING_BUDGET";
+export declare const MIDSCENE_MODEL_RESPONSE_FORMAT = "MIDSCENE_MODEL_RESPONSE_FORMAT";
+export type TModelReasoningEnabled = boolean | 'default';
+export type TModelResponseFormat = 'none' | 'auto';
 /**
  * @deprecated Use MIDSCENE_MODEL_API_KEY instead. This is kept for backward compatibility.
  */
@@ -68,6 +72,7 @@ export declare const MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE"
 export declare const MIDSCENE_CACHE_MAX_FILENAME_LENGTH = "MIDSCENE_CACHE_MAX_FILENAME_LENGTH";
 export declare const MIDSCENE_REPLANNING_CYCLE_LIMIT = "MIDSCENE_REPLANNING_CYCLE_LIMIT";
 export declare const MIDSCENE_RUN_DIR = "MIDSCENE_RUN_DIR";
+export declare const MIDSCENE_RECORD_MODEL_CALL = "MIDSCENE_RECORD_MODEL_CALL";
 export declare const MIDSCENE_INSIGHT_MODEL_NAME = "MIDSCENE_INSIGHT_MODEL_NAME";
 export declare const MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY = "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY";
 export declare const MIDSCENE_INSIGHT_MODEL_HTTP_PROXY = "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY";
@@ -83,6 +88,7 @@ export declare const MIDSCENE_INSIGHT_MODEL_FAMILY = "MIDSCENE_INSIGHT_MODEL_FAM
 export declare const MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT = "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT";
 export declare const MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED = "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED";
 export declare const MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET = "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET";
+export declare const MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT = "MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT";
 export declare const MIDSCENE_PLANNING_MODEL_NAME = "MIDSCENE_PLANNING_MODEL_NAME";
 export declare const MIDSCENE_PLANNING_MODEL_SOCKS_PROXY = "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY";
 export declare const MIDSCENE_PLANNING_MODEL_HTTP_PROXY = "MIDSCENE_PLANNING_MODEL_HTTP_PROXY";
@@ -98,6 +104,7 @@ export declare const MIDSCENE_PLANNING_MODEL_FAMILY = "MIDSCENE_PLANNING_MODEL_F
 export declare const MIDSCENE_PLANNING_MODEL_REASONING_EFFORT = "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT";
 export declare const MIDSCENE_PLANNING_MODEL_REASONING_ENABLED = "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED";
 export declare const MIDSCENE_PLANNING_MODEL_REASONING_BUDGET = "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET";
+export declare const MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT = "MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT";
 export declare const MIDSCENE_MODEL_FAMILY = "MIDSCENE_MODEL_FAMILY";
 /**
  * env keys declared but unused
@@ -107,29 +114,29 @@ export declare const UNUSED_ENV_KEYS: string[];
  * env keys for debug or basic run
  * can not be override by overrideAIConfig
  */
-export declare const BASIC_ENV_KEYS: readonly ["MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR"];
+export declare const BASIC_ENV_KEYS: readonly ["MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_RECORD_MODEL_CALL"];
 export declare const BOOLEAN_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET"];
 export declare const NUMBER_ENV_KEYS: readonly ["MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT"];
-export declare const STRING_ENV_KEYS: readonly ["MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
+export declare const STRING_ENV_KEYS: readonly ["MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
 /**
  * Non model related env keys, used for globally controlling the behavior of midscene
  * Can not be override by agent.modelConfig but can be override by overrideAIConfig
  * Can be access at any time
  */
-export declare const GLOBAL_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
+export declare const GLOBAL_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
 /**
  * Model related eve keys, used for declare which model to use.
  * Can be override by both agent.modelConfig and overrideAIConfig
  * Can only be access after agent.constructor
  */
-export declare const MODEL_ENV_KEYS: readonly ["MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
-export declare const ALL_ENV_KEYS: readonly [...string[], "MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER", "MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
+export declare const MODEL_ENV_KEYS: readonly ["MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_RESPONSE_FORMAT", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT", "MIDSCENE_MODEL_FAMILY"];
+export declare const ALL_ENV_KEYS: readonly [...string[], "MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_RECORD_MODEL_CALL", "MIDSCENE_CACHE", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_MODEL_MAX_TOKENS", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_MODEL_MAX_TOKENS", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER", "MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_RESPONSE_FORMAT", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT", "MIDSCENE_MODEL_FAMILY"];
 export type TEnvKeys = (typeof ALL_ENV_KEYS)[number];
 export type TGlobalConfig = Record<TEnvKeys, string | undefined>;
 /**
  * valid Model family types
  */
-export type TModelFamily = 'qwen2.5-vl' | 'qwen3-vl' | 'qwen3.5' | 'qwen3.6' | 'doubao-vision' | 'doubao-seed' | 'gemini' | 'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5' | 'glm-v' | 'auto-glm' | 'auto-glm-multilingual' | 'gpt-5';
+export type TModelFamily = 'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6' | 'doubao-vision' | 'doubao-seed' | 'gemini' | 'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5' | 'glm-v' | 'auto-glm' | 'auto-glm-multilingual' | 'gpt-5' | 'deepseek' | 'kimi' | 'kimi3' | 'xiaomi-mimo';
 export declare const MODEL_FAMILY_VALUES: TModelFamily[];
 export interface IModelConfigForInsight {
     [MIDSCENE_INSIGHT_MODEL_NAME]: string;
@@ -176,6 +183,7 @@ export interface IModelConfigForDefault {
     [MIDSCENE_MODEL_REASONING_EFFORT]?: string;
     [MIDSCENE_MODEL_REASONING_ENABLED]?: string;
     [MIDSCENE_MODEL_REASONING_BUDGET]?: string;
+    [MIDSCENE_MODEL_RESPONSE_FORMAT]?: TModelResponseFormat;
 }
 export interface IModelConfigForDefaultLegacy {
     [MIDSCENE_MODEL_NAME]: string;
@@ -280,12 +288,13 @@ export interface IModelConfig {
      * Enable/disable reasoning for the model.
      * Passed through to model-family-specific parameters (e.g., enable_thinking for qwen, thinking.type for doubao/glm-v).
      */
-    reasoningEnabled?: boolean;
+    reasoningEnabled?: TModelReasoningEnabled;
     /**
      * Reasoning token budget for the model.
      * Passed through to model-family-specific parameters (e.g., thinking_budget for qwen).
      */
     reasoningBudget?: number;
+    responseFormat?: TModelResponseFormat;
     /**
      * Model family - unified model configuration
      * Maps directly to model families like 'qwen2.5-vl', 'qwen3-vl', 'doubao-vision', 'doubao-seed', etc.

package/dist/types/img/index.d.ts

@@ -1,3 +1,3 @@
-export { imageInfoOfBase64, isValidPNGImageBuffer, isValidJPEGImageBuffer, isValidImageBuffer, validateScreenshotBuffer, type ValidateScreenshotBufferOptions, } from './info';
-export { resizeAndConvertImgBuffer, resizeImgBase64, zoomForGPT4o, saveBase64Image, paddingToMatchBlockByBase64, cropByRect, scaleImage, localImg2Base64, httpImg2Base64, preProcessImageUrl, parseBase64, createImgBase64ByFormat, } from './transform';
+export { encodedImageInfoOfBuffer, imageInfoOfBase64, isValidPNGImageBuffer, isValidJPEGImageBuffer, isValidImageBuffer, validateScreenshotBuffer, type ValidateScreenshotBufferOptions, } from './info';
+export { resizeAndConvertImgBuffer, convertImgBufferToJpeg, convertBase64ImageToJpeg, constrainBase64ImageToMaxSize, resizeBase64ImageToJpeg, resizeImgBase64, zoomForGPT4o, saveBase64Image, paddingToMatchBlockByBase64, cropByRect, scaleImage, localImg2Base64, httpImg2Base64, preProcessImageUrl, parseBase64, createImgBase64ByFormat, inferBase64ImageFormat, normalizeBase64Image, normalizeScreenshotBase64, type NormalizeScreenshotBase64Options, type ConstrainBase64ImageToMaxSizeOptions, type JpegBase64DataUrl, type ResizeBase64ImageToJpegOptions, } from './transform';
 export { processImageElementInfo, compositeElementInfoImg, annotateRects, } from './box-select';

package/dist/types/img/info.d.ts

@@ -1,6 +1,8 @@
+import { Buffer } from 'node:buffer';
 import type { Size } from '../types';
 export interface ImageInfo extends Size {
 }
+export declare function encodedImageInfoOfBuffer(imageBuffer: Buffer): ImageInfo;
 /**
  * Retrieves the dimensions of an image from a base64-encoded string
  *

package/dist/types/img/photon-loader.d.ts

@@ -0,0 +1,2 @@
+export type PhotonModule = typeof import('@silvia-odwyer/photon');
+export declare function loadPhotonModule(): Promise<PhotonModule>;

package/dist/types/img/transform.d.ts

@@ -1,6 +1,6 @@
 import { Buffer } from 'node:buffer';
 import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon-node';
-import type { Rect } from '../types';
+import type { Rect, Size } from '../types';
 /**
  * Saves a Base64-encoded image to a file
  *
@@ -26,7 +26,27 @@ export declare function resizeAndConvertImgBuffer(inputFormat: string, inputData
     buffer: Buffer;
     format: string;
 }>;
+export declare function inferBase64ImageFormat(base64Body: string): string;
 export declare const createImgBase64ByFormat: (format: string, body: string) => string;
+export interface NormalizeScreenshotBase64Options {
+    label?: string;
+}
+export declare const normalizeScreenshotBase64: (base64: string, options?: NormalizeScreenshotBase64Options) => string;
+export declare const normalizeBase64Image: (base64: string) => string;
+export type JpegBase64DataUrl = `data:image/jpeg;base64,${string}`;
+export interface ResizeBase64ImageToJpegOptions {
+    sourceSize?: Size;
+    targetSize: Size;
+    jpegQuality?: number;
+}
+export interface ConstrainBase64ImageToMaxSizeOptions {
+    maxSize: number;
+    jpegQuality?: number;
+}
+export declare function convertImgBufferToJpeg(inputData: Buffer, jpegQuality?: number): Promise<Buffer>;
+export declare function convertBase64ImageToJpeg(inputBase64: string, jpegQuality?: number): Promise<JpegBase64DataUrl>;
+export declare function resizeBase64ImageToJpeg(inputBase64: string, options: ResizeBase64ImageToJpegOptions): Promise<JpegBase64DataUrl>;
+export declare function constrainBase64ImageToMaxSize(inputBase64: string, options: ConstrainBase64ImageToMaxSizeOptions): Promise<string>;
 export declare function resizeImgBase64(inputBase64: string, newSize: {
     width: number;
     height: number;
@@ -58,7 +78,7 @@ export declare function paddingToMatchBlockByBase64(imageBase64: string, blockSi
     height: number;
     imageBase64: string;
 }>;
-export declare function cropByRect(imageBase64: string, rect: Rect, paddingImage: boolean): Promise<{
+export declare function cropByRect(imageBase64: string, rect: Rect, paddingImage?: boolean): Promise<{
     width: number;
     height: number;
     imageBase64: string;

package/dist/types/mcp/types.d.ts

@@ -95,6 +95,7 @@ export interface BaseAgent {
     aiAction?: (description: string, params?: Record<string, unknown>) => Promise<unknown>;
     aiWaitFor?: (assertion: string, options: Record<string, unknown>) => Promise<unknown>;
     aiAssert?: (assertion: string, msg?: string, options?: Record<string, unknown>) => Promise<unknown>;
+    aiQuery?: (demand: any, opt?: any) => Promise<any>;
 }
 /**
  * Base device interface for temporary device instances