@aiscene/core 8.1.1 → 9.0.0

package/dist/types/agent/tasks.d.ts

@@ -3,8 +3,8 @@ import type { AbstractInterface } from '../device';
 import type Service from '../service';
 import type { TaskRunner } from '../task-runner';
 import { TaskExecutionError } from '../task-runner';
-import type { DeepThinkOption, DeviceAction, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam } from '../types';
-import { type IModelConfig } from '@aiscene/shared/env';
+import type { DeviceAction, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam } from '../types';
+import type { IModelConfig } from '@aiscene/shared/env';
 import type { TaskCache } from './task-cache';
 export { locatePlanForLocate } from './task-builder';
 interface ExecutionResult<OutputType = any> {
@@ -26,22 +26,23 @@ export declare class TaskExecutor {
     private readonly hooks?;
     replanningCycleLimit?: number;
     waitAfterAction?: number;
-    useDeviceTimestamp?: boolean;
+    useDeviceTime?: boolean;
     get page(): AbstractInterface;
     constructor(interfaceInstance: AbstractInterface, service: Service, opts: {
         taskCache?: TaskCache;
         onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
         replanningCycleLimit?: number;
         waitAfterAction?: number;
-        useDeviceTimestamp?: boolean;
+        useDeviceTime?: boolean;
         hooks?: TaskExecutorHooks;
         actionSpace: DeviceAction[];
     });
     private createExecutionSession;
     private getActionSpace;
     /**
-     * Get a readable time string using device time when configured.
-     * This method respects the useDeviceTimestamp configuration.
+     * Get a readable time string. When device time is enabled, use the
+     * device-formatted wall-clock time directly so host timezone formatting does
+     * not reinterpret a device timestamp.
      * @param format - Optional format string
      * @returns A formatted time string
      */
@@ -57,7 +58,7 @@ export declare class TaskExecutor {
         runner: TaskRunner;
     }>;
     runPlans(title: string, plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig): Promise<ExecutionResult>;
-    action(userPrompt: string, modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, includeBboxInPlanning: boolean, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number, deepThink?: DeepThinkOption, fileChooserAccept?: string[], deepLocate?: boolean, abortSignal?: AbortSignal): Promise<ExecutionResult<{
+    action(userPrompt: string, modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, includeBboxInPlanning: boolean, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number, planningModeDeepThink?: boolean, fileChooserAccept?: string[], deepLocate?: boolean, abortSignal?: AbortSignal): Promise<ExecutionResult<{
         yamlFlow?: MidsceneYamlFlowItem[];
         output?: string;
     } | undefined>>;

package/dist/types/agent/usage-intent.d.ts

@@ -0,0 +1,3 @@
+import type { AIUsageInfo } from '../types';
+import type { TIntent } from '@aiscene/shared/env';
+export declare function withUsageIntent(usage: AIUsageInfo | undefined, intent: TIntent): AIUsageInfo | undefined;

package/dist/types/ai-model/llm-planning.d.ts

@@ -1,4 +1,4 @@
-import type { DeepThinkOption, DeviceAction, InterfaceType, PlanningAIResponse, RawResponsePlanningAIResponse, UIContext } from '../types';
+import type { DeviceAction, InterfaceType, PlanningAIResponse, RawResponsePlanningAIResponse, UIContext } from '../types';
 import type { IModelConfig, TModelFamily } from '@aiscene/shared/env';
 import type { ConversationHistory } from './conversation-history';
 /**
@@ -14,6 +14,6 @@ export declare function plan(userInstruction: string, opts: {
     conversationHistory: ConversationHistory;
     includeBbox: boolean;
     imagesIncludeCount?: number;
-    deepThink?: DeepThinkOption;
+    planningModeDeepThink?: boolean;
     abortSignal?: AbortSignal;
 }): Promise<PlanningAIResponse>;

package/dist/types/ai-model/service-caller/codex-app-server.d.ts

@@ -1,7 +1,7 @@
-import type { AIUsageInfo, DeepThinkOption, StreamingCallback } from '../../types';
+import type { AIUsageInfo, StreamingCallback } from '../../types';
 import type { IModelConfig } from '@aiscene/shared/env';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
-type CodexReasoningEffort = 'low' | 'medium' | 'high' | 'xhigh';
+type CodexReasoningEffort = 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
 type CodexTextInput = {
     type: 'text';
     text: string;
@@ -24,8 +24,8 @@ type CodexTurnResult = {
 };
 export declare const isCodexAppServerProvider: (baseURL?: string) => boolean;
 export declare const normalizeCodexLocalImagePath: (imageUrl: string, platform?: NodeJS.Platform) => string;
-export declare const resolveCodexReasoningEffort: ({ deepThink, modelConfig, }: {
-    deepThink?: DeepThinkOption;
+export declare const resolveCodexReasoningEffort: ({ reasoningEnabled, modelConfig, }: {
+    reasoningEnabled?: boolean;
     modelConfig: IModelConfig;
 }) => CodexReasoningEffort | undefined;
 export declare const buildCodexTurnPayloadFromMessages: (messages: ChatCompletionMessageParam[]) => {
@@ -35,7 +35,7 @@ export declare const buildCodexTurnPayloadFromMessages: (messages: ChatCompletio
 export declare function callAIWithCodexAppServer(messages: ChatCompletionMessageParam[], modelConfig: IModelConfig, options?: {
     stream?: boolean;
     onChunk?: StreamingCallback;
-    deepThink?: DeepThinkOption;
+    reasoningEnabled?: boolean;
     abortSignal?: AbortSignal;
 }): Promise<CodexTurnResult>;
 export declare function __shutdownCodexAppServerForTests(): Promise<void>;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -1,4 +1,4 @@
-import type { AIUsageInfo, DeepThinkOption } from '../../types';
+import type { AIUsageInfo } from '../../types';
 import type { StreamingCallback } from '../../types';
 export declare class AIResponseParseError extends Error {
     usage?: AIUsageInfo;
@@ -11,7 +11,6 @@ import type { AIArgs } from '../../common';
 export declare function callAI(messages: ChatCompletionMessageParam[], modelConfig: IModelConfig, options?: {
     stream?: boolean;
     onChunk?: StreamingCallback;
-    deepThink?: DeepThinkOption;
     abortSignal?: AbortSignal;
 }): Promise<{
     content: string;
@@ -20,7 +19,6 @@ export declare function callAI(messages: ChatCompletionMessageParam[], modelConf
     isStreamed: boolean;
 }>;
 export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], modelConfig: IModelConfig, options?: {
-    deepThink?: DeepThinkOption;
     abortSignal?: AbortSignal;
 }): Promise<{
     content: T;
@@ -44,6 +42,5 @@ export declare function resolveReasoningConfig({ reasoningEnabled, reasoningEffo
 }): {
     config: Record<string, unknown>;
     debugMessage?: string;
-    warningMessage?: string;
 };
 export declare function safeParseJson(input: string, modelFamily: TModelFamily | undefined): any;

package/dist/types/ai-model/service-caller/request-timeout.d.ts

@@ -0,0 +1,32 @@
+import type { IModelConfig } from '@aiscene/shared/env';
+/**
+ * Default hard timeout (ms) applied to every AI HTTP call.
+ *
+ * We need an end-to-end timeout for the whole request lifecycle, not just the
+ * time until response headers arrive. Some providers can return headers
+ * quickly and then stall while the body is still being read.
+ *
+ * Override per intent via `MIDSCENE_MODEL_TIMEOUT`,
+ * `MIDSCENE_INSIGHT_MODEL_TIMEOUT`, or `MIDSCENE_PLANNING_MODEL_TIMEOUT`.
+ * Set the env var (or `modelConfig.timeout`) to `0` to disable the hard
+ * timeout entirely; only a caller-provided `abortSignal` will cancel the
+ * request in that case.
+ */
+export declare const DEFAULT_AI_CALL_TIMEOUT_MS = 180000;
+/** Identifying code set on the AbortError raised by our hard timeout. */
+export declare const AI_CALL_HARD_TIMEOUT_CODE = "AI_CALL_HARD_TIMEOUT";
+/**
+ * Resolve the hard request timeout for an AI call.
+ * Returns `null` when the user explicitly opted out (`timeout === 0`).
+ */
+export declare function resolveEffectiveTimeoutMs(modelConfig: Pick<IModelConfig, 'timeout'>): number | null;
+/**
+ * True if the error was raised by our hard-timeout AbortSignal (vs any other
+ * abort/network/HTTP error). Used to drive observability without having to
+ * string-match the message.
+ */
+export declare function isHardTimeoutError(err: unknown): boolean;
+export declare function buildRequestAbortSignal(timeoutMs: number | null, userSignal?: AbortSignal): {
+    signal: AbortSignal;
+    cleanup: () => void;
+};

package/dist/types/device/device-options.d.ts

@@ -35,7 +35,12 @@ export type AndroidDeviceOpt = {
     screenshotResizeScale?: number;
     /** Always fetch screen info on each call; if false, cache the first result */
     alwaysRefreshScreenInfo?: boolean;
-    /** Minimum screenshot buffer size in bytes (default: 10240 = 10KB). Set to 0 to disable validation. */
+    /**
+     * Screenshot buffer size validation threshold in bytes. Buffers below this
+     * value are treated as failed or corrupted captures. Defaults to 1024 (1KB).
+     * Set to 0 to skip only this size check; empty-buffer and image-format
+     * validation still run.
+     */
     minScreenshotBufferSize?: number;
     /**
      * Scrcpy screenshot configuration for high-performance screen capture.
@@ -111,6 +116,11 @@ export type IOSDeviceInputOpt = {
 export type IOSDeviceOpt = {
     /** Device ID (UDID) to connect to */
     deviceId?: string;
+    /**
+     * Optional npm module path used to override the default iOS device implementation.
+     * The target module must export an `IOSDevice` class (or default export) compatible with Midscene's iOS device interface.
+     */
+    iOSDeviceClassOverride?: string;
     /** Custom device actions to register */
     customActions?: DeviceAction<any>[];
     /** WebDriverAgent port (default: 8100) */

package/dist/types/device/index.d.ts

@@ -6,6 +6,110 @@ import type { ElementCacheFeature, Rect, Size } from '../types';
 export interface FileChooserHandler {
     accept(files: string[]): Promise<void>;
 }
+export interface MjpegStreamFrame {
+    /** Raw base64-encoded image bytes WITHOUT a `data:image/...;base64,` prefix. */
+    data: string;
+    contentType?: string;
+}
+export interface MjpegStreamHandle {
+    stop(): void | Promise<void>;
+}
+export interface MjpegStreamOptions {
+    signal?: AbortSignal;
+    onFrame(frame: MjpegStreamFrame): void;
+    onError?(error: unknown): void;
+}
+/** A point in device-pixel coordinates on the screen. */
+export interface PointerPoint {
+    x: number;
+    y: number;
+}
+export interface PointerInputPrimitives {
+    tap(p: PointerPoint, opts?: {
+        duration?: number;
+    }): Promise<void>;
+    doubleClick?(p: PointerPoint): Promise<void>;
+    rightClick?(p: PointerPoint): Promise<void>;
+    hover?(p: PointerPoint): Promise<void>;
+    longPress?(p: PointerPoint, opts?: {
+        duration?: number;
+    }): Promise<void>;
+    dragAndDrop?(from: PointerPoint, to: PointerPoint): Promise<void>;
+}
+export interface TouchInputPrimitives {
+    swipe(start: PointerPoint, end: PointerPoint, opts?: {
+        duration?: number;
+        repeat?: number;
+    }): Promise<void>;
+    pinch?(center: PointerPoint, opts: {
+        startDistance: number;
+        endDistance: number;
+        duration: number;
+    }): Promise<void>;
+}
+export interface KeyboardInputPrimitives {
+    keyboardPress(keyName: string, opts?: {
+        target?: unknown;
+    }): Promise<void>;
+    cursorMove?(direction: 'left' | 'right', times?: number): Promise<void>;
+    typeText(value: string, opts?: {
+        autoDismissKeyboard?: boolean;
+        target?: unknown;
+        replace?: boolean;
+        focusOnly?: boolean;
+    }): Promise<void>;
+    clearInput(target?: unknown): Promise<void>;
+}
+export interface ScrollInputPrimitives {
+    scroll(param: ActionScrollParam): Promise<void>;
+}
+export interface SystemInputPrimitives {
+    backButton?(): Promise<void>;
+    homeButton?(): Promise<void>;
+    recentAppsButton?(): Promise<void>;
+}
+export interface InputPrimitives {
+    pointer?: PointerInputPrimitives;
+    keyboard?: KeyboardInputPrimitives;
+    touch?: TouchInputPrimitives;
+    scroll?: ScrollInputPrimitives;
+    system?: SystemInputPrimitives;
+}
+export interface MobileInputPrimitives extends InputPrimitives {
+    pointer: PointerInputPrimitives & {
+        doubleClick(p: PointerPoint): Promise<void>;
+        longPress(p: PointerPoint, opts?: {
+            duration?: number;
+        }): Promise<void>;
+        dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;
+    };
+    keyboard: KeyboardInputPrimitives;
+    touch: TouchInputPrimitives;
+}
+export interface BrowserInputPrimitives extends InputPrimitives {
+    pointer: PointerInputPrimitives & {
+        doubleClick(p: PointerPoint): Promise<void>;
+        rightClick(p: PointerPoint): Promise<void>;
+        hover(p: PointerPoint): Promise<void>;
+        dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;
+        longPress(p: PointerPoint, opts?: {
+            duration?: number;
+        }): Promise<void>;
+    };
+    keyboard: KeyboardInputPrimitives;
+    scroll: ScrollInputPrimitives;
+    touch: TouchInputPrimitives;
+}
+export interface ComputerInputPrimitives extends InputPrimitives {
+    pointer: PointerInputPrimitives & {
+        doubleClick(p: PointerPoint): Promise<void>;
+        rightClick(p: PointerPoint): Promise<void>;
+        hover(p: PointerPoint): Promise<void>;
+        dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;
+    };
+    keyboard: KeyboardInputPrimitives;
+    scroll: ScrollInputPrimitives;
+}
 export declare abstract class AbstractInterface {
     abstract interfaceType: string;
     abstract screenshotBase64(): Promise<string>;
@@ -28,13 +132,39 @@ export declare abstract class AbstractInterface {
     abstract url?: () => string | Promise<string>;
     abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;
     /**
-     * Get the current time from the device.
-     * Returns the device's current timestamp in milliseconds.
-     * This is useful when the system time and device time are not synchronized.
+     * Get the current device-local time as a formatted string.
+     * Prefer this for user-visible time because timestamps alone do not preserve
+     * the target device's timezone when formatted on the host machine.
      */
-    getTimestamp?(): Promise<number>;
+    getDeviceLocalTimeString?(format?: string): Promise<string>;
     /** URL of native MJPEG stream for real-time screen preview (e.g. WDA MJPEG server) */
     mjpegStreamUrl?: string;
+    /**
+     * Optional in-process MJPEG frame producer. Implementations can push raw
+     * base64 frames here when there is no standalone native MJPEG URL, e.g.
+     * Chromium CDP Page.startScreencast for web previews.
+     */
+    startMjpegStream?(options: MjpegStreamOptions): MjpegStreamHandle | undefined | Promise<MjpegStreamHandle | undefined>;
+    /**
+     * Optional hook used after keyboard-only actions to force a fresh frame on
+     * the active MJPEG stream. Implementations should be a no-op when no stream
+     * is active.
+     */
+    flushPendingVisualUpdate?(): Promise<void>;
+    /**
+     * Optional navigation state probe for browser-like interfaces, used to drive
+     * loading indicators in playground UIs. Returning `undefined` means the
+     * interface does not expose this concept.
+     */
+    navigationState?(): Promise<{
+        isLoading: boolean;
+    }>;
+    /**
+     * Low-level device input surface. Platform implementations expose transport
+     * primitives here; higher-level AI actions and manual pointer dispatch should
+     * adapt to this instead of duplicating platform gesture logic.
+     */
+    inputPrimitives?: InputPrimitives;
 }
 export declare const defineAction: <TSchema extends z.ZodType | undefined = undefined, TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined, TReturn = any>(config: {
     name: string;
@@ -193,7 +323,7 @@ export declare const actionTapParamSchema: z.ZodObject<{
 export type ActionTapParam = {
     locate: LocateResultElement;
 };
-export declare const defineActionTap: (call: (param: ActionTapParam) => Promise<void>) => DeviceAction<ActionTapParam>;
+export declare const defineActionTap: (tap: PointerInputPrimitives["tap"]) => DeviceAction<ActionTapParam>;
 export declare const actionRightClickParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -344,7 +474,7 @@ export declare const actionRightClickParamSchema: z.ZodObject<{
 export type ActionRightClickParam = {
     locate: LocateResultElement;
 };
-export declare const defineActionRightClick: (call: (param: ActionRightClickParam) => Promise<void>) => DeviceAction<ActionRightClickParam>;
+export declare const defineActionRightClick: (rightClick: NonNullable<PointerInputPrimitives["rightClick"]>) => DeviceAction<ActionRightClickParam>;
 export declare const actionDoubleClickParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -495,7 +625,7 @@ export declare const actionDoubleClickParamSchema: z.ZodObject<{
 export type ActionDoubleClickParam = {
     locate: LocateResultElement;
 };
-export declare const defineActionDoubleClick: (call: (param: ActionDoubleClickParam) => Promise<void>) => DeviceAction<ActionDoubleClickParam>;
+export declare const defineActionDoubleClick: (doubleClick: NonNullable<PointerInputPrimitives["doubleClick"]>) => DeviceAction<ActionDoubleClickParam>;
 export declare const actionHoverParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -646,7 +776,7 @@ export declare const actionHoverParamSchema: z.ZodObject<{
 export type ActionHoverParam = {
     locate: LocateResultElement;
 };
-export declare const defineActionHover: (call: (param: ActionHoverParam) => Promise<void>) => DeviceAction<ActionHoverParam>;
+export declare const defineActionHover: (hover: NonNullable<PointerInputPrimitives["hover"]>) => DeviceAction<ActionHoverParam>;
 export declare const actionInputParamSchema: z.ZodObject<{
     value: z.ZodEffects<z.ZodUnion<[z.ZodString, z.ZodNumber]>, string, string | number>;
     locate: z.ZodOptional<z.ZodObject<{
@@ -759,6 +889,7 @@ export declare const actionInputParamSchema: z.ZodObject<{
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough">>>;
     mode: z.ZodDefault<z.ZodEnum<["replace", "clear", "typeOnly"]>>;
+    autoDismissKeyboard: z.ZodOptional<z.ZodBoolean>;
 }, "strip", z.ZodTypeAny, {
     value: string;
     mode: "replace" | "clear" | "typeOnly";
@@ -799,6 +930,7 @@ export declare const actionInputParamSchema: z.ZodObject<{
         cacheable: z.ZodOptional<z.ZodBoolean>;
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
+    autoDismissKeyboard?: boolean | undefined;
 }, {
     value: string | number;
     locate?: z.objectInputType<{
@@ -839,13 +971,15 @@ export declare const actionInputParamSchema: z.ZodObject<{
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
     mode?: "replace" | "clear" | "typeOnly" | undefined;
+    autoDismissKeyboard?: boolean | undefined;
 }>;
 export type ActionInputParam = {
     value: string;
     locate?: LocateResultElement;
     mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
+    autoDismissKeyboard?: boolean;
 };
-export declare const defineActionInput: (call: (param: ActionInputParam) => Promise<void>) => DeviceAction<ActionInputParam>;
+export declare const defineActionInput: (keyboard: KeyboardInputPrimitives) => DeviceAction<ActionInputParam>;
 export declare const actionKeyboardPressParamSchema: z.ZodObject<{
     locate: z.ZodOptional<z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1040,7 +1174,7 @@ export type ActionKeyboardPressParam = {
     locate?: LocateResultElement;
     keyName: string;
 };
-export declare const defineActionKeyboardPress: (call: (param: ActionKeyboardPressParam) => Promise<void>) => DeviceAction<ActionKeyboardPressParam>;
+export declare const defineActionKeyboardPress: (keyboardPress: KeyboardInputPrimitives["keyboardPress"]) => DeviceAction<ActionKeyboardPressParam>;
 export declare const actionScrollParamSchema: z.ZodObject<{
     scrollType: z.ZodDefault<z.ZodEnum<["singleAction", "scrollToBottom", "scrollToTop", "scrollToRight", "scrollToLeft"]>>;
     direction: z.ZodDefault<z.ZodEnum<["down", "up", "right", "left"]>>;
@@ -1156,7 +1290,7 @@ export declare const actionScrollParamSchema: z.ZodObject<{
     }, z.ZodTypeAny, "passthrough">>>;
 }, "strip", z.ZodTypeAny, {
     scrollType: "singleAction" | "scrollToBottom" | "scrollToTop" | "scrollToRight" | "scrollToLeft";
-    direction: "down" | "up" | "right" | "left";
+    direction: "left" | "right" | "down" | "up";
     locate?: z.objectOutputType<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
             prompt: z.ZodString;
@@ -1234,10 +1368,10 @@ export declare const actionScrollParamSchema: z.ZodObject<{
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
     scrollType?: "singleAction" | "scrollToBottom" | "scrollToTop" | "scrollToRight" | "scrollToLeft" | undefined;
-    direction?: "down" | "up" | "right" | "left" | undefined;
+    direction?: "left" | "right" | "down" | "up" | undefined;
     distance?: number | null | undefined;
 }>;
-export declare const defineActionScroll: (call: (param: ActionScrollParam) => Promise<void>) => DeviceAction<ActionScrollParam>;
+export declare const defineActionScroll: (scroll: ScrollInputPrimitives["scroll"]) => DeviceAction<ActionScrollParam>;
 export declare const actionDragAndDropParamSchema: z.ZodObject<{
     from: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1532,7 +1666,7 @@ export type ActionDragAndDropParam = {
     from: LocateResultElement;
     to: LocateResultElement;
 };
-export declare const defineActionDragAndDrop: (call: (param: ActionDragAndDropParam) => Promise<void>) => DeviceAction<ActionDragAndDropParam>;
+export declare const defineActionDragAndDrop: (dragAndDrop: NonNullable<PointerInputPrimitives["dragAndDrop"]>) => DeviceAction<ActionDragAndDropParam>;
 export declare const ActionLongPressParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1643,7 +1777,7 @@ export declare const ActionLongPressParamSchema: z.ZodObject<{
         cacheable: z.ZodOptional<z.ZodBoolean>;
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough">>;
-    duration: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
+    duration: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     locate: {
         prompt: string | ({
@@ -1687,7 +1821,7 @@ export type ActionLongPressParam = {
     locate: LocateResultElement;
     duration?: number;
 };
-export declare const defineActionLongPress: (call: (param: ActionLongPressParam) => Promise<void>) => DeviceAction<ActionLongPressParam>;
+export declare const defineActionLongPress: (longPress: NonNullable<PointerInputPrimitives["longPress"]>) => DeviceAction<ActionLongPressParam>;
 export declare const ActionSwipeParamSchema: z.ZodObject<{
     start: z.ZodOptional<z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1913,7 +2047,7 @@ export declare const ActionSwipeParamSchema: z.ZodObject<{
     repeat: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     duration: number;
-    direction?: "down" | "up" | "right" | "left" | undefined;
+    direction?: "left" | "right" | "down" | "up" | undefined;
     distance?: number | undefined;
     start?: z.objectOutputType<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1991,7 +2125,7 @@ export declare const ActionSwipeParamSchema: z.ZodObject<{
     }, z.ZodTypeAny, "passthrough"> | undefined;
     repeat?: number | undefined;
 }, {
-    direction?: "down" | "up" | "right" | "left" | undefined;
+    direction?: "left" | "right" | "down" | "up" | undefined;
     distance?: number | undefined;
     duration?: number | undefined;
     start?: z.objectInputType<{
@@ -2093,7 +2227,10 @@ export declare function normalizeMobileSwipeParam(param: ActionSwipeParam, scree
     duration: number;
     repeatCount: number;
 };
-export declare const defineActionSwipe: (call: (param: ActionSwipeParam) => Promise<void>) => DeviceAction<ActionSwipeParam>;
+export declare const defineActionSwipe: (config: {
+    swipe: TouchInputPrimitives["swipe"];
+    size(): Promise<Size>;
+}) => DeviceAction<ActionSwipeParam>;
 export declare const actionClearInputParamSchema: z.ZodObject<{
     locate: z.ZodOptional<z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -2284,22 +2421,25 @@ export declare const actionClearInputParamSchema: z.ZodObject<{
 export type ActionClearInputParam = {
     locate?: LocateResultElement;
 };
-export declare const defineActionClearInput: (call: (param: ActionClearInputParam) => Promise<void>) => DeviceAction<ActionClearInputParam>;
+export declare const defineActionClearInput: (clearInput: KeyboardInputPrimitives["clearInput"]) => DeviceAction<ActionClearInputParam>;
 export declare const actionCursorMoveParamSchema: z.ZodObject<{
     direction: z.ZodEnum<["left", "right"]>;
     times: z.ZodDefault<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
-    direction: "right" | "left";
+    direction: "left" | "right";
     times: number;
 }, {
-    direction: "right" | "left";
+    direction: "left" | "right";
     times?: number | undefined;
 }>;
 export type ActionCursorMoveParam = {
     direction: 'left' | 'right';
     times?: number;
 };
-export declare const defineActionCursorMove: (call: (param: ActionCursorMoveParam) => Promise<void>) => DeviceAction<ActionCursorMoveParam>;
+export declare const defineActionCursorMove: (config: {
+    keyboard: Pick<KeyboardInputPrimitives, "keyboardPress" | "cursorMove">;
+    sleep?(timeMs: number): Promise<void>;
+}) => DeviceAction<ActionCursorMoveParam>;
 export declare const ActionPinchParamSchema: z.ZodObject<{
     locate: z.ZodOptional<z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -2502,7 +2642,10 @@ export type ActionPinchParam = {
     distance?: number;
     duration?: number;
 };
-export declare const defineActionPinch: (call: (param: ActionPinchParam) => Promise<void>) => DeviceAction<ActionPinchParam>;
+export declare const defineActionPinch: (config: {
+    pinch: TouchInputPrimitives["pinch"];
+    size(): Promise<Size>;
+}) => DeviceAction<ActionPinchParam> | undefined;
 export declare function normalizePinchParam(param: ActionPinchParam, screenSize: {
     width: number;
     height: number;
@@ -2513,6 +2656,34 @@ export declare function normalizePinchParam(param: ActionPinchParam, screenSize:
     endDistance: number;
     duration: number;
 };
+export interface MobileInputActionContext {
+    input: MobileInputPrimitives;
+    size(): Promise<Size>;
+    sleep?(timeMs: number): Promise<void>;
+    getDefaultAutoDismissKeyboard?(): boolean | undefined;
+    systemActions?: SystemInputActionOptions;
+}
+export interface SystemInputActionConfig {
+    name: string;
+    description: string;
+    interfaceAlias?: string;
+    delayBeforeRunner?: number;
+    delayAfterRunner?: number;
+}
+export interface SystemInputActionOptions {
+    backButton?: SystemInputActionConfig;
+    homeButton?: SystemInputActionConfig;
+    recentAppsButton?: SystemInputActionConfig;
+}
+export interface InputPrimitiveActionOptions {
+    size?: () => Promise<Size>;
+    sleep?: (timeMs: number) => Promise<void>;
+    includeSwipe?: boolean;
+    includePinch?: boolean;
+    systemActions?: SystemInputActionOptions;
+}
+export declare function defineActionsFromInputPrimitives(input: InputPrimitives, options?: InputPrimitiveActionOptions): DeviceAction<any>[];
+export declare function createDefaultMobileActions(context: MobileInputActionContext): DeviceAction<any>[];
 export declare const ActionSleepParamSchema: z.ZodObject<{
     timeMs: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
 }, "strip", z.ZodTypeAny, {

package/dist/types/dump/screenshot-store.d.ts

@@ -36,12 +36,12 @@ export declare class ScreenshotStore {
         mode: 'inline' | 'directory';
         reportPath: string;
         screenshotsDir?: string;
-        writeInlineImage?: (id: string, base64: string) => void;
+        writeInlineImage?: (id: string, base64: string) => void | Promise<void>;
         alsoWriteFileCopy?: boolean;
         /** @deprecated Use alsoWriteFileCopy instead. */
         ensureFileCopy?: boolean;
     });
-    persist(screenshot: ScreenshotItem): ScreenshotRef;
+    persist(screenshot: ScreenshotItem): Promise<ScreenshotRef>;
     private persistToSharedFileIfNeeded;
     loadBase64(refInput: unknown): string;
     cleanup(): void;

package/dist/types/index.d.ts

@@ -5,7 +5,6 @@ import { getVersion } from './utils';
 export { plan, AiLocateElement, runConnectivityTest, getMidsceneLocationSchema, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, type ConnectivityCheckResultItem, type ConnectivityTestConfig, type ConnectivityTestResult, } from './ai-model/index';
 export { MIDSCENE_MODEL_NAME, type CreateOpenAIClientFn, } from '@aiscene/shared/env';
 export type * from './types';
-export type { DeviceAction, } from './types';
 export { ServiceError, ExecutionDump, ReportActionDump, GroupedActionDump, type IExecutionDump, type IReportActionDump, type IGroupedActionDump, type ReportMeta, type GroupMeta, } from './types';
 export { z };
 export default Service;

package/dist/types/report-generator.d.ts

@@ -49,12 +49,14 @@ export declare class ReportGenerator implements IReportGenerator {
         screenshotMode: 'inline' | 'directory';
         persistExecutionDump?: boolean;
         autoPrint?: boolean;
+        reuseExistingReport?: boolean;
     });
     static create(reportFileName: string, opts: {
         generateReport?: boolean;
         persistExecutionDump?: boolean;
         outputFormat?: 'single-html' | 'html-and-external-assets';
         autoPrintReportMsg?: boolean;
+        reuseExistingReport?: boolean;
     }): IReportGenerator;
     onExecutionUpdate(execution: ExecutionDump, reportMeta: ReportMeta, attributes?: ReportAttributes): void;
     flush(): Promise<void>;
@@ -73,6 +75,11 @@ export declare class ReportGenerator implements IReportGenerator {
      * Append-only inline mode: write new screenshots and a dump tag on every call.
      * The frontend deduplicates executions with the same id/name (keeps last).
      * Duplicate dump JSON is acceptable; only screenshots are deduplicated.
+     *
+     * All writes go through `fs/promises` so they run on libuv's thread pool
+     * rather than blocking the Node event loop. A long agent run previously
+     * appended multi-MB dumps (screenshots + serialized tasks) per progress
+     * tick on the main thread, starving IPC and UI for >10s per stall.
      */
     private writeInlineExecution;
     private writeDirectoryExecution;