@godscene/core 1.7.11

package/dist/types/common.d.ts

@@ -0,0 +1,288 @@
+import type { BaseElement, DeviceAction, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from './types';
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+import type { PlanningLocateParam } from './types';
+import type { TModelFamily } from '@godscene/shared/env';
+import { z } from 'zod';
+export type AIArgs = ChatCompletionMessageParam[];
+type AdaptBboxInput = number[] | string[] | string | (number[] | string[])[];
+/**
+ * Convert a point coordinate [0, 1000] to a small bbox [0, 1000]
+ * Creates a small bbox around the center point in the same coordinate space
+ *
+ * @param x - X coordinate in [0, 1000] range
+ * @param y - Y coordinate in [0, 1000] range
+ * @param bboxSize - Size of the bbox to create (default: 20)
+ * @returns [x1, y1, x2, y2] bbox in [0, 1000] coordinate space
+ */
+export declare function pointToBbox(x: number, y: number, bboxSize?: number): [number, number, number, number];
+export declare function fillBboxParam(locate: PlanningLocateParam, width: number, height: number, modelFamily: TModelFamily | undefined): PlanningLocateParam;
+export declare function adaptQwen2_5Bbox(bbox: number[]): [number, number, number, number];
+export declare function adaptGpt5Bbox(bbox: number[] | string[] | string): [number, number, number, number];
+export declare function adaptDoubaoBbox(bbox: string[] | number[] | string, width: number, height: number): [number, number, number, number];
+export declare function adaptBbox(bbox: AdaptBboxInput, width: number, height: number, modelFamily: TModelFamily | undefined): [number, number, number, number];
+export declare function normalized01000(bbox: number[], width: number, height: number): [number, number, number, number];
+export declare function adaptGeminiBbox(bbox: number[], width: number, height: number): [number, number, number, number];
+export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number, rightLimit?: number, bottomLimit?: number, modelFamily?: TModelFamily | undefined, scale?: number): Rect;
+export declare function mergeRects(rects: Rect[]): {
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+};
+/**
+ * Expand the search area to at least 400 x 400 pixels
+ *
+ * Step 1: Extend 100px on each side (top, right, bottom, left)
+ * - If the element is near a boundary, expansion on that side will be limited
+ * - No compensation is made for boundary limitations (this is intentional)
+ *
+ * Step 2: Ensure the area is at least 400x400 pixels
+ * - Scale up proportionally from the center if needed
+ * - Final result is clamped to screen boundaries
+ */
+export declare function expandSearchArea(rect: Rect, screenSize: Size): Rect;
+export declare function markupImageForLLM(screenshotBase64: string, tree: ElementTreeNode<BaseElement>, size: Size): Promise<string>;
+export declare function buildYamlFlowFromPlans(plans: PlanningAction[], actionSpace: DeviceAction<any>[]): MidsceneYamlFlowItem[];
+export declare const PointSchema: z.ZodObject<{
+    left: z.ZodNumber;
+    top: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    left: number;
+    top: number;
+}, {
+    left: number;
+    top: number;
+}>;
+export declare const SizeSchema: z.ZodObject<{
+    width: z.ZodNumber;
+    height: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    width: number;
+    height: number;
+}, {
+    width: number;
+    height: number;
+}>;
+export declare const RectSchema: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
+    left: z.ZodNumber;
+    top: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    left: number;
+    top: number;
+}, {
+    left: number;
+    top: number;
+}>, z.ZodObject<{
+    width: z.ZodNumber;
+    height: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    width: number;
+    height: number;
+}, {
+    width: number;
+    height: number;
+}>>, z.ZodObject<{
+    zoom: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    zoom?: number | undefined;
+}, {
+    zoom?: number | undefined;
+}>>;
+export declare const TMultimodalPromptSchema: z.ZodObject<{
+    images: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        url: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        url: string;
+    }, {
+        name: string;
+        url: string;
+    }>, "many">>;
+    convertHttpImage2Base64: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    images?: {
+        name: string;
+        url: string;
+    }[] | undefined;
+    convertHttpImage2Base64?: boolean | undefined;
+}, {
+    images?: {
+        name: string;
+        url: string;
+    }[] | undefined;
+    convertHttpImage2Base64?: boolean | undefined;
+}>;
+export declare const TUserPromptSchema: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+    prompt: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    prompt: string;
+}, {
+    prompt: string;
+}>, z.ZodObject<{
+    images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        url: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        url: string;
+    }, {
+        name: string;
+        url: string;
+    }>, "many">>>;
+    convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+}, "strip", z.ZodTypeAny, {
+    images?: {
+        name: string;
+        url: string;
+    }[] | undefined;
+    convertHttpImage2Base64?: boolean | undefined;
+}, {
+    images?: {
+        name: string;
+        url: string;
+    }[] | undefined;
+    convertHttpImage2Base64?: boolean | undefined;
+}>>]>;
+export type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;
+export type TUserPrompt = z.infer<typeof TUserPromptSchema>;
+/**
+ * Returns the schema for locator fields.
+ * This now returns the input schema which is more permissive and suitable for validation.
+ */
+export declare const getMidsceneLocationSchema: () => z.ZodObject<{
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepLocate: z.ZodOptional<z.ZodBoolean>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepLocate: z.ZodOptional<z.ZodBoolean>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepLocate: z.ZodOptional<z.ZodBoolean>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
+}, z.ZodTypeAny, "passthrough">>;
+export declare const ifMidsceneLocatorField: (field: any) => boolean;
+export declare const dumpMidsceneLocatorField: (field: any) => string;
+export declare const findAllMidsceneLocatorField: (zodType?: z.ZodType<any>, requiredOnly?: boolean) => string[];
+export declare const dumpActionParam: (jsonObject: Record<string, any>, zodSchema: z.ZodType<any>) => Record<string, any>;
+/**
+ * Parse and validate action parameters using Zod schema.
+ * All fields are validated through Zod, EXCEPT locator fields which are skipped.
+ * Default values defined in the schema are automatically applied.
+ *
+ * Locator fields are special business logic fields with complex validation requirements,
+ * so they are intentionally excluded from Zod parsing and use existing validation logic.
+ *
+ * When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields
+ * are transformed from screenshot space to logical space.
+ */
+export declare const parseActionParam: (rawParam: Record<string, any> | undefined, zodSchema?: z.ZodType<any>, options?: {
+    shrunkShotToLogicalRatio?: number;
+}) => Record<string, any> | undefined;
+export declare const finalizeActionName = "Finalize";
+/**
+ * Get a readable time string for a given timestamp or the current time
+ * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'
+ * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.
+ * @returns A formatted time string with format label
+ */
+export declare const getReadableTimeString: (format?: string, timestamp?: number) => string;
+export {};

package/dist/types/device/device-options.d.ts

@@ -0,0 +1,155 @@
+import type { DeviceAction } from '../types';
+/**
+ * Android device input options
+ */
+export type AndroidDeviceInputOpt = {
+    /** Automatically dismiss the keyboard after input is completed */
+    autoDismissKeyboard?: boolean;
+    /** Strategy for dismissing the keyboard: 'esc-first' tries ESC before BACK, 'back-first' tries BACK before ESC */
+    keyboardDismissStrategy?: 'esc-first' | 'back-first';
+};
+/**
+ * Android device options
+ */
+export type AndroidDeviceOpt = {
+    /** Path to the ADB executable */
+    androidAdbPath?: string;
+    /** Remote ADB host address */
+    remoteAdbHost?: string;
+    /** Remote ADB port */
+    remoteAdbPort?: number;
+    /** Input method editor strategy: 'always-yadb' always uses yadb, 'yadb-for-non-ascii' uses yadb only for non-ASCII characters */
+    imeStrategy?: 'always-yadb' | 'yadb-for-non-ascii';
+    /** Display ID to use for this device */
+    displayId?: number;
+    /** Use physical display ID for screenshot operations */
+    usePhysicalDisplayIdForScreenshot?: boolean;
+    /** Use physical display ID when looking up display information */
+    usePhysicalDisplayIdForDisplayLookup?: boolean;
+    /** Custom device actions to register */
+    customActions?: DeviceAction<any>[];
+    /**
+     * @deprecated This option has been removed and no longer has any effect.
+     * Use `screenshotShrinkFactor` in AgentOpt instead to control screenshot size sent to AI model.
+     */
+    screenshotResizeScale?: number;
+    /** Always fetch screen info on each call; if false, cache the first result */
+    alwaysRefreshScreenInfo?: boolean;
+    /**
+     * Screenshot buffer size validation threshold in bytes. Buffers below this
+     * value are treated as failed or corrupted captures. Defaults to 1024 (1KB).
+     * Set to 0 to skip only this size check; empty-buffer and image-format
+     * validation still run.
+     */
+    minScreenshotBufferSize?: number;
+    /**
+     * Scrcpy screenshot configuration for high-performance screen capture.
+     *
+     * Scrcpy provides 6-8x faster screenshots by streaming H.264 video from the device.
+     * When enabled, scrcpy will:
+     * 1. Start a video stream from the device on first screenshot request
+     * 2. Keep the connection alive for subsequent screenshots (16-50ms each)
+     * 3. Automatically disconnect after idle timeout to save resources
+     * 4. Fallback to standard ADB mode if unavailable
+     *
+     * @example
+     * ```typescript
+     * // Enable scrcpy for high-performance screenshots
+     * const device = new AndroidDevice(deviceId, {
+     *   scrcpyConfig: {
+     *     enabled: true,
+     *   },
+     * });
+     *
+     * // Custom configuration
+     * const device = new AndroidDevice(deviceId, {
+     *   scrcpyConfig: {
+     *     enabled: true,
+     *     maxSize: 0,        // 0 = no scaling
+     *     idleTimeoutMs: 30000,
+     *     videoBitRate: 8_000_000,
+     *   },
+     * });
+     * ```
+     */
+    scrcpyConfig?: {
+        /**
+         * Enable scrcpy for high-performance screenshots.
+         * @default false
+         */
+        enabled?: boolean;
+        /**
+         * Maximum video dimension (width or height).
+         * Video stream will be scaled down if device resolution exceeds this value.
+         * Lower values reduce bandwidth but may affect image quality.
+         *
+         * @default 0 (no scaling, use original resolution)
+         * @example
+         * { maxSize: 1024 } // Always scale to 1024
+         */
+        maxSize?: number;
+        /**
+         * Idle timeout in milliseconds before disconnecting scrcpy.
+         * Connection auto-closes after this period of inactivity to save resources.
+         * Set to 0 to disable auto-disconnect.
+         * @default 30000 (30 seconds)
+         */
+        idleTimeoutMs?: number;
+        /**
+         * Video bit rate for H.264 encoding in bits per second.
+         * Higher values improve quality but increase bandwidth usage.
+         * @default 2000000 (2 Mbps)
+         */
+        videoBitRate?: number;
+    };
+} & AndroidDeviceInputOpt;
+/**
+ * iOS device input options
+ */
+export type IOSDeviceInputOpt = {
+    /** Automatically dismiss the keyboard after input is completed */
+    autoDismissKeyboard?: boolean;
+};
+/**
+ * iOS device options
+ */
+export type IOSDeviceOpt = {
+    /** Device ID (UDID) to connect to */
+    deviceId?: string;
+    /**
+     * Optional npm module path used to override the default iOS device implementation.
+     * The target module must export an `IOSDevice` class (or default export) compatible with Midscene's iOS device interface.
+     */
+    iOSDeviceClassOverride?: string;
+    /** Custom device actions to register */
+    customActions?: DeviceAction<any>[];
+    /** WebDriverAgent port (default: 8100) */
+    wdaPort?: number;
+    /** WebDriverAgent host (default: 'localhost') */
+    wdaHost?: string;
+    /** Whether to use WebDriverAgent */
+    useWDA?: boolean;
+    /** WDA MJPEG server port for real-time screen streaming (default: 9100) */
+    wdaMjpegPort?: number;
+} & IOSDeviceInputOpt;
+/**
+ * HarmonyOS device input options
+ */
+export type HarmonyDeviceInputOpt = {
+    /** Automatically dismiss the keyboard after input is completed */
+    autoDismissKeyboard?: boolean;
+};
+/**
+ * HarmonyOS device options
+ */
+export type HarmonyDeviceOpt = {
+    /** Path to the HDC executable */
+    hdcPath?: string;
+    /** Custom device actions to register */
+    customActions?: DeviceAction<any>[];
+    /**
+     * @deprecated This option has been removed and no longer has any effect.
+     * Use `screenshotShrinkFactor` in AgentOpt instead to control screenshot size sent to AI model.
+     */
+    screenshotResizeScale?: number;
+} & HarmonyDeviceInputOpt;