@xiuchang-midscene/android 1.6.0

package/dist/types/cli.d.ts

@@ -0,0 +1 @@
+export { }

package/dist/types/index.d.ts

@@ -0,0 +1,465 @@
+import { AbstractInterface } from '@midscene/core/device';
+import type { ActionParam } from '@midscene/core';
+import type { ActionReturn } from '@midscene/core';
+import { ADB } from 'appium-adb';
+import type { Adb } from '@yume-chan/adb';
+import { Agent } from '@midscene/core/agent';
+import { AgentOpt } from '@midscene/core/agent';
+import { AndroidDeviceInputOpt } from '@midscene/core/device';
+import { AndroidDeviceOpt } from '@midscene/core/device';
+import { BaseMidsceneTools } from '@midscene/shared/mcp';
+import { Device } from 'appium-adb';
+import { DeviceAction } from '@midscene/core';
+import type { ElementInfo } from '@midscene/shared/extractor';
+import { InterfaceType } from '@midscene/core';
+import { overrideAIConfig } from '@midscene/shared/env';
+import { Point } from '@midscene/core';
+import { Size } from '@midscene/core';
+import { ToolDefinition } from '@midscene/shared/mcp';
+
+declare type ActionArgs<T extends DeviceAction> = [ActionParam<T>] extends [undefined] ? [] : [ActionParam<T>];
+
+export declare function agentFromAdbDevice(deviceId?: string, opts?: AndroidAgentOpt & AndroidDeviceOpt): Promise<AndroidAgent>;
+
+export declare class AndroidAgent extends Agent<AndroidDevice> {
+    /**
+     * Trigger the system back operation on Android devices
+     */
+    back: WrappedAction<DeviceActionAndroidBackButton>;
+    /**
+     * Trigger the system home operation on Android devices
+     */
+    home: WrappedAction<DeviceActionAndroidHomeButton>;
+    /**
+     * Trigger the system recent apps operation on Android devices
+     */
+    recentApps: WrappedAction<DeviceActionAndroidRecentAppsButton>;
+    /**
+     * User-provided app name to package name mapping
+     */
+    private appNameMapping;
+    constructor(device: AndroidDevice, opts?: AndroidAgentOpt);
+    /**
+     * Launch an Android app or URL
+     * @param uri - App package name, URL, or app name to launch
+     */
+    launch(uri: string): Promise<void>;
+    /**
+     * Execute ADB shell command on Android device
+     * @param command - ADB shell command to execute
+     */
+    runAdbShell(command: string): Promise<string>;
+    private createActionWrapper;
+}
+
+export declare type AndroidAgentOpt = AgentOpt & {
+    /**
+     * Custom mapping of app names to package names
+     * User-provided mappings will take precedence over default mappings
+     */
+    appNameMapping?: Record<string, string>;
+};
+
+export declare interface AndroidConnectedDevice extends Device {
+    model?: string;
+    brand?: string;
+    resolution?: string;
+    density?: number;
+}
+
+export declare class AndroidDevice implements AbstractInterface {
+    private deviceId;
+    private yadbPushed;
+    private devicePixelRatio;
+    private devicePixelRatioInitialized;
+    private adb;
+    private connectingAdb;
+    private destroyed;
+    private description;
+    private customActions?;
+    private cachedScreenSize;
+    private cachedOrientation;
+    private cachedPhysicalDisplayId;
+    private scrcpyAdapter;
+    private appNameMapping;
+    private cachedAdjustScale;
+    private takeScreenshotFailCount;
+    private static readonly TAKE_SCREENSHOT_FAIL_THRESHOLD;
+    interfaceType: InterfaceType;
+    uri: string | undefined;
+    /** Set by AndroidAgent to provide AI-based verification for IME fallback. */
+    inputVerifyFn?: (text: string) => Promise<boolean>;
+    options?: AndroidDeviceOpt;
+    actionSpace(): DeviceAction<any>[];
+    constructor(deviceId: string, options?: AndroidDeviceOpt);
+    describe(): string;
+    connect(): Promise<ADB>;
+    getAdb(): Promise<ADB>;
+    private createAdbProxy;
+    /**
+     * Get or create the scrcpy adapter (lazy initialization)
+     */
+    private getScrcpyAdapter;
+    /**
+     * Get device physical info needed by scrcpy adapter
+     */
+    private getDevicePhysicalInfo;
+    /**
+     * Set the app name to package name mapping
+     */
+    setAppNameMapping(mapping: Record<string, string>): void;
+    /**
+     * Resolve app name to package name using the mapping
+     * Comparison is case-insensitive and ignores spaces, dashes, and underscores.
+     * Keys in appNameMapping are pre-normalized, so we only need to normalize the input.
+     * @param appName The app name to resolve
+     */
+    private resolvePackageName;
+    launch(uri: string): Promise<AndroidDevice>;
+    execYadb(keyboardContent: string): Promise<void>;
+    /**
+     * Write text to the device clipboard using yadb's -writeClipboard command.
+     * Requires yadb v1.1.0+.
+     */
+    execYadbWriteClipboard(text: string): Promise<void>;
+    /**
+     * Input text via ADBKeyboard IME.
+     * Flow:
+     *   1. Record the current default IME.
+     *   2. Enable and switch to ADBKeyboard, wait for activation.
+     *   3. Send text via broadcast (base64-encoded to handle any Unicode safely).
+     *   4. Restore the original IME.
+     *
+     * Requires ADBKeyboard (com.android.adbkeyboard) to be installed on the device.
+     */
+    private typeViaAdbKeyboard;
+    getElementsInfo(): Promise<ElementInfo[]>;
+    getElementsNodeTree(): Promise<any>;
+    getScreenSize(): Promise<{
+        override: string;
+        physical: string;
+        orientation: number;
+        isCurrentOrientation?: boolean;
+    }>;
+    private initializeDevicePixelRatio;
+    getDisplayDensity(): Promise<number>;
+    getDisplayOrientation(): Promise<number>;
+    /**
+     * Get physical screen dimensions adjusted for current orientation.
+     * Swaps width/height when the device is in landscape and the reported
+     * dimensions do not already reflect the current orientation.
+     */
+    private getOrientedPhysicalSize;
+    size(): Promise<Size>;
+    /**
+     * Compute and cache the coordinate adjustment scale by comparing
+     * physical dimensions with logical dimensions from size().
+     * Cached after first call; invalidated on destroy().
+     */
+    private getAdjustScale;
+    /**
+     * Convert logical coordinates (from AI) back to physical coordinates (for ADB).
+     * The ratio is derived from size(), so overriding size() alone is sufficient.
+     */
+    private adjustCoordinates;
+    /**
+     * Calculate the end point for scroll operations based on start point, scroll delta, and screen boundaries.
+     * This method ensures that scroll operations stay within screen bounds and maintain a minimum scroll distance
+     * for effective scrolling gestures on Android devices.
+     *
+     * @param start - The starting point of the scroll gesture
+     * @param deltaX - The horizontal scroll distance (positive = scroll right, negative = scroll left)
+     * @param deltaY - The vertical scroll distance (positive = scroll down, negative = scroll up)
+     * @param maxWidth - The maximum width boundary (screen width)
+     * @param maxHeight - The maximum height boundary (screen height)
+     * @returns The calculated end point for the scroll gesture
+     */
+    private calculateScrollEndPoint;
+    screenshotBase64(): Promise<string>;
+    clearInput(element?: ElementInfo): Promise<void>;
+    forceScreenshot(path: string): Promise<void>;
+    url(): Promise<string>;
+    scrollUntilTop(startPoint?: Point): Promise<void>;
+    scrollUntilBottom(startPoint?: Point): Promise<void>;
+    scrollUntilLeft(startPoint?: Point): Promise<void>;
+    scrollUntilRight(startPoint?: Point): Promise<void>;
+    scrollUp(distance?: number, startPoint?: Point): Promise<void>;
+    scrollDown(distance?: number, startPoint?: Point): Promise<void>;
+    scrollLeft(distance?: number, startPoint?: Point): Promise<void>;
+    scrollRight(distance?: number, startPoint?: Point): Promise<void>;
+    ensureYadb(): Promise<void>;
+    /**
+     * Check if text contains characters that may cause issues with ADB inputText.
+     * appium-adb's inputText has known bugs with certain characters:
+     * - Backslash causes broken shell quoting
+     * - Backtick is not escaped at all
+     * - Text containing both " and ' throws an error
+     * - Dollar sign can cause variable expansion issues
+     *
+     * For these characters, we route through yadb which handles them correctly
+     * via escapeForShell + double-quoted shell context.
+     */
+    private shouldUseYadbForText;
+    /**
+     * Execute text input using a specific strategy.
+     * Pure execution — no keyboard-dismiss logic, no strategy resolution.
+     */
+    private _typeWithStrategy;
+    keyboardType(text: string, options?: AndroidDeviceInputOpt): Promise<void>;
+    private normalizeKeyName;
+    keyboardPress(key: string): Promise<void>;
+    mouseClick(x: number, y: number): Promise<void>;
+    mouseDoubleClick(x: number, y: number): Promise<void>;
+    mouseMove(): Promise<void>;
+    mouseDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }, duration?: number): Promise<void>;
+    scroll(deltaX: number, deltaY: number, duration?: number): Promise<void>;
+    destroy(): Promise<void>;
+    /**
+     * Get the current time from the Android device.
+     * Returns the device's current timestamp in milliseconds.
+     * This is useful when the system time and device time are not synchronized.
+     */
+    getTimestamp(): Promise<number>;
+    back(): Promise<void>;
+    home(): Promise<void>;
+    recentApps(): Promise<void>;
+    longPress(x: number, y: number, duration?: number): Promise<void>;
+    pullDown(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
+    pullDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }, duration: number): Promise<void>;
+    pullUp(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
+    private getDisplayArg;
+    getPhysicalDisplayId(): Promise<string | null>;
+    hideKeyboard(options?: AndroidDeviceInputOpt, timeoutMs?: number): Promise<boolean>;
+}
+
+/**
+ * Android-specific tools manager
+ * Extends BaseMidsceneTools to provide Android ADB device connection tools
+ */
+export declare class AndroidMidsceneTools extends BaseMidsceneTools<AndroidAgent> {
+    protected createTemporaryDevice(): AndroidDevice;
+    protected ensureAgent(deviceId?: string): Promise<AndroidAgent>;
+    /**
+     * Provide Android-specific platform tools
+     */
+    protected preparePlatformTools(): ToolDefinition[];
+}
+
+declare type DeviceActionAndroidBackButton = DeviceAction<undefined, void>;
+
+declare type DeviceActionAndroidHomeButton = DeviceAction<undefined, void>;
+
+declare type DeviceActionAndroidRecentAppsButton = DeviceAction<undefined, void>;
+
+declare interface DevicePhysicalInfo {
+    physicalWidth: number;
+    physicalHeight: number;
+    dpr: number;
+    orientation: number;
+    isCurrentOrientation?: boolean;
+}
+
+export declare function getConnectedDevices(): Promise<Device[]>;
+
+export declare function getConnectedDevicesWithDetails(): Promise<AndroidConnectedDevice[]>;
+
+export { overrideAIConfig }
+
+declare interface ResolvedScrcpyConfig {
+    enabled: boolean;
+    maxSize: number;
+    videoBitRate: number;
+    idleTimeoutMs: number;
+}
+
+declare interface ScrcpyConfig {
+    enabled?: boolean;
+    maxSize?: number;
+    videoBitRate?: number;
+    idleTimeoutMs?: number;
+}
+
+/**
+ * Adapter that encapsulates all scrcpy-related logic for AndroidDevice.
+ * Handles config normalization, manager lifecycle, screenshot, and resolution.
+ */
+export declare class ScrcpyDeviceAdapter {
+    private deviceId;
+    private scrcpyConfig;
+    private manager;
+    private resolvedConfig;
+    private initFailed;
+    constructor(deviceId: string, scrcpyConfig: ScrcpyConfig | undefined);
+    isEnabled(): boolean;
+    /**
+     * Initialize scrcpy connection. Called once during device.connect().
+     * If initialization fails, marks scrcpy as permanently disabled (no further retries).
+     */
+    initialize(deviceInfo: DevicePhysicalInfo): Promise<void>;
+    /**
+     * Resolve scrcpy config.
+     * maxSize defaults to 0 (no scaling, full physical resolution) so the Agent layer
+     * receives the highest quality image for AI processing.
+     * videoBitRate is auto-scaled based on physical pixel count to ensure
+     * sufficient quality for all-I-frame H.264 encoding.
+     */
+    resolveConfig(deviceInfo: DevicePhysicalInfo): ResolvedScrcpyConfig;
+    /**
+     * Get or create the ScrcpyScreenshotManager.
+     * Uses dynamic import for @yume-chan packages (ESM-only, must use await import in CJS builds).
+     */
+    ensureManager(deviceInfo: DevicePhysicalInfo): Promise<ScrcpyScreenshotManager>;
+    /**
+     * Take a screenshot via scrcpy, returns base64 string.
+     * Throws on failure (caller should fallback to ADB).
+     */
+    screenshotBase64(deviceInfo: DevicePhysicalInfo): Promise<string>;
+    /**
+     * Get scrcpy's actual video resolution.
+     * Returns null if scrcpy is not connected yet.
+     */
+    getResolution(): {
+        width: number;
+        height: number;
+    } | null;
+    /**
+     * Compute size from scrcpy resolution.
+     * Returns null if scrcpy is not connected.
+     */
+    getSize(deviceInfo: DevicePhysicalInfo): Size | null;
+    /**
+     * Calculate the scaling ratio from physical to scrcpy resolution.
+     */
+    getScalingRatio(physicalWidth: number): number | null;
+    disconnect(): Promise<void>;
+}
+
+declare class ScrcpyScreenshotManager {
+    private adb;
+    private scrcpyClient;
+    private videoStream;
+    private spsHeader;
+    private idleTimer;
+    private isConnecting;
+    private isInitialized;
+    private options;
+    private ffmpegAvailable;
+    private keyframeResolvers;
+    private lastRawKeyframe;
+    private videoResolution;
+    private streamReader;
+    constructor(adb: Adb, options?: ScrcpyScreenshotOptions);
+    /**
+     * Validate environment prerequisites (ffmpeg, scrcpy-server, etc.)
+     * Must be called once after construction, before any screenshot operations.
+     * Throws if prerequisites are not met.
+     */
+    validateEnvironment(): Promise<void>;
+    /**
+     * Ensure scrcpy connection is active
+     */
+    ensureConnected(): Promise<void>;
+    /**
+     * Resolve path to scrcpy server binary
+     */
+    private resolveServerBinPath;
+    /**
+     * Get ffmpeg executable path
+     * Priority: @ffmpeg-installer/ffmpeg > system ffmpeg
+     */
+    private getFfmpegPath;
+    /**
+     * Consume video frames and keep latest frame
+     */
+    private startFrameConsumer;
+    /**
+     * Main frame consumption loop
+     * Includes busy-loop detection: if reader.read() resolves too fast
+     * (e.g. broken stream returning immediately), we throttle to prevent 100% CPU.
+     */
+    private consumeFramesLoop;
+    /**
+     * Process a single video packet from the scrcpy stream.
+     * With sendFrameMeta: true, the stream emits properly framed packets:
+     * - "configuration" packets contain SPS/PPS header data
+     * - "data" packets contain complete video frames with correct boundaries
+     * This avoids the frame-splitting issue that occurs with sendFrameMeta: false
+     * at high resolutions where raw chunks may not align with frame boundaries.
+     */
+    private processFrame;
+    /**
+     * Get screenshot as JPEG.
+     * Tries to get a fresh frame within a short timeout. If the screen is static
+     * (no new frames arrive), falls back to the latest cached keyframe.
+     */
+    getScreenshotJpeg(): Promise<Buffer>;
+    /**
+     * Get the actual video stream resolution
+     * Returns null if scrcpy is not connected yet
+     */
+    getResolution(): {
+        width: number;
+        height: number;
+    } | null;
+    /**
+     * Notify all pending keyframe waiters
+     */
+    private notifyKeyframeWaiters;
+    /**
+     * Wait for the next keyframe to arrive
+     */
+    private waitForNextKeyframe;
+    /**
+     * Ensure ffmpeg is available for PNG conversion
+     */
+    private ensureFfmpegAvailable;
+    /**
+     * Wait for first keyframe with SPS/PPS header
+     */
+    private waitForKeyframe;
+    /**
+     * Check if ffmpeg is available in the system
+     */
+    private checkFfmpegAvailable;
+    /**
+     * Decode H.264 data to JPEG using ffmpeg
+     */
+    private decodeH264ToJpeg;
+    /**
+     * Reset idle timeout timer
+     */
+    private resetIdleTimer;
+    /**
+     * Disconnect scrcpy
+     */
+    disconnect(): Promise<void>;
+    /**
+     * Check if scrcpy is initialized and connected
+     */
+    isConnected(): boolean;
+}
+
+declare interface ScrcpyScreenshotOptions {
+    maxSize?: number;
+    videoBitRate?: number;
+    idleTimeoutMs?: number;
+}
+
+/**
+ * Helper type to convert DeviceAction to wrapped method signature
+ */
+declare type WrappedAction<T extends DeviceAction> = (...args: ActionArgs<T>) => Promise<ActionReturn<T>>;
+
+export { }