@aiscene/android 1.3.4

package/dist/types/index.d.ts

@@ -0,0 +1,225 @@
+import { AbstractInterface } from '@midscene/core/device';
+import type { ActionParam } from '@midscene/core';
+import type { ActionReturn } from '@midscene/core';
+import { ADB } from 'appium-adb';
+import { Agent } from '@midscene/core/agent';
+import { AgentOpt } from '@midscene/core/agent';
+import { AndroidDeviceInputOpt } from '@midscene/core/device';
+import { AndroidDeviceOpt } from '@midscene/core/device';
+import { Device } from 'appium-adb';
+import { DeviceAction } from '@midscene/core';
+import { ElementCacheFeature } from '@midscene/core';
+import type { ElementInfo } from '@midscene/shared/extractor';
+import { IModelConfig } from '@midscene/shared/env';
+import { InterfaceType } from '@midscene/core';
+import { overrideAIConfig } from '@midscene/shared/env';
+import { Point } from '@midscene/core';
+import { Rect } from '@midscene/core';
+import { Size } from '@midscene/core';
+
+declare type ActionArgs<T extends DeviceAction> = [ActionParam<T>] extends [undefined] ? [] : [ActionParam<T>];
+
+export declare function agentFromAdbDevice(deviceId?: string, opts?: AndroidAgentOpt & AndroidDeviceOpt): Promise<AndroidAgent>;
+
+export declare class AndroidAgent extends Agent<AndroidDevice> {
+    /**
+     * Trigger the system back operation on Android devices
+     */
+    back: WrappedAction<DeviceActionAndroidBackButton>;
+    /**
+     * Trigger the system home operation on Android devices
+     */
+    home: WrappedAction<DeviceActionAndroidHomeButton>;
+    /**
+     * Trigger the system recent apps operation on Android devices
+     */
+    recentApps: WrappedAction<DeviceActionAndroidRecentAppsButton>;
+    /**
+     * User-provided app name to package name mapping
+     */
+    private appNameMapping;
+    constructor(device: AndroidDevice, opts?: AndroidAgentOpt);
+    /**
+     * Launch an Android app or URL
+     * @param uri - App package name, URL, or app name to launch
+     */
+    launch(uri: string): Promise<void>;
+    /**
+     * Execute ADB shell command on Android device
+     * @param command - ADB shell command to execute
+     */
+    runAdbShell(command: string): Promise<string>;
+    private createActionWrapper;
+}
+
+export declare type AndroidAgentOpt = AgentOpt & {
+    /**
+     * Custom mapping of app names to package names
+     * User-provided mappings will take precedence over default mappings
+     */
+    appNameMapping?: Record<string, string>;
+};
+
+export declare class AndroidDevice implements AbstractInterface {
+    private deviceId;
+    private yadbPushed;
+    private devicePixelRatio;
+    private devicePixelRatioInitialized;
+    private scalingRatio;
+    private adb;
+    private connectingAdb;
+    private destroyed;
+    private description;
+    private customActions?;
+    private cachedScreenSize;
+    private cachedOrientation;
+    private cachedPhysicalDisplayId;
+    private scrcpyAdapter;
+    private appNameMapping;
+    interfaceType: InterfaceType;
+    uri: string | undefined;
+    options?: AndroidDeviceOpt;
+    actionSpace(): DeviceAction<any>[];
+    constructor(deviceId: string, options?: AndroidDeviceOpt);
+    describe(): string;
+    connect(): Promise<ADB>;
+    getAdb(): Promise<ADB>;
+    private createAdbProxy;
+    /**
+     * Get or create the scrcpy adapter (lazy initialization)
+     */
+    private getScrcpyAdapter;
+    /**
+     * Get device physical info needed by scrcpy adapter
+     */
+    private getDevicePhysicalInfo;
+    /**
+     * Set the app name to package name mapping
+     */
+    setAppNameMapping(mapping: Record<string, string>): void;
+    /**
+     * Resolve app name to package name using the mapping
+     * Comparison is case-insensitive and ignores spaces, dashes, and underscores.
+     * Keys in appNameMapping are pre-normalized, so we only need to normalize the input.
+     * @param appName The app name to resolve
+     */
+    private resolvePackageName;
+    launch(uri: string): Promise<AndroidDevice>;
+    execYadb(keyboardContent: string): Promise<void>;
+    getElementsInfo(): Promise<ElementInfo[]>;
+    getElementsNodeTree(): Promise<any>;
+    getScreenSize(): Promise<{
+        override: string;
+        physical: string;
+        orientation: number;
+        isCurrentOrientation?: boolean;
+    }>;
+    private initializeDevicePixelRatio;
+    getDisplayDensity(): Promise<number>;
+    getDisplayOrientation(): Promise<number>;
+    size(): Promise<Size>;
+    /**
+     * Generate cache feature for an element at given center point
+     * Cache is based on test case ID + device ID, so screen size is guaranteed to be consistent
+     * for same test case on same device. We store exact coordinates for maximum accuracy.
+     */
+    cacheFeatureForPoint(center: [number, number], options?: {
+        targetDescription?: string;
+        modelConfig?: IModelConfig;
+    }): Promise<ElementCacheFeature>;
+    /**
+     * Find element rect from cache feature
+     * Since cache is keyed by test case ID + device ID, screen size will be identical
+     * We return the exact cached coordinates for maximum accuracy.
+     */
+    rectMatchesCacheFeature(feature: ElementCacheFeature): Promise<Rect>;
+    private adjustCoordinates;
+    /**
+     * Calculate the end point for scroll operations based on start point, scroll delta, and screen boundaries.
+     * This method ensures that scroll operations stay within screen bounds and maintain a minimum scroll distance
+     * for effective scrolling gestures on Android devices.
+     *
+     * @param start - The starting point of the scroll gesture
+     * @param deltaX - The horizontal scroll distance (positive = scroll right, negative = scroll left)
+     * @param deltaY - The vertical scroll distance (positive = scroll down, negative = scroll up)
+     * @param maxWidth - The maximum width boundary (screen width)
+     * @param maxHeight - The maximum height boundary (screen height)
+     * @returns The calculated end point for the scroll gesture
+     */
+    private calculateScrollEndPoint;
+    screenshotBase64(): Promise<string>;
+    clearInput(element?: ElementInfo): Promise<void>;
+    forceScreenshot(path: string): Promise<void>;
+    url(): Promise<string>;
+    scrollUntilTop(startPoint?: Point): Promise<void>;
+    scrollUntilBottom(startPoint?: Point): Promise<void>;
+    scrollUntilLeft(startPoint?: Point): Promise<void>;
+    scrollUntilRight(startPoint?: Point): Promise<void>;
+    scrollUp(distance?: number, startPoint?: Point): Promise<void>;
+    scrollDown(distance?: number, startPoint?: Point): Promise<void>;
+    scrollLeft(distance?: number, startPoint?: Point): Promise<void>;
+    scrollRight(distance?: number, startPoint?: Point): Promise<void>;
+    ensureYadb(): Promise<void>;
+    /**
+     * Check if text contains characters that may cause issues with ADB inputText
+     * This includes:
+     * - Non-ASCII characters (Unicode characters like ö, é, ñ, Chinese, Japanese, etc.)
+     * - Format specifiers that may be interpreted by shell (%, $)
+     * - Special shell characters that need escaping
+     */
+    private shouldUseYadbForText;
+    keyboardType(text: string, options?: AndroidDeviceInputOpt): Promise<void>;
+    private normalizeKeyName;
+    keyboardPress(key: string): Promise<void>;
+    mouseClick(x: number, y: number): Promise<void>;
+    mouseDoubleClick(x: number, y: number): Promise<void>;
+    mouseMove(): Promise<void>;
+    mouseDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }, duration?: number): Promise<void>;
+    scroll(deltaX: number, deltaY: number, duration?: number): Promise<void>;
+    destroy(): Promise<void>;
+    /**
+     * Get the current time from the Android device.
+     * Returns the device's current timestamp in milliseconds.
+     * This is useful when the system time and device time are not synchronized.
+     */
+    getTimestamp(): Promise<number>;
+    back(): Promise<void>;
+    home(): Promise<void>;
+    recentApps(): Promise<void>;
+    longPress(x: number, y: number, duration?: number): Promise<void>;
+    pullDown(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
+    pullDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }, duration: number): Promise<void>;
+    pullUp(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
+    private getDisplayArg;
+    getPhysicalDisplayId(): Promise<string | null>;
+    hideKeyboard(options?: AndroidDeviceInputOpt, timeoutMs?: number): Promise<boolean>;
+}
+
+declare type DeviceActionAndroidBackButton = DeviceAction<undefined, void>;
+
+declare type DeviceActionAndroidHomeButton = DeviceAction<undefined, void>;
+
+declare type DeviceActionAndroidRecentAppsButton = DeviceAction<undefined, void>;
+
+export declare function getConnectedDevices(): Promise<Device[]>;
+
+export { overrideAIConfig }
+
+/**
+ * Helper type to convert DeviceAction to wrapped method signature
+ */
+declare type WrappedAction<T extends DeviceAction> = (...args: ActionArgs<T>) => Promise<ActionReturn<T>>;
+
+export { }

package/dist/types/mcp-server.d.ts

@@ -0,0 +1,263 @@
+import { AbstractInterface } from '@midscene/core/device';
+import type { ActionParam } from '@midscene/core';
+import type { ActionReturn } from '@midscene/core';
+import { ADB } from 'appium-adb';
+import { Agent } from '@midscene/core/agent';
+import { AgentOpt } from '@midscene/core/agent';
+import { AndroidDeviceInputOpt } from '@midscene/core/device';
+import { AndroidDeviceOpt } from '@midscene/core/device';
+import { BaseMCPServer } from '@midscene/shared/mcp';
+import { BaseMidsceneTools } from '@midscene/shared/mcp';
+import { DeviceAction } from '@midscene/core';
+import { ElementCacheFeature } from '@midscene/core';
+import type { ElementInfo } from '@midscene/shared/extractor';
+import { IModelConfig } from '@midscene/shared/env';
+import { InterfaceType } from '@midscene/core';
+import { LaunchMCPServerOptions } from '@midscene/shared/mcp';
+import { LaunchMCPServerResult } from '@midscene/shared/mcp';
+import { Point } from '@midscene/core';
+import { Rect } from '@midscene/core';
+import { Size } from '@midscene/core';
+import { Tool } from '@midscene/shared/mcp';
+import { ToolDefinition } from '@midscene/shared/mcp';
+
+declare type ActionArgs<T extends DeviceAction> = [ActionParam<T>] extends [undefined] ? [] : [ActionParam<T>];
+
+declare class AndroidAgent extends Agent<AndroidDevice> {
+    /**
+     * Trigger the system back operation on Android devices
+     */
+    back: WrappedAction<DeviceActionAndroidBackButton>;
+    /**
+     * Trigger the system home operation on Android devices
+     */
+    home: WrappedAction<DeviceActionAndroidHomeButton>;
+    /**
+     * Trigger the system recent apps operation on Android devices
+     */
+    recentApps: WrappedAction<DeviceActionAndroidRecentAppsButton>;
+    /**
+     * User-provided app name to package name mapping
+     */
+    private appNameMapping;
+    constructor(device: AndroidDevice, opts?: AndroidAgentOpt);
+    /**
+     * Launch an Android app or URL
+     * @param uri - App package name, URL, or app name to launch
+     */
+    launch(uri: string): Promise<void>;
+    /**
+     * Execute ADB shell command on Android device
+     * @param command - ADB shell command to execute
+     */
+    runAdbShell(command: string): Promise<string>;
+    private createActionWrapper;
+}
+
+declare type AndroidAgentOpt = AgentOpt & {
+    /**
+     * Custom mapping of app names to package names
+     * User-provided mappings will take precedence over default mappings
+     */
+    appNameMapping?: Record<string, string>;
+};
+
+declare class AndroidDevice implements AbstractInterface {
+    private deviceId;
+    private yadbPushed;
+    private devicePixelRatio;
+    private devicePixelRatioInitialized;
+    private scalingRatio;
+    private adb;
+    private connectingAdb;
+    private destroyed;
+    private description;
+    private customActions?;
+    private cachedScreenSize;
+    private cachedOrientation;
+    private cachedPhysicalDisplayId;
+    private scrcpyAdapter;
+    private appNameMapping;
+    interfaceType: InterfaceType;
+    uri: string | undefined;
+    options?: AndroidDeviceOpt;
+    actionSpace(): DeviceAction<any>[];
+    constructor(deviceId: string, options?: AndroidDeviceOpt);
+    describe(): string;
+    connect(): Promise<ADB>;
+    getAdb(): Promise<ADB>;
+    private createAdbProxy;
+    /**
+     * Get or create the scrcpy adapter (lazy initialization)
+     */
+    private getScrcpyAdapter;
+    /**
+     * Get device physical info needed by scrcpy adapter
+     */
+    private getDevicePhysicalInfo;
+    /**
+     * Set the app name to package name mapping
+     */
+    setAppNameMapping(mapping: Record<string, string>): void;
+    /**
+     * Resolve app name to package name using the mapping
+     * Comparison is case-insensitive and ignores spaces, dashes, and underscores.
+     * Keys in appNameMapping are pre-normalized, so we only need to normalize the input.
+     * @param appName The app name to resolve
+     */
+    private resolvePackageName;
+    launch(uri: string): Promise<AndroidDevice>;
+    execYadb(keyboardContent: string): Promise<void>;
+    getElementsInfo(): Promise<ElementInfo[]>;
+    getElementsNodeTree(): Promise<any>;
+    getScreenSize(): Promise<{
+        override: string;
+        physical: string;
+        orientation: number;
+        isCurrentOrientation?: boolean;
+    }>;
+    private initializeDevicePixelRatio;
+    getDisplayDensity(): Promise<number>;
+    getDisplayOrientation(): Promise<number>;
+    size(): Promise<Size>;
+    /**
+     * Generate cache feature for an element at given center point
+     * Cache is based on test case ID + device ID, so screen size is guaranteed to be consistent
+     * for same test case on same device. We store exact coordinates for maximum accuracy.
+     */
+    cacheFeatureForPoint(center: [number, number], options?: {
+        targetDescription?: string;
+        modelConfig?: IModelConfig;
+    }): Promise<ElementCacheFeature>;
+    /**
+     * Find element rect from cache feature
+     * Since cache is keyed by test case ID + device ID, screen size will be identical
+     * We return the exact cached coordinates for maximum accuracy.
+     */
+    rectMatchesCacheFeature(feature: ElementCacheFeature): Promise<Rect>;
+    private adjustCoordinates;
+    /**
+     * Calculate the end point for scroll operations based on start point, scroll delta, and screen boundaries.
+     * This method ensures that scroll operations stay within screen bounds and maintain a minimum scroll distance
+     * for effective scrolling gestures on Android devices.
+     *
+     * @param start - The starting point of the scroll gesture
+     * @param deltaX - The horizontal scroll distance (positive = scroll right, negative = scroll left)
+     * @param deltaY - The vertical scroll distance (positive = scroll down, negative = scroll up)
+     * @param maxWidth - The maximum width boundary (screen width)
+     * @param maxHeight - The maximum height boundary (screen height)
+     * @returns The calculated end point for the scroll gesture
+     */
+    private calculateScrollEndPoint;
+    screenshotBase64(): Promise<string>;
+    clearInput(element?: ElementInfo): Promise<void>;
+    forceScreenshot(path: string): Promise<void>;
+    url(): Promise<string>;
+    scrollUntilTop(startPoint?: Point): Promise<void>;
+    scrollUntilBottom(startPoint?: Point): Promise<void>;
+    scrollUntilLeft(startPoint?: Point): Promise<void>;
+    scrollUntilRight(startPoint?: Point): Promise<void>;
+    scrollUp(distance?: number, startPoint?: Point): Promise<void>;
+    scrollDown(distance?: number, startPoint?: Point): Promise<void>;
+    scrollLeft(distance?: number, startPoint?: Point): Promise<void>;
+    scrollRight(distance?: number, startPoint?: Point): Promise<void>;
+    ensureYadb(): Promise<void>;
+    /**
+     * Check if text contains characters that may cause issues with ADB inputText
+     * This includes:
+     * - Non-ASCII characters (Unicode characters like ö, é, ñ, Chinese, Japanese, etc.)
+     * - Format specifiers that may be interpreted by shell (%, $)
+     * - Special shell characters that need escaping
+     */
+    private shouldUseYadbForText;
+    keyboardType(text: string, options?: AndroidDeviceInputOpt): Promise<void>;
+    private normalizeKeyName;
+    keyboardPress(key: string): Promise<void>;
+    mouseClick(x: number, y: number): Promise<void>;
+    mouseDoubleClick(x: number, y: number): Promise<void>;
+    mouseMove(): Promise<void>;
+    mouseDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }, duration?: number): Promise<void>;
+    scroll(deltaX: number, deltaY: number, duration?: number): Promise<void>;
+    destroy(): Promise<void>;
+    /**
+     * Get the current time from the Android device.
+     * Returns the device's current timestamp in milliseconds.
+     * This is useful when the system time and device time are not synchronized.
+     */
+    getTimestamp(): Promise<number>;
+    back(): Promise<void>;
+    home(): Promise<void>;
+    recentApps(): Promise<void>;
+    longPress(x: number, y: number, duration?: number): Promise<void>;
+    pullDown(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
+    pullDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }, duration: number): Promise<void>;
+    pullUp(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
+    private getDisplayArg;
+    getPhysicalDisplayId(): Promise<string | null>;
+    hideKeyboard(options?: AndroidDeviceInputOpt, timeoutMs?: number): Promise<boolean>;
+}
+
+/**
+ * Android MCP Server
+ * Provides MCP tools for Android automation through ADB
+ */
+export declare class AndroidMCPServer extends BaseMCPServer {
+    constructor(toolsManager?: AndroidMidsceneTools);
+    protected createToolsManager(): AndroidMidsceneTools;
+}
+
+/**
+ * Android-specific tools manager
+ * Extends BaseMidsceneTools to provide Android ADB device connection tools
+ */
+declare class AndroidMidsceneTools extends BaseMidsceneTools<AndroidAgent> {
+    protected createTemporaryDevice(): AndroidDevice;
+    protected ensureAgent(deviceId?: string): Promise<AndroidAgent>;
+    /**
+     * Provide Android-specific platform tools
+     */
+    protected preparePlatformTools(): ToolDefinition[];
+}
+
+declare type DeviceActionAndroidBackButton = DeviceAction<undefined, void>;
+
+declare type DeviceActionAndroidHomeButton = DeviceAction<undefined, void>;
+
+declare type DeviceActionAndroidRecentAppsButton = DeviceAction<undefined, void>;
+
+/**
+ * Create MCP kit for a specific Android Agent
+ */
+export declare function mcpKitForAgent(agent: Agent | AndroidAgent): Promise<{
+    description: string;
+    tools: Tool[];
+}>;
+
+/**
+ * Create an MCP server launcher for a specific Android Agent
+ */
+export declare function mcpServerForAgent(agent: Agent | AndroidAgent): {
+    launch(options?: {
+        verbose?: boolean;
+    }): Promise<LaunchMCPServerResult>;
+    launchHttp(options: LaunchMCPServerOptions): Promise<LaunchMCPServerResult>;
+};
+
+/**
+ * Helper type to convert DeviceAction to wrapped method signature
+ */
+declare type WrappedAction<T extends DeviceAction> = (...args: ActionArgs<T>) => Promise<ActionReturn<T>>;
+
+export { }

package/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@aiscene/android",
+  "version": "1.3.4",
+  "description": "Android automation library for Midscene",
+  "keywords": [
+    "Android UI automation",
+    "Android AI testing",
+    "Android automation library",
+    "Android automation tool",
+    "Android use"
+  ],
+  "main": "./dist/lib/index.js",
+  "module": "./dist/es/index.mjs",
+  "types": "./dist/types/index.d.ts",
+  "files": ["bin", "dist", "README.md"],
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/es/index.mjs",
+      "require": "./dist/lib/index.js"
+    },
+    "./mcp-server": {
+      "types": "./dist/types/mcp-server.d.ts",
+      "import": "./dist/es/mcp-server.mjs",
+      "require": "./dist/lib/mcp-server.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "dev": "npm run build:watch",
+    "prebuild": "node scripts/download-scrcpy-server.mjs && node scripts/download-yadb.mjs",
+    "build": "rslib build",
+    "build:watch": "rslib build --watch --no-clean",
+    "prepack": "node scripts/download-scrcpy-server.mjs && node scripts/download-yadb.mjs",
+    "playground": "DEBUG=midscene:* tsx demo/playground.ts",
+    "test": "vitest --run",
+    "test:u": "vitest --run -u",
+    "test:ai": "AI_TEST_TYPE=android npm run test",
+    "test:ai:cache": "MIDSCENE_CACHE=true AI_TEST_TYPE=android npm run test"
+  },
+  "dependencies": {
+    "@midscene/core": "^1.3.4",
+    "@midscene/shared": "^1.3.4",
+    "@yume-chan/adb": "^1.1.0",
+    "@yume-chan/adb-scrcpy": "^1.1.0",
+    "@yume-chan/adb-server-node-tcp": "^1.1.0",
+    "@yume-chan/scrcpy": "^1.1.0",
+    "@yume-chan/stream-extra": "^1.0.0",
+    "appium-adb": "12.12.1",
+    "sharp": "^0.34.3"
+  },
+  "optionalDependencies": {
+    "@ffmpeg-installer/ffmpeg": "^1.1.0"
+  },
+  "devDependencies": {
+    "@midscene/playground": "workspace:*",
+    "@rslib/core": "^0.18.3",
+    "@types/node": "^18.0.0",
+    "dotenv": "^16.4.5",
+    "gh-release-fetch": "^4.0.3",
+    "typescript": "^5.8.3",
+    "tsx": "^4.19.2",
+    "vitest": "3.0.5",
+    "zod": "3.24.3"
+  },
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  }
+}