@aiscene/android 1.7.15 → 1.8.0

package/dist/types/cli.d.ts

@@ -0,0 +1 @@
+export { }

package/dist/types/mcp-server.d.ts

@@ -0,0 +1,282 @@
+import { AbstractInterface } from '@midscene/core/device';
+import type { ActionParam } from '@midscene/core';
+import type { ActionReturn } from '@midscene/core';
+import { ADB } from 'appium-adb';
+import { Agent } from '@midscene/core/agent';
+import { AgentOpt } from '@midscene/core/agent';
+import { AndroidDeviceInputOpt } from '@midscene/core/device';
+import { AndroidDeviceOpt } from '@midscene/core/device';
+import { BaseMCPServer } from '@midscene/shared/mcp';
+import { BaseMidsceneTools } from '@midscene/shared/mcp';
+import { DeviceAction } from '@midscene/core';
+import type { ElementInfo } from '@midscene/shared/extractor';
+import { InterfaceType } from '@midscene/core';
+import { LaunchMCPServerOptions } from '@midscene/shared/mcp';
+import { LaunchMCPServerResult } from '@midscene/shared/mcp';
+import { Point } from '@midscene/core';
+import { Size } from '@midscene/core';
+import { Tool } from '@midscene/shared/mcp';
+import { ToolDefinition } from '@midscene/shared/mcp';
+
+declare type ActionArgs<T extends DeviceAction> = [ActionParam<T>] extends [undefined] ? [] : [ActionParam<T>];
+
+declare class AndroidAgent extends Agent<AndroidDevice> {
+    /**
+     * Trigger the system back operation on Android devices
+     */
+    back: WrappedAction<DeviceActionAndroidBackButton>;
+    /**
+     * Trigger the system home operation on Android devices
+     */
+    home: WrappedAction<DeviceActionAndroidHomeButton>;
+    /**
+     * Trigger the system recent apps operation on Android devices
+     */
+    recentApps: WrappedAction<DeviceActionAndroidRecentAppsButton>;
+    /**
+     * User-provided app name to package name mapping
+     */
+    private appNameMapping;
+    constructor(device: AndroidDevice, opts?: AndroidAgentOpt);
+    /**
+     * Launch an Android app or URL
+     * @param uri - App package name, URL, or app name to launch
+     */
+    launch(uri: string): Promise<void>;
+    /**
+     * Execute ADB shell command on Android device
+     * @param command - ADB shell command to execute
+     */
+    runAdbShell(command: string): Promise<string>;
+    private createActionWrapper;
+}
+
+declare type AndroidAgentOpt = AgentOpt & {
+    /**
+     * Custom mapping of app names to package names
+     * User-provided mappings will take precedence over default mappings
+     */
+    appNameMapping?: Record<string, string>;
+};
+
+declare class AndroidDevice implements AbstractInterface {
+    private deviceId;
+    private yadbPushed;
+    private devicePixelRatio;
+    private devicePixelRatioInitialized;
+    private adb;
+    private connectingAdb;
+    private destroyed;
+    private description;
+    private customActions?;
+    private cachedScreenSize;
+    private cachedOrientation;
+    private cachedPhysicalDisplayId;
+    private scrcpyAdapter;
+    private appNameMapping;
+    private scalingRatio;
+    private takeScreenshotFailCount;
+    private static readonly TAKE_SCREENSHOT_FAIL_THRESHOLD;
+    interfaceType: InterfaceType;
+    uri: string | undefined;
+    options?: AndroidDeviceOpt;
+    actionSpace(): DeviceAction<any>[];
+    constructor(deviceId: string, options?: AndroidDeviceOpt);
+    describe(): string;
+    connect(): Promise<ADB>;
+    getAdb(): Promise<ADB>;
+    private createAdbProxy;
+    /**
+     * Get or create the scrcpy adapter (lazy initialization)
+     */
+    private getScrcpyAdapter;
+    /**
+     * Get device physical info needed by scrcpy adapter
+     */
+    private getDevicePhysicalInfo;
+    /**
+     * Set the app name to package name mapping
+     */
+    setAppNameMapping(mapping: Record<string, string>): void;
+    /**
+     * Resolve app name to package name using the mapping
+     * Comparison is case-insensitive and ignores spaces, dashes, and underscores.
+     * Keys in appNameMapping are pre-normalized, so we only need to normalize the input.
+     * @param appName The app name to resolve
+     */
+    private resolvePackageName;
+    launch(uri: string): Promise<AndroidDevice>;
+    execYadb(keyboardContent: string): Promise<void>;
+    getElementsInfo(): Promise<ElementInfo[]>;
+    getElementsNodeTree(): Promise<any>;
+    getScreenSize(): Promise<{
+        override: string;
+        physical: string;
+        orientation: number;
+        isCurrentOrientation?: boolean;
+    }>;
+    private initializeDevicePixelRatio;
+    getDisplayDensity(): Promise<number>;
+    getDisplayOrientation(): Promise<number>;
+    /**
+     * Get physical screen dimensions adjusted for current orientation.
+     * Swaps width/height when the device is in landscape and the reported
+     * dimensions do not already reflect the current orientation.
+     */
+    private getOrientedPhysicalSize;
+    size(): Promise<Size>;
+    cacheFeatureForPoint(center: [number, number]): Promise<{
+        centerX: number;
+        centerY: number;
+        screenSize: {
+            width: number;
+            height: number;
+        };
+    }>;
+    rectMatchesCacheFeature(feature: {
+        centerX: number;
+        centerY: number;
+        screenSize: {
+            width: number;
+            height: number;
+        };
+    }): Promise<{
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+    }>;
+    /**
+     * Convert logical coordinates (from AI) back to physical coordinates (for ADB).
+     * The ratio is derived from size(), so overriding size() alone is sufficient.
+     */
+    private adjustCoordinates;
+    /**
+     * Calculate the end point for scroll operations based on start point, scroll delta, and screen boundaries.
+     * This method ensures that scroll operations stay within screen bounds and maintain a minimum scroll distance
+     * for effective scrolling gestures on Android devices.
+     *
+     * @param start - The starting point of the scroll gesture
+     * @param deltaX - The horizontal scroll distance (positive = scroll right, negative = scroll left)
+     * @param deltaY - The vertical scroll distance (positive = scroll down, negative = scroll up)
+     * @param maxWidth - The maximum width boundary (screen width)
+     * @param maxHeight - The maximum height boundary (screen height)
+     * @returns The calculated end point for the scroll gesture
+     */
+    private calculateScrollEndPoint;
+    screenshotBase64(): Promise<string>;
+    clearInput(element?: ElementInfo): Promise<void>;
+    forceScreenshot(path: string): Promise<void>;
+    url(): Promise<string>;
+    scrollUntilTop(startPoint?: Point): Promise<void>;
+    scrollUntilBottom(startPoint?: Point): Promise<void>;
+    scrollUntilLeft(startPoint?: Point): Promise<void>;
+    scrollUntilRight(startPoint?: Point): Promise<void>;
+    scrollUp(distance?: number, startPoint?: Point): Promise<void>;
+    scrollDown(distance?: number, startPoint?: Point): Promise<void>;
+    scrollLeft(distance?: number, startPoint?: Point): Promise<void>;
+    scrollRight(distance?: number, startPoint?: Point): Promise<void>;
+    ensureYadb(): Promise<void>;
+    /**
+     * Check if text contains characters that may cause issues with ADB inputText.
+     * appium-adb's inputText has known bugs with certain characters:
+     * - Backslash causes broken shell quoting
+     * - Backtick is not escaped at all
+     * - Text containing both " and ' throws an error
+     * - Dollar sign can cause variable expansion issues
+     *
+     * For these characters, we route through yadb which handles them correctly
+     * via escapeForShell + double-quoted shell context.
+     */
+    private shouldUseYadbForText;
+    keyboardType(text: string, options?: AndroidDeviceInputOpt): Promise<void>;
+    private normalizeKeyName;
+    keyboardPress(key: string): Promise<void>;
+    mouseClick(x: number, y: number): Promise<void>;
+    mouseDoubleClick(x: number, y: number): Promise<void>;
+    mouseMove(): Promise<void>;
+    mouseDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }, duration?: number): Promise<void>;
+    scroll(deltaX: number, deltaY: number, duration?: number): Promise<void>;
+    destroy(): Promise<void>;
+    /**
+     * Get the current time from the Android device.
+     * Returns the device's current timestamp in milliseconds.
+     * This is useful when the system time and device time are not synchronized.
+     */
+    getTimestamp(): Promise<number>;
+    back(): Promise<void>;
+    home(): Promise<void>;
+    recentApps(): Promise<void>;
+    longPress(x: number, y: number, duration?: number): Promise<void>;
+    pullDown(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
+    pullDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }, duration: number): Promise<void>;
+    pullUp(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
+    private getDisplayArg;
+    getPhysicalDisplayId(): Promise<string | null>;
+    hideKeyboard(options?: AndroidDeviceInputOpt, timeoutMs?: number): Promise<boolean>;
+}
+
+/**
+ * Android MCP Server
+ * Provides MCP tools for Android automation through ADB
+ */
+export declare class AndroidMCPServer extends BaseMCPServer {
+    constructor(toolsManager?: AndroidMidsceneTools);
+    protected createToolsManager(): AndroidMidsceneTools;
+}
+
+/**
+ * Android-specific tools manager
+ * Extends BaseMidsceneTools to provide Android ADB device connection tools
+ */
+declare class AndroidMidsceneTools extends BaseMidsceneTools<AndroidAgent> {
+    protected createTemporaryDevice(): AndroidDevice;
+    protected ensureAgent(deviceId?: string): Promise<AndroidAgent>;
+    /**
+     * Provide Android-specific platform tools
+     */
+    protected preparePlatformTools(): ToolDefinition[];
+}
+
+declare type DeviceActionAndroidBackButton = DeviceAction<undefined, void>;
+
+declare type DeviceActionAndroidHomeButton = DeviceAction<undefined, void>;
+
+declare type DeviceActionAndroidRecentAppsButton = DeviceAction<undefined, void>;
+
+/**
+ * Create MCP kit for a specific Android Agent
+ */
+export declare function mcpKitForAgent(agent: Agent | AndroidAgent): Promise<{
+    description: string;
+    tools: Tool[];
+}>;
+
+/**
+ * Create an MCP server launcher for a specific Android Agent
+ */
+export declare function mcpServerForAgent(agent: Agent | AndroidAgent): {
+    launch(options?: {
+        verbose?: boolean;
+    }): Promise<LaunchMCPServerResult>;
+    launchHttp(options: LaunchMCPServerOptions): Promise<LaunchMCPServerResult>;
+};
+
+/**
+ * Helper type to convert DeviceAction to wrapped method signature
+ */
+declare type WrappedAction<T extends DeviceAction> = (...args: ActionArgs<T>) => Promise<ActionReturn<T>>;
+
+export { }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiscene/android",
-  "version": "1.7.15",
+  "version": "1.8.0",
   "description": "Android automation library for Midscene",
   "keywords": [
     "Android UI automation",
@@ -15,11 +15,7 @@
   "bin": {
     "midscene-android": "./bin/midscene-android"
   },
-  "files": [
-    "bin",
-    "dist",
-    "README.md"
-  ],
+  "files": ["bin", "dist", "README.md"],
   "exports": {
     ".": {
       "types": "./dist/types/index.d.ts",
@@ -46,8 +42,8 @@
     "test:ai:cache": "MIDSCENE_CACHE=true AI_TEST_TYPE=android npm run test"
   },
   "dependencies": {
-    "@aiscene/core": "1.7.3",
-    "@midscene/shared": "1.6.0",
+    "@midscene/core": "workspace:*",
+    "@midscene/shared": "workspace:*",
     "@yume-chan/adb": "2.5.1",
     "@yume-chan/adb-scrcpy": "2.3.2",
     "@yume-chan/adb-server-node-tcp": "2.5.2",