@midscene/computer 1.9.8-beta-20260618014851.0 → 1.9.8

package/dist/types/index.d.ts

@@ -1,17 +1,17 @@
 import { AbstractInterface } from '@midscene/core/device';
 import { Agent } from '@midscene/core/agent';
-import { AgentBehaviorInitArgs } from '@midscene/shared/agent-tools/agent-behavior-init-args';
+import { AgentBehaviorInitArgs } from '@midscene/shared/mcp/agent-behavior-init-args';
 import { AgentOpt } from '@midscene/core/agent';
-import { BaseMidsceneTools } from '@midscene/shared/agent-tools/base-tools';
+import { BaseMidsceneTools } from '@midscene/shared/mcp/base-tools';
 import { ChildProcessWithoutNullStreams } from 'node:child_process';
 import { ComputerInputPrimitives } from '@midscene/core/device';
 import type { DeviceAction } from '@midscene/core';
-import { InitArgSpec } from '@midscene/shared/agent-tools/base-tools';
+import { InitArgSpec } from '@midscene/shared/mcp/base-tools';
 import type { InterfaceType } from '@midscene/core';
 import { overrideAIConfig } from '@midscene/shared/env';
 import type { Size } from '@midscene/core';
 import { SpawnOptionsWithoutStdio } from 'node:child_process';
-import type { ToolDefinition } from '@midscene/shared/agent-tools/types';
+import type { ToolDefinition } from '@midscene/shared/mcp/types';
 
 declare interface AccessibilityCheckResult {
     hasPermission: boolean;
@@ -153,7 +153,7 @@ export declare interface ComputerDeviceOpt {
 /**
  * Discriminated union describing the two ways `computer_*` tools can spawn an
  * agent. `mode` is filled in by `initArgSpec.adapt` based on whether `host` is
- * set, so callers (CLI/YAML) never have to provide it explicitly.
+ * set, so callers (CLI/MCP/YAML) never have to provide it explicitly.
  */
 declare type ComputerInitArgs = ComputerLocalInitArgs | ComputerRDPInitArgs;
 

package/dist/types/mcp-server.d.ts

@@ -0,0 +1,245 @@
+import { AbstractInterface } from '@midscene/core/device';
+import { Agent } from '@midscene/core/agent';
+import { AgentBehaviorInitArgs } from '@midscene/shared/mcp/agent-behavior-init-args';
+import { BaseMCPServer } from '@midscene/shared/mcp';
+import { BaseMidsceneTools } from '@midscene/shared/mcp/base-tools';
+import { ComputerInputPrimitives } from '@midscene/core/device';
+import type { DeviceAction } from '@midscene/core';
+import { InitArgSpec } from '@midscene/shared/mcp/base-tools';
+import type { InterfaceType } from '@midscene/core';
+import { LaunchMCPServerOptions } from '@midscene/shared/mcp';
+import { LaunchMCPServerResult } from '@midscene/shared/mcp';
+import type { Size } from '@midscene/core';
+import { Tool } from '@midscene/shared/mcp';
+import type { ToolDefinition } from '@midscene/shared/mcp/types';
+
+declare class ComputerAgent<InterfaceType extends AbstractInterface = ComputerInterface> extends Agent<InterfaceType> {
+}
+
+declare class ComputerDevice implements AbstractInterface {
+    interfaceType: InterfaceType;
+    private options?;
+    private displayId?;
+    private displayGeometry?;
+    private description?;
+    private destroyed;
+    private xvfbInstance?;
+    private xvfbCleanup?;
+    private readonly inputDriver;
+    /**
+     * On macOS, use AppleScript for keyboard operations by default
+     * to avoid focus issues with system overlays (e.g. Spotlight).
+     */
+    private useAppleScript;
+    /** Cached result of the elevation check; see isRunningAsAdmin(). */
+    private adminCheckCache?;
+    uri?: string;
+    readonly inputPrimitives: ComputerInputPrimitives;
+    constructor(options?: ComputerDeviceOpt);
+    describe(): string;
+    /**
+     * Get all available displays
+     */
+    static listDisplays(): Promise<DisplayInfo[]>;
+    connect(): Promise<void>;
+    private healthCheck;
+    /**
+     * Check if the current process is running with Administrator privileges.
+     * Uses "net session" which succeeds only when elevated.
+     *
+     * The result is cached because elevation cannot change during the process
+     * lifetime, and the underlying `execSync('net session')` is a blocking
+     * subprocess spawn that should not run on every connect / health check.
+     */
+    private isRunningAsAdmin;
+    screenshotBase64(): Promise<string>;
+    size(): Promise<Size>;
+    private toGlobalPoint;
+    /**
+     * Type text via clipboard (paste)
+     * This method:
+     * 1. Saves the old clipboard content
+     * 2. Writes new content to clipboard
+     * 3. Simulates paste shortcut (Ctrl+V / Cmd+V)
+     * 4. Restores old clipboard content
+     */
+    private typeViaClipboard;
+    /**
+     * Always use clipboard paste to input text, avoiding IME interference.
+     * Keystroke-based input (AppleScript/libnut) goes through the active input method,
+     * which can swallow characters or convert them when a non-English IME is active.
+     */
+    private smartTypeString;
+    private selectAllAndDelete;
+    private pressKeyboardShortcut;
+    private resolveUntargetedScrollPoint;
+    private moveMouseToScrollTarget;
+    private performScroll;
+    actionSpace(): DeviceAction<any>[];
+    destroy(): Promise<void>;
+    url(): Promise<string>;
+}
+
+declare interface ComputerDeviceOpt {
+    displayId?: string;
+    customActions?: DeviceAction<any>[];
+    /**
+     * Keyboard driver for sending key events (macOS only)
+     * - 'applescript': Use AppleScript via osascript (default on macOS, more reliable)
+     * - 'libnut': Use libnut's keyTap (faster but may not work with some TUI apps)
+     */
+    keyboardDriver?: 'applescript' | 'libnut';
+    /**
+     * Headless mode via Xvfb (Linux only).
+     * - true: start Xvfb virtual display
+     * - false/undefined: do not start Xvfb
+     * Can also be set via MIDSCENE_COMPUTER_HEADLESS_LINUX=true environment variable.
+     */
+    headless?: boolean;
+    /**
+     * Resolution for Xvfb virtual display (default '1920x1080x24')
+     */
+    xvfbResolution?: string;
+}
+
+/**
+ * Discriminated union describing the two ways `computer_*` tools can spawn an
+ * agent. `mode` is filled in by `initArgSpec.adapt` based on whether `host` is
+ * set, so callers (CLI/MCP/YAML) never have to provide it explicitly.
+ */
+declare type ComputerInitArgs = ComputerLocalInitArgs | ComputerRDPInitArgs;
+
+declare type ComputerInterface = ComputerDevice | RDPDevice;
+
+/** Init args for the local desktop agent (macOS/Windows/Linux). */
+declare type ComputerLocalInitArgs = {
+    mode: 'local';
+} & Pick<ComputerDeviceOpt, 'displayId' | 'headless'> & AgentBehaviorInitArgs;
+
+/**
+ * Computer MCP Server
+ * Provides MCP tools for computer desktop automation
+ */
+export declare class ComputerMCPServer extends BaseMCPServer {
+    constructor(toolsManager?: ComputerMidsceneTools);
+    protected createToolsManager(): ComputerMidsceneTools;
+}
+
+/**
+ * Computer-specific tools manager
+ * Extends BaseMidsceneTools to provide desktop automation tools
+ */
+declare class ComputerMidsceneTools extends BaseMidsceneTools<ComputerAgent, ComputerInitArgs> {
+    private lastInitArgsSignature?;
+    protected getCliReportSessionName(): string;
+    protected readonly initArgSpec: InitArgSpec<ComputerInitArgs>;
+    protected createTemporaryDevice(): ComputerDevice;
+    protected ensureAgent(opts?: ComputerInitArgs): Promise<ComputerAgent>;
+    /**
+     * Provide Computer-specific platform tools
+     */
+    protected preparePlatformTools(): ToolDefinition[];
+}
+
+/** Init args for the RDP remote-desktop agent. */
+declare type ComputerRDPInitArgs = {
+    mode: 'rdp';
+} & RDPConnectionConfig & AgentBehaviorInitArgs;
+
+declare interface DisplayInfo {
+    id: string;
+    name: string;
+    primary?: boolean;
+}
+
+/**
+ * Create MCP kit for a specific Computer Agent
+ */
+export declare function mcpKitForAgent(agent: Agent | ComputerAgent): Promise<{
+    description: string;
+    tools: Tool[];
+}>;
+
+/**
+ * Create an MCP server launcher for a specific Computer Agent
+ */
+export declare function mcpServerForAgent(agent: Agent | ComputerAgent): {
+    launch(options?: {
+        verbose?: boolean;
+    }): Promise<LaunchMCPServerResult>;
+    launchHttp(options: LaunchMCPServerOptions): Promise<LaunchMCPServerResult>;
+};
+
+declare interface RDPBackendClient {
+    connect(config: RDPConnectionConfig): Promise<RDPConnectionInfo>;
+    disconnect(): Promise<void>;
+    screenshotBase64(): Promise<string>;
+    size(): Promise<Size>;
+    mouseMove(x: number, y: number): Promise<void>;
+    mouseButton(button: RDPMouseButton, action: RDPMouseButtonAction): Promise<void>;
+    wheel(direction: RDPScrollDirection, amount: number, x?: number, y?: number): Promise<void>;
+    keyPress(keyName: string): Promise<void>;
+    typeText(text: string): Promise<void>;
+    clearInput?(): Promise<void>;
+}
+
+declare interface RDPConnectionConfig {
+    host: string;
+    port?: number;
+    username?: string;
+    password?: string;
+    domain?: string;
+    localAddress?: string;
+    adminSession?: boolean;
+    ignoreCertificate?: boolean;
+    securityProtocol?: RDPSecurityProtocol;
+    desktopWidth?: number;
+    desktopHeight?: number;
+}
+
+declare interface RDPConnectionInfo {
+    sessionId: string;
+    size: Size;
+    server: string;
+}
+
+declare class RDPDevice implements AbstractInterface {
+    interfaceType: InterfaceType;
+    private readonly options;
+    private readonly backend;
+    private connectionInfo?;
+    private destroyed;
+    private cursorPosition?;
+    uri?: string;
+    readonly inputPrimitives: ComputerInputPrimitives;
+    constructor(options: RDPDeviceOpt);
+    describe(): string;
+    connect(): Promise<void>;
+    screenshotBase64(): Promise<string>;
+    size(): Promise<Size>;
+    destroy(): Promise<void>;
+    actionSpace(): DeviceAction<any>[];
+    private assertConnected;
+    private throwIfDestroyed;
+    private moveToElement;
+    private clearInput;
+    private edgeScrollDirection;
+    private defaultScrollDistance;
+    private movePointer;
+    private performWheel;
+}
+
+declare interface RDPDeviceOpt extends RDPConnectionConfig {
+    backend?: RDPBackendClient;
+    customActions?: DeviceAction<any>[];
+}
+
+declare type RDPMouseButton = 'left' | 'right' | 'middle';
+
+declare type RDPMouseButtonAction = 'down' | 'up' | 'click' | 'doubleClick';
+
+declare type RDPScrollDirection = 'up' | 'down' | 'left' | 'right';
+
+declare type RDPSecurityProtocol = 'auto' | 'tls' | 'nla' | 'rdp';
+
+export { }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/computer",
-  "version": "1.9.8-beta-20260618014851.0",
+  "version": "1.9.8",
   "description": "Midscene.js Computer Desktop Automation",
   "license": "MIT",
   "repository": {
@@ -27,14 +27,19 @@
       "import": "./dist/es/index.mjs",
       "require": "./dist/lib/index.js"
     },
+    "./mcp-server": {
+      "types": "./dist/types/mcp-server.d.ts",
+      "import": "./dist/es/mcp-server.mjs",
+      "require": "./dist/lib/mcp-server.js"
+    },
     "./package.json": "./package.json"
   },
   "dependencies": {
     "@computer-use/libnut": "^4.2.0",
     "clipboardy": "^4.0.0",
     "screenshot-desktop": "^1.15.3",
-    "@midscene/core": "1.9.8-beta-20260618014851.0",
-    "@midscene/shared": "1.9.8-beta-20260618014851.0"
+    "@midscene/core": "1.9.8",
+    "@midscene/shared": "1.9.8"
   },
   "optionalDependencies": {
     "node-mac-permissions": "2.5.0"