@anusornneal/chat-relay 0.7.2 → 0.8.1

package/agent/desktop-manager.mjs

@@ -1,13 +1,15 @@
-import { spawn } from "node:child_process";
-import { fileURLToPath } from "node:url";
-import path from "node:path";
-import readline from "node:readline";
-import { randomUUID } from "node:crypto";
+import { BoundedLane } from "./capability-scheduler.mjs";
+import { PersistentDesktopRunner } from "./windows-desktop-runner.mjs";
+import { MacOSDesktopRunner } from "./macos-desktop-runner.mjs";
 
 const MAX_SCREENSHOT_BINARY_BYTES = 32 * 1024;
-const POWERSHELL_TIMEOUT_MS = 15_000;
 const MAX_TEXT_LENGTH = 8192;
-const helperPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "windows-desktop-worker.ps1");
+const MAX_CLIPBOARD_TEXT_LENGTH = 8192;
+const MIN_SCREENSHOT_WIDTH = 320;
+const MAX_SCREENSHOT_WIDTH = 1920;
+const MIN_SCREENSHOT_QUALITY = 20;
+const MAX_SCREENSHOT_QUALITY = 85;
+const MAX_WINDOW_LIST_LIMIT = 100;
 
 const KEY_CODES = new Map([
   ["enter", 0x0D],
@@ -54,6 +56,7 @@ function normalizeKeyboardInput(input = {}) {
   return {
     ok: true,
     payload: {
+      key,
       keyCode,
       ctrl: input.ctrl === true,
       alt: input.alt === true,
@@ -63,129 +66,132 @@ function normalizeKeyboardInput(input = {}) {
   };
 }
 
-class PersistentDesktopRunner {
-  constructor(options = {}) {
-    this.platform = options.platform ?? process.platform;
-    this.timeoutMs = Number(options.timeoutMs ?? POWERSHELL_TIMEOUT_MS);
-    this.process = null;
-    this.stdout = null;
-    this.pending = new Map();
-    this.stderr = "";
+function compactStepActions(actions) {
+  const compacted = [];
+  for (const action of actions) {
+    const previous = compacted.at(-1);
+    if (action.type === "wait" && action.ms === 0) continue;
+    if (action.type === "text" && previous?.type === "text" &&
+        previous.text.length + action.text.length <= MAX_TEXT_LENGTH) {
+      previous.text += action.text;
+      continue;
+    }
+    if (action.type === "wait" && previous?.type === "wait" && previous.ms + action.ms <= 5000) {
+      previous.ms += action.ms;
+      continue;
+    }
+    compacted.push({ ...action });
   }
+  return compacted.length > 0 ? compacted : actions.map((action) => ({ ...action }));
+}
 
-  ensureStarted() {
-    if (this.platform !== "win32") return false;
-    if (this.process && !this.process.killed) return true;
-
-    const child = spawn("powershell.exe", [
-      "-NoLogo",
-      "-NoProfile",
-      "-NonInteractive",
-      "-ExecutionPolicy",
-      "Bypass",
-      "-File",
-      helperPath,
-    ], {
-      windowsHide: true,
-      stdio: ["pipe", "pipe", "pipe"],
-    });
-
-    this.process = child;
-    this.stderr = "";
-    this.stdout = readline.createInterface({ input: child.stdout });
-
-    this.stdout.on("line", (line) => {
-      let message;
-      try {
-        message = JSON.parse(line);
-      } catch {
-        return;
-      }
-      const pending = this.pending.get(message?.id);
-      if (!pending) return;
-      clearTimeout(pending.timer);
-      this.pending.delete(message.id);
-      pending.resolve(message.result ?? { ok: false, error: "desktop_worker_error" });
-    });
-
-    child.stderr.on("data", (chunk) => {
-      this.stderr = (this.stderr + String(chunk)).slice(-8192);
-    });
-
-    child.on("exit", () => {
-      const error = { ok: false, error: "desktop_worker_exited" };
-      for (const pending of this.pending.values()) {
-        clearTimeout(pending.timer);
-        pending.resolve(error);
-      }
-      this.pending.clear();
-      this.stdout?.close();
-      this.stdout = null;
-      this.process = null;
-    });
-
-    child.on("error", () => {
-      // exit handler settles pending work.
-    });
+function adaptiveActionSettleMs(action, nextAction) {
+  if (!nextAction || action.type === "wait") return 0;
+  if (action.type === "click") {
+    if (action.clicks === 2) return 45;
+    return nextAction.type === "text" || nextAction.type === "key" ? 25 : 15;
+  }
+  if (action.type === "text") return 0;
+  if (action.type === "key") {
+    if ((action.alt && action.key === "tab") || action.win) return 120;
+    if (["enter", "tab", "escape"].includes(action.key)) return 25;
+    return 5;
+  }
+  return 0;
+}
 
-    return true;
+function adaptiveFinalSettleMs(actions, wantsCapture) {
+  if (!wantsCapture || actions.length === 0) return 0;
+  const action = actions.at(-1);
+  if (action.type === "wait") return 0;
+  if (action.type === "click") return action.clicks === 2 ? 60 : 35;
+  if (action.type === "text") return 10;
+  if (action.type === "key") {
+    if ((action.alt && action.key === "tab") || action.win) return 180;
+    if (["enter", "tab", "escape"].includes(action.key)) return 50;
+    return 15;
   }
+  return 0;
+}
 
-  async run(operation, args) {
-    if (!this.ensureStarted() || !this.process?.stdin?.writable) {
-      return { ok: false, error: operation === "screenshot" ? "capture_failed" : "input_failed" };
-    }
+export function createDesktopPlatformAdapter(options = {}) {
+  const platform = options.platform ?? process.platform;
+  const customRunner = typeof options.runner === "function" ? options.runner : null;
 
-    const id = randomUUID();
-    const request = JSON.stringify({ id, operation, args: args || {} }) + "\n";
-
-    return new Promise((resolve) => {
-      const timer = setTimeout(() => {
-        this.pending.delete(id);
-        resolve({ ok: false, error: operation === "screenshot" ? "capture_timeout" : "input_timeout" });
-      }, this.timeoutMs);
-
-      this.pending.set(id, { resolve, timer });
-      this.process.stdin.write(request, (error) => {
-        if (!error) return;
-        clearTimeout(timer);
-        this.pending.delete(id);
-        resolve({ ok: false, error: operation === "screenshot" ? "capture_failed" : "input_failed" });
-      });
-    });
+  if (platform === "win32") {
+    const persistentRunner = customRunner ? null : (options.persistentRunner ?? new PersistentDesktopRunner({ platform }));
+    return {
+      platform,
+      supported: true,
+      transport: "persistent-worker",
+      run: customRunner ?? ((operation, args) => persistentRunner.run(operation, args)),
+      close: () => persistentRunner?.close?.(),
+    };
   }
 
-  close() {
-    try { this.stdout?.close(); } catch {}
-    try { this.process?.kill(); } catch {}
-    this.stdout = null;
-    this.process = null;
+  if (platform === "darwin") {
+    const macosRunner = customRunner ? null : (options.macosRunner ?? new MacOSDesktopRunner({ platform }));
+    return {
+      platform,
+      supported: true,
+      transport: "native-macos",
+      run: customRunner ?? ((operation, args) => macosRunner.run(operation, args)),
+      close: () => macosRunner?.close?.(),
+    };
   }
+
+  return {
+    platform,
+    supported: false,
+    transport: "unsupported",
+    run: customRunner ?? (async () => ({ ok: false, error: "unsupported_platform" })),
+    close: () => {},
+  };
 }
 
 export class DesktopManager {
   constructor(options = {}) {
     this.enabled = options.enabled ?? process.env.DESKTOP_ENABLED === "1";
-    this.platform = options.platform ?? process.platform;
-    this.persistentRunner = options.persistentRunner ?? new PersistentDesktopRunner({ platform: this.platform });
-    this.runner = options.runner ?? ((operation, args) => this.persistentRunner.run(operation, args));
+    this.adapter = options.adapter ?? createDesktopPlatformAdapter({
+      platform: options.platform,
+      persistentRunner: options.persistentRunner,
+      macosRunner: options.macosRunner,
+      runner: options.runner,
+    });
+    this.platform = this.adapter.platform;
+    this.runner = (operation, args) => this.adapter.run(operation, args);
+    this.controlLane = options.controlLane ?? new BoundedLane({
+      name: "desktop_control",
+      concurrency: 1,
+      maxQueued: options.controlMaxQueued ?? process.env.DESKTOP_CONTROL_MAX_QUEUED,
+      queueTimeoutMs: options.controlQueueTimeoutMs ?? process.env.AGENT_QUEUE_TIMEOUT_MS,
+    });
   }
 
   getConfig() {
     return {
       enabled: this.enabled,
-      supported: this.platform === "win32",
-      platform: this.platform,
-      transport: this.platform === "win32" ? "persistent-worker" : "unsupported",
+      supported: this.adapter.supported,
+      platform: this.adapter.platform,
+      transport: this.adapter.transport,
+      controlQueue: this.controlLane.snapshot(),
     };
   }
 
   gate() {
     if (!this.enabled) return { ok: false, error: "desktop_disabled" };
-    if (this.platform !== "win32") return { ok: false, error: "unsupported_platform" };
+    if (!this.adapter.supported) return { ok: false, error: "unsupported_platform" };
     return null;
   }
 
+  async runControl(work) {
+    try {
+      return await this.controlLane.run(work);
+    } catch (error) {
+      return { ok: false, error: error instanceof Error ? error.message : "desktop_queue_error" };
+    }
+  }
+
   async screenshot(input = {}) {
     const gate = this.gate();
     if (gate) return gate;
@@ -195,7 +201,16 @@ export class DesktopManager {
       return { ok: false, error: "invalid_monitor" };
     }
 
-    const result = await this.runner("screenshot", { monitor });
+    const maxWidth = input.maxWidth === undefined ? 960 : Number(input.maxWidth);
+    if (!Number.isInteger(maxWidth) || maxWidth < MIN_SCREENSHOT_WIDTH || maxWidth > MAX_SCREENSHOT_WIDTH) {
+      return { ok: false, error: "invalid_max_width" };
+    }
+    const quality = input.quality === undefined ? 58 : Number(input.quality);
+    if (!Number.isInteger(quality) || quality < MIN_SCREENSHOT_QUALITY || quality > MAX_SCREENSHOT_QUALITY) {
+      return { ok: false, error: "invalid_quality" };
+    }
+
+    const result = await this.runner("screenshot", { monitor, maxWidth, quality });
     return this.normalizeScreenshotResult(result);
   }
 
@@ -224,10 +239,60 @@ export class DesktopManager {
       monitorIndex: Number(result.monitorIndex),
       isPrimary: result.isPrimary === true,
       deviceName: typeof result.deviceName === "string" ? result.deviceName : undefined,
+      virtualDesktopOriginX: Number(result.virtualDesktopOriginX),
+      virtualDesktopOriginY: Number(result.virtualDesktopOriginY),
+      virtualDesktopWidth: Number(result.virtualDesktopWidth),
+      virtualDesktopHeight: Number(result.virtualDesktopHeight),
+      monitorCount: Number(result.monitorCount),
+      jpegQuality: Number(result.jpegQuality),
       byteLength: binaryBytes,
     };
   }
 
+  async clipboardRead() {
+    const gate = this.gate();
+    if (gate) return gate;
+    const result = await this.runner("clipboard_read", {});
+    if (!result?.ok || typeof result.text !== "string") return result || { ok: false, error: "clipboard_failed" };
+    return { ok: true, text: result.text.slice(0, MAX_CLIPBOARD_TEXT_LENGTH), truncated: result.truncated === true };
+  }
+
+  async clipboardWrite(input = {}) {
+    const gate = this.gate();
+    if (gate) return gate;
+    if (typeof input.text !== "string") return { ok: false, error: "invalid_clipboard_text" };
+    if (input.text.length > MAX_CLIPBOARD_TEXT_LENGTH) return { ok: false, error: "text_too_large" };
+    return this.runControl(() => this.runner("clipboard_write", { text: input.text }));
+  }
+
+  async listWindows(input = {}) {
+    const gate = this.gate();
+    if (gate) return gate;
+    const limit = input.limit === undefined ? 50 : Number(input.limit);
+    if (!Number.isInteger(limit) || limit < 1 || limit > MAX_WINDOW_LIST_LIMIT) return { ok: false, error: "invalid_window_limit" };
+    const result = await this.runner("window_list", { limit });
+    if (!result?.ok || !Array.isArray(result.windows)) return result || { ok: false, error: "window_list_failed" };
+    const windows = result.windows.slice(0, limit).map((window) => ({
+      windowId: String(window.windowId ?? window.WindowId ?? ""),
+      processId: Number(window.processId ?? window.ProcessId),
+      title: String(window.title ?? window.Title ?? ""),
+      x: Number(window.x ?? window.X),
+      y: Number(window.y ?? window.Y),
+      width: Number(window.width ?? window.Width),
+      height: Number(window.height ?? window.Height),
+      isForeground: (window.isForeground ?? window.IsForeground) === true,
+    }));
+    return { ok: true, windows, count: Math.min(Number(result.count ?? result.Count) || windows.length, limit) };
+  }
+
+  async focusWindow(input = {}) {
+    const gate = this.gate();
+    if (gate) return gate;
+    const windowId = String(input.windowId ?? "").trim();
+    if (!/^[1-9][0-9]{0,19}$/.test(windowId)) return { ok: false, error: "invalid_window_id" };
+    return this.runControl(() => this.runner("window_focus", { windowId }));
+  }
+
   async mouseClick(input = {}) {
     const gate = this.gate();
     if (gate) return gate;
@@ -241,7 +306,7 @@ export class DesktopManager {
     if (!["left", "right", "middle"].includes(button)) return { ok: false, error: "invalid_button" };
     if (clicks !== 1 && clicks !== 2) return { ok: false, error: "invalid_click_count" };
 
-    return this.runner("mouse_click", { x, y, button, clicks });
+    return this.runControl(() => this.runner("mouse_click", { x, y, button, clicks }));
   }
 
   async keyboardInput(input = {}) {
@@ -250,7 +315,7 @@ export class DesktopManager {
 
     const normalized = normalizeKeyboardInput(input);
     if (!normalized.ok) return normalized;
-    return this.runner("keyboard_input", normalized.payload);
+    return this.runControl(() => this.runner("keyboard_input", normalized.payload));
   }
 
   async step(input = {}) {
@@ -292,28 +357,68 @@ export class DesktopManager {
       return { ok: false, error: "invalid_step_action" };
     }
 
+    const compactedActions = compactStepActions(normalizedActions);
+    const optimizedActions = compactedActions.map((action, index) => {
+      const settleAfterMs = adaptiveActionSettleMs(action, compactedActions[index + 1]);
+      return settleAfterMs > 0 ? { ...action, settleAfterMs } : action;
+    });
+
     const monitor = input.monitor ?? "primary";
     if (!(monitor === "primary" || monitor === "secondary" || (Number.isInteger(monitor) && monitor >= 0 && monitor <= 15))) {
       return { ok: false, error: "invalid_monitor" };
     }
 
-    const settleMs = input.settleMs === undefined ? 120 : Number(input.settleMs);
+    const wantsCapture = input.captureAfter === true;
+    const settleMs = input.settleMs === undefined
+      ? adaptiveFinalSettleMs(compactedActions, wantsCapture)
+      : Number(input.settleMs);
     if (!Number.isInteger(settleMs) || settleMs < 0 || settleMs > 5000) return { ok: false, error: "invalid_settle_ms" };
 
-    const result = await this.runner("step", {
-      actions: normalizedActions,
-      captureAfter: input.captureAfter !== false,
-      settleMs,
-      monitor,
+    const maxWidth = input.maxWidth === undefined ? 960 : Number(input.maxWidth);
+    const quality = input.quality === undefined ? 58 : Number(input.quality);
+    if (wantsCapture && (!Number.isInteger(maxWidth) || maxWidth < MIN_SCREENSHOT_WIDTH || maxWidth > MAX_SCREENSHOT_WIDTH)) {
+      return { ok: false, error: "invalid_max_width" };
+    }
+    if (wantsCapture && (!Number.isInteger(quality) || quality < MIN_SCREENSHOT_QUALITY || quality > MAX_SCREENSHOT_QUALITY)) {
+      return { ok: false, error: "invalid_quality" };
+    }
+
+    const managerStartedAt = Date.now();
+    const result = await this.runControl(async (queueMeta = {}) => {
+      const handlerStartedAt = Date.now();
+      const workerResult = await this.runner("step", {
+        actions: optimizedActions,
+        captureAfter: wantsCapture,
+        settleMs,
+        monitor,
+        ...(wantsCapture ? { maxWidth, quality } : {}),
+      });
+      if (!workerResult || typeof workerResult !== "object") return workerResult;
+      return {
+        ...workerResult,
+        timing: {
+          ...(workerResult.timing && typeof workerResult.timing === "object" ? workerResult.timing : {}),
+          queueWaitMs: Math.max(0, Number(queueMeta.queueWaitMs) || 0),
+          managerMs: Math.max(0, Date.now() - handlerStartedAt),
+          requestedActions: normalizedActions.length,
+          executedActions: optimizedActions.length,
+        },
+      };
     });
+    if (result && typeof result === "object") {
+      result.timing = {
+        ...(result.timing && typeof result.timing === "object" ? result.timing : {}),
+        totalMs: Math.max(0, Date.now() - managerStartedAt),
+      };
+    }
 
-    if (input.captureAfter === false) return result;
+    if (!wantsCapture) return result;
     return this.normalizeScreenshotResult(result);
   }
 
   close() {
-    this.persistentRunner?.close?.();
+    this.adapter.close();
   }
 }
 
-export { KEY_CODES, MAX_SCREENSHOT_BINARY_BYTES, PersistentDesktopRunner };
+export { KEY_CODES, MAX_SCREENSHOT_BINARY_BYTES, PersistentDesktopRunner, MacOSDesktopRunner };