@jian-ai/remote-control 0.8.1 → 0.8.2

package/bin/computer-macos.mjs

@@ -1,6 +1,6 @@
 /**
  * macOS computer-use: primary-display screenshots, coord mapping, Jian overlay helper.
- * English-only messages. Imported by jian-remote.mjs on darwin.
+ * English-only messages. Imported by computer.mjs on darwin.
  */
 import { spawn, spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
@@ -9,40 +9,38 @@ import { chmod, copyFile, mkdir, readFile, writeFile } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { COMPUTER_CAPABILITIES, imagePixelToPoint, scaleFromGeometry } from './computer-shared.mjs';
 
-export const COMPUTER_CAPABILITY = 'computer-macos-v1';
+export { imagePixelToPoint, scaleFromGeometry } from './computer-shared.mjs';
+
+export const COMPUTER_CAPABILITY = COMPUTER_CAPABILITIES.darwin;
 const IDLE_MS = 30_000;
 const HELPER_NAME = 'computer-helper';
+/** How long we cache lock / display-asleep probes between HTTPS polls. */
+const UI_STATE_CACHE_MS = 5_000;
+export const MAC_UI_STATES = Object.freeze(['unlocked', 'locked', 'display_asleep', 'unknown']);
 
-/** Map screenshot image pixels → display points (primary). Half-pixel centers the click. */
-export function imagePixelToPoint(x, y, geometry) {
-  const { originX, originY, pointWidth, pointHeight, widthPx, heightPx } = geometry;
-  if (!(widthPx > 0 && heightPx > 0 && pointWidth > 0 && pointHeight > 0)) {
-    throw new Error('Invalid screenshot geometry for coordinate mapping');
-  }
-  return {
-    pointX: originX + (x + 0.5) * (pointWidth / widthPx),
-    pointY: originY + (y + 0.5) * (pointHeight / heightPx),
-  };
-}
+const LOCKED_COMPUTER_MESSAGE =
+  'Mac is locked — unlock the screen to use Computer Use. Shell and file tools may still work while the worker is online.';
 
-export function scaleFromGeometry(geometry) {
-  return {
-    imageToPointX: geometry.pointWidth / geometry.widthPx,
-    imageToPointY: geometry.pointHeight / geometry.heightPx,
-  };
-}
+let cachedUiState = { at: 0, state: 'unknown' };
+let workerIdleCaffeinate = null;
+let sessionDisplayCaffeinate = null;
 
-/** Capture primary display and compress like the gateway’s former screenshotCommand. */
+/**
+ * Capture primary display and compress for the worker channel.
+ * Single-pass ladder starts at 1600/42 (skips a full-res 1920@50 encode that often
+ * exceeded the budget and forced a second sips pass — common waste on Retina).
+ */
 export function capturePrimaryScreenshotBase64() {
   const script = [
     'tmp=$(mktemp /tmp/jian-shot.XXXXXX.jpg);',
     'screencapture -x -t jpg -D 1 "$tmp" || { rm -f "$tmp"; exit 1; };',
-    'sips -Z 1920 -s format jpeg -s formatOptions 50 "$tmp" >/dev/null 2>&1 || true;',
+    'sips -Z 1600 -s format jpeg -s formatOptions 42 "$tmp" >/dev/null 2>&1 || true;',
     'size=$(wc -c < "$tmp" | tr -d " ");',
-    'if [ "$size" -gt 280000 ]; then sips -Z 1440 -s format jpeg -s formatOptions 40 "$tmp" >/dev/null 2>&1 || true; fi;',
+    'if [ "$size" -gt 280000 ]; then sips -Z 1440 -s format jpeg -s formatOptions 38 "$tmp" >/dev/null 2>&1 || true; fi;',
     'size=$(wc -c < "$tmp" | tr -d " ");',
-    'if [ "$size" -gt 280000 ]; then sips -Z 1280 -s format jpeg -s formatOptions 35 "$tmp" >/dev/null 2>&1 || true; fi;',
+    'if [ "$size" -gt 280000 ]; then sips -Z 1280 -s format jpeg -s formatOptions 34 "$tmp" >/dev/null 2>&1 || true; fi;',
     'w=$(sips -g pixelWidth "$tmp" 2>/dev/null | awk "/pixelWidth/{print \\$2}");',
     'h=$(sips -g pixelHeight "$tmp" 2>/dev/null | awk "/pixelHeight/{print \\$2}");',
     'echo "$w $h";',
@@ -137,7 +135,6 @@ async function ensureHelperBinary() {
       clearQuarantine(paths.binary);
       await writeFile(paths.stamp, `${hash}\n`, { mode: 0o644 });
     }
-    // Keep source nearby for debugging / swiftc fallback after a partial install.
     try {
       const swift = packageSwiftPath();
       if (existsSync(swift)) await copyFile(swift, paths.source);
@@ -227,8 +224,11 @@ export class MacComputerSession {
 
   async ensure() {
     if (this.child && !this.child.killed) {
-      await this.send({ cmd: 'ensureSession' });
-      this.active = true;
+      if (!this.active) {
+        await this.send({ cmd: 'ensureSession' });
+        this.active = true;
+      }
+      startSessionDisplayAssertion(this.child.pid || process.pid);
       this.bumpIdle();
       return;
     }
@@ -247,6 +247,7 @@ export class MacComputerSession {
       this.failAll('Computer-use helper exited unexpectedly.');
       this.child = null;
       this.active = false;
+      stopSessionDisplayAssertion();
     });
     const ping = await this.send({ cmd: 'ping' });
     if (!ping.ok) throw new Error(ping.error || 'Computer-use helper did not respond.');
@@ -263,6 +264,7 @@ export class MacComputerSession {
     };
     await this.send({ cmd: 'ensureSession' });
     this.active = true;
+    startSessionDisplayAssertion(this.child.pid || process.pid);
     this.bumpIdle();
   }
 
@@ -343,6 +345,7 @@ export class MacComputerSession {
   async end() {
     if (this.idleTimer) clearTimeout(this.idleTimer);
     this.idleTimer = null;
+    stopSessionDisplayAssertion();
     if (this.child) {
       try {
         await this.send({ cmd: 'endSession' });
@@ -371,7 +374,7 @@ export class MacComputerSession {
     }
 
     if (action === 'wait') {
-      const ms = Math.round((computer.duration ?? 0.5) * 1000);
+      const ms = Math.round((computer.duration ?? 0.35) * 1000);
       await new Promise((r) => setTimeout(r, ms));
       if (this.active) this.bumpIdle();
       return {
@@ -383,6 +386,17 @@ export class MacComputerSession {
       };
     }
 
+    const uiBlocked = assertComputerUiAllowed();
+    if (uiBlocked) {
+      return {
+        exitCode: 1,
+        stdout: '',
+        stderr: uiBlocked,
+        timedOut: false,
+        computer: { ok: false, error: uiBlocked },
+      };
+    }
+
     if (action === 'screenshot') {
       await this.ensure();
       const shot = capturePrimaryScreenshotBase64();
@@ -410,22 +424,19 @@ export class MacComputerSession {
     await this.ensure();
 
     try {
-      if (
-        action === 'mouse_move' ||
-        action === 'left_click' ||
-        action === 'right_click' ||
-        action === 'double_click'
-      ) {
+      if (action === 'mouse_move') {
         if (!computer.coordinate) throw new Error('coordinate is required for this action.');
         const { pointX, pointY } = this.mapCoordinate(computer.coordinate);
         const moved = await this.send({ cmd: 'move', pointX, pointY });
         if (!moved.ok) throw new Error(moved.error || 'Move failed.');
-        if (action !== 'mouse_move') {
-          const button = action === 'right_click' ? 'right' : 'left';
-          const count = action === 'double_click' ? 2 : 1;
-          const clicked = await this.send({ cmd: 'click', button, count });
-          if (!clicked.ok) throw new Error(clicked.error || 'Click failed.');
-        }
+      } else if (action === 'left_click' || action === 'right_click' || action === 'double_click') {
+        if (!computer.coordinate) throw new Error('coordinate is required for this action.');
+        const { pointX, pointY } = this.mapCoordinate(computer.coordinate);
+        const button = action === 'right_click' ? 'right' : 'left';
+        const count = action === 'double_click' ? 2 : 1;
+        // One RPC: animate overlay, click, restore user cursor (helper-side).
+        const clicked = await this.send({ cmd: 'clickAt', pointX, pointY, button, count });
+        if (!clicked.ok) throw new Error(clicked.error || 'Click failed.');
       } else if (action === 'left_click_drag') {
         if (!computer.startCoordinate || !computer.coordinate) {
           throw new Error('startCoordinate and coordinate are required for drag.');
@@ -535,11 +546,130 @@ export async function probeComputerCapability() {
     await ensureHelperBinary();
     return true;
   } catch {
-    // Still advertise: screenshot-only path can work; input fails with TCC guidance.
-    // Prefer advertising when swiftc exists so gateway can send jobs after first grant.
     return (
       Boolean(spawnSync('which', ['swiftc'], { encoding: 'utf8' }).stdout?.trim()) ||
       existsSync(helperPaths().binary)
     );
   }
 }
+
+function normalizeUiState(value) {
+  return MAC_UI_STATES.includes(value) ? value : 'unknown';
+}
+
+/**
+ * Lock screen vs display sleep vs unlocked. Prefer the helper binary (CGSession +
+ * CGDisplayIsAsleep); fall back to unknown so we never invent "offline".
+ */
+export function probeMacUiState({ force = false } = {}) {
+  if (process.platform !== 'darwin') return 'unknown';
+  const now = Date.now();
+  if (!force && now - cachedUiState.at < UI_STATE_CACHE_MS) return cachedUiState.state;
+
+  let state = 'unknown';
+  const binary = helperPaths().binary;
+  if (existsSync(binary)) {
+    const probed = spawnSync(binary, ['--session-status'], {
+      encoding: 'utf8',
+      timeout: 4_000,
+      maxBuffer: 16_000,
+    });
+    if (probed.status === 0) {
+      try {
+        const line = (probed.stdout || '').trim().split('\n').pop();
+        const parsed = JSON.parse(line || '{}');
+        if (parsed?.ok) state = normalizeUiState(parsed.state);
+      } catch {
+        state = 'unknown';
+      }
+    }
+  }
+
+  cachedUiState = { at: now, state };
+  return state;
+}
+
+/**
+ * Prevent idle system sleep for the worker process so HTTPS polls keep firing when the
+ * screen is locked. Does not claim lid-closed / clamshell immunity (see docs/remote.md).
+ */
+export function startWorkerIdleAssertion(pid = process.pid) {
+  if (process.platform !== 'darwin') return null;
+  stopWorkerIdleAssertion();
+  try {
+    workerIdleCaffeinate = spawn('/usr/bin/caffeinate', ['-i', '-w', String(pid)], {
+      stdio: 'ignore',
+      detached: false,
+    });
+    workerIdleCaffeinate.on('error', () => {
+      workerIdleCaffeinate = null;
+    });
+    workerIdleCaffeinate.unref?.();
+    return workerIdleCaffeinate;
+  } catch {
+    workerIdleCaffeinate = null;
+    return null;
+  }
+}
+
+export function stopWorkerIdleAssertion() {
+  if (!workerIdleCaffeinate) return;
+  try {
+    workerIdleCaffeinate.kill('SIGTERM');
+  } catch {
+    // already gone
+  }
+  workerIdleCaffeinate = null;
+}
+
+/** While a computer-use session is open, also keep the display from idling off. */
+function startSessionDisplayAssertion(pid = process.pid) {
+  if (process.platform !== 'darwin') return;
+  stopSessionDisplayAssertion();
+  try {
+    sessionDisplayCaffeinate = spawn('/usr/bin/caffeinate', ['-dimsu', '-w', String(pid)], {
+      stdio: 'ignore',
+      detached: false,
+    });
+    sessionDisplayCaffeinate.on('error', () => {
+      sessionDisplayCaffeinate = null;
+    });
+    sessionDisplayCaffeinate.unref?.();
+  } catch {
+    sessionDisplayCaffeinate = null;
+  }
+}
+
+function stopSessionDisplayAssertion() {
+  if (!sessionDisplayCaffeinate) return;
+  try {
+    sessionDisplayCaffeinate.kill('SIGTERM');
+  } catch {
+    // already gone
+  }
+  sessionDisplayCaffeinate = null;
+}
+
+/** User-activity assertion: nudge a sleeping (unlocked) display awake. */
+export function wakeMacDisplay(seconds = 3) {
+  if (process.platform !== 'darwin') return;
+  const secs = Math.max(1, Math.min(30, Math.round(seconds)));
+  spawnSync('/usr/bin/caffeinate', ['-u', '-t', String(secs)], {
+    encoding: 'utf8',
+    timeout: (secs + 2) * 1000,
+  });
+}
+
+/**
+ * Before GUI actions: wake a sleeping display; refuse clearly when the session is locked.
+ * Returns null when Computer Use may proceed, or an English error string.
+ */
+export function assertComputerUiAllowed() {
+  let state = probeMacUiState({ force: true });
+  if (state === 'display_asleep' || state === 'unknown') {
+    wakeMacDisplay(3);
+    state = probeMacUiState({ force: true });
+  }
+  if (state === 'locked') return LOCKED_COMPUTER_MESSAGE;
+  return null;
+}

package/bin/computer-shared.mjs

@@ -0,0 +1,61 @@
+/**
+ * Shared computer-use helpers: coords, capabilities, motion easing.
+ * Platform backends live in computer-{macos,linux,windows}.mjs.
+ */
+
+export const COMPUTER_CAPABILITIES = Object.freeze({
+  darwin: 'computer-macos-v1',
+  linux: 'computer-linux-v1',
+  win32: 'computer-windows-v1',
+});
+
+const CAPABILITY_SET = new Set(Object.values(COMPUTER_CAPABILITIES));
+
+/** True when a worker capability string means remote_computer is supported. */
+export function isComputerCapability(value) {
+  return CAPABILITY_SET.has(value);
+}
+
+export function capabilityForPlatform(platform = process.platform) {
+  return COMPUTER_CAPABILITIES[platform] ?? null;
+}
+
+/** Map screenshot image pixels → display points (primary). Half-pixel centers the click. */
+export function imagePixelToPoint(x, y, geometry) {
+  const { originX, originY, pointWidth, pointHeight, widthPx, heightPx } = geometry;
+  if (!(widthPx > 0 && heightPx > 0 && pointWidth > 0 && pointHeight > 0)) {
+    throw new Error('Invalid screenshot geometry for coordinate mapping');
+  }
+  return {
+    pointX: originX + (x + 0.5) * (pointWidth / widthPx),
+    pointY: originY + (y + 0.5) * (pointHeight / heightPx),
+  };
+}
+
+export function scaleFromGeometry(geometry) {
+  return {
+    imageToPointX: geometry.pointWidth / geometry.widthPx,
+    imageToPointY: geometry.pointHeight / geometry.heightPx,
+  };
+}
+
+/** Ease-out cubic — snappy agent loops, still readable as motion. */
+export function easeOutCubic(t) {
+  const x = Math.min(1, Math.max(0, t));
+  return 1 - (1 - x) ** 3;
+}
+
+/**
+ * Duration (ms) for overlay travel. Caps keep agent loops fast on long jumps.
+ * ~0.25 ms/px with floor 28 ms / ceiling 110 ms.
+ */
+export function motionDurationMs(distancePx) {
+  const d = Math.max(0, distancePx);
+  return Math.round(Math.min(110, Math.max(28, d * 0.25)));
+}
+
+export function pointDistance(a, b) {
+  const dx = (b.pointX ?? b.x) - (a.pointX ?? a.x);
+  const dy = (b.pointY ?? b.y) - (a.pointY ?? a.y);
+  return Math.hypot(dx, dy);
+}