@jian-ai/remote-control 0.8.1 → 0.8.3

package/bin/computer-macos.mjs

@@ -1,6 +1,6 @@
 /**
  * macOS computer-use: primary-display screenshots, coord mapping, Jian overlay helper.
- * English-only messages. Imported by jian-remote.mjs on darwin.
+ * English-only messages. Imported by computer.mjs on darwin.
  */
 import { spawn, spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
@@ -9,40 +9,41 @@ import { chmod, copyFile, mkdir, readFile, writeFile } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { COMPUTER_CAPABILITIES, imagePixelToPoint, scaleFromGeometry } from './computer-shared.mjs';
 
-export const COMPUTER_CAPABILITY = 'computer-macos-v1';
+export { imagePixelToPoint, scaleFromGeometry } from './computer-shared.mjs';
+
+export const COMPUTER_CAPABILITY = COMPUTER_CAPABILITIES.darwin;
 const IDLE_MS = 30_000;
 const HELPER_NAME = 'computer-helper';
+/** How long we cache lock / display-asleep probes between HTTPS polls. */
+const UI_STATE_CACHE_MS = 5_000;
+export const MAC_UI_STATES = Object.freeze(['unlocked', 'locked', 'display_asleep', 'unknown']);
 
-/** Map screenshot image pixels → display points (primary). Half-pixel centers the click. */
-export function imagePixelToPoint(x, y, geometry) {
-  const { originX, originY, pointWidth, pointHeight, widthPx, heightPx } = geometry;
-  if (!(widthPx > 0 && heightPx > 0 && pointWidth > 0 && pointHeight > 0)) {
-    throw new Error('Invalid screenshot geometry for coordinate mapping');
-  }
-  return {
-    pointX: originX + (x + 0.5) * (pointWidth / widthPx),
-    pointY: originY + (y + 0.5) * (pointHeight / heightPx),
-  };
-}
+const LOCKED_SCREENSHOT_NOTE =
+  'Note: Mac session reports locked — image may be the lock screen (or black if the display stayed asleep). Desktop apps behind loginwindow are often not visible.';
 
-export function scaleFromGeometry(geometry) {
-  return {
-    imageToPointX: geometry.pointWidth / geometry.widthPx,
-    imageToPointY: geometry.pointHeight / geometry.heightPx,
-  };
-}
+const LOCKED_ACTION_HINT =
+  ' Mac session reports locked; if this failed, unlock or keep the display awake (post-login lock ≠ FileVault loginwindow).';
 
-/** Capture primary display and compress like the gateway’s former screenshotCommand. */
+let cachedUiState = { at: 0, state: 'unknown' };
+let workerIdleCaffeinate = null;
+let sessionDisplayCaffeinate = null;
+
+/**
+ * Capture primary display and compress for the worker channel.
+ * Single-pass ladder starts at 1600/42 (skips a full-res 1920@50 encode that often
+ * exceeded the budget and forced a second sips pass — common waste on Retina).
+ */
 export function capturePrimaryScreenshotBase64() {
   const script = [
     'tmp=$(mktemp /tmp/jian-shot.XXXXXX.jpg);',
     'screencapture -x -t jpg -D 1 "$tmp" || { rm -f "$tmp"; exit 1; };',
-    'sips -Z 1920 -s format jpeg -s formatOptions 50 "$tmp" >/dev/null 2>&1 || true;',
+    'sips -Z 1600 -s format jpeg -s formatOptions 42 "$tmp" >/dev/null 2>&1 || true;',
     'size=$(wc -c < "$tmp" | tr -d " ");',
-    'if [ "$size" -gt 280000 ]; then sips -Z 1440 -s format jpeg -s formatOptions 40 "$tmp" >/dev/null 2>&1 || true; fi;',
+    'if [ "$size" -gt 280000 ]; then sips -Z 1440 -s format jpeg -s formatOptions 38 "$tmp" >/dev/null 2>&1 || true; fi;',
     'size=$(wc -c < "$tmp" | tr -d " ");',
-    'if [ "$size" -gt 280000 ]; then sips -Z 1280 -s format jpeg -s formatOptions 35 "$tmp" >/dev/null 2>&1 || true; fi;',
+    'if [ "$size" -gt 280000 ]; then sips -Z 1280 -s format jpeg -s formatOptions 34 "$tmp" >/dev/null 2>&1 || true; fi;',
     'w=$(sips -g pixelWidth "$tmp" 2>/dev/null | awk "/pixelWidth/{print \\$2}");',
     'h=$(sips -g pixelHeight "$tmp" 2>/dev/null | awk "/pixelHeight/{print \\$2}");',
     'echo "$w $h";',
@@ -137,7 +138,6 @@ async function ensureHelperBinary() {
       clearQuarantine(paths.binary);
       await writeFile(paths.stamp, `${hash}\n`, { mode: 0o644 });
     }
-    // Keep source nearby for debugging / swiftc fallback after a partial install.
     try {
       const swift = packageSwiftPath();
       if (existsSync(swift)) await copyFile(swift, paths.source);
@@ -227,8 +227,11 @@ export class MacComputerSession {
 
   async ensure() {
     if (this.child && !this.child.killed) {
-      await this.send({ cmd: 'ensureSession' });
-      this.active = true;
+      if (!this.active) {
+        await this.send({ cmd: 'ensureSession' });
+        this.active = true;
+      }
+      startSessionDisplayAssertion(this.child.pid || process.pid);
       this.bumpIdle();
       return;
     }
@@ -247,6 +250,7 @@ export class MacComputerSession {
       this.failAll('Computer-use helper exited unexpectedly.');
       this.child = null;
       this.active = false;
+      stopSessionDisplayAssertion();
     });
     const ping = await this.send({ cmd: 'ping' });
     if (!ping.ok) throw new Error(ping.error || 'Computer-use helper did not respond.');
@@ -263,6 +267,7 @@ export class MacComputerSession {
     };
     await this.send({ cmd: 'ensureSession' });
     this.active = true;
+    startSessionDisplayAssertion(this.child.pid || process.pid);
     this.bumpIdle();
   }
 
@@ -343,6 +348,7 @@ export class MacComputerSession {
   async end() {
     if (this.idleTimer) clearTimeout(this.idleTimer);
     this.idleTimer = null;
+    stopSessionDisplayAssertion();
     if (this.child) {
       try {
         await this.send({ cmd: 'endSession' });
@@ -371,7 +377,7 @@ export class MacComputerSession {
     }
 
     if (action === 'wait') {
-      const ms = Math.round((computer.duration ?? 0.5) * 1000);
+      const ms = Math.round((computer.duration ?? 0.35) * 1000);
       await new Promise((r) => setTimeout(r, ms));
       if (this.active) this.bumpIdle();
       return {
@@ -383,49 +389,66 @@ export class MacComputerSession {
       };
     }
 
+    // Wake display when locked/asleep, then always attempt — never soft-block on uiState.
+    const uiPrep = prepareComputerUi();
+
     if (action === 'screenshot') {
-      await this.ensure();
-      const shot = capturePrimaryScreenshotBase64();
-      if (!this.geometry) throw new Error('Display geometry unavailable.');
-      this.shotGeometry = {
-        ...this.geometry,
-        widthPx: shot.widthPx,
-        heightPx: shot.heightPx,
-      };
-      this.bumpIdle();
-      return {
-        exitCode: 0,
-        stdout: shot.base64,
-        stderr: '',
-        timedOut: false,
-        computer: {
-          ok: true,
+      try {
+        await this.ensure();
+        const shot = capturePrimaryScreenshotBase64();
+        if (!this.geometry) throw new Error('Display geometry unavailable.');
+        this.shotGeometry = {
+          ...this.geometry,
           widthPx: shot.widthPx,
           heightPx: shot.heightPx,
-          scale: scaleFromGeometry(this.shotGeometry),
-        },
-      };
+        };
+        this.bumpIdle();
+        const stderr = uiPrep.sessionLocked ? LOCKED_SCREENSHOT_NOTE : '';
+        return {
+          exitCode: 0,
+          stdout: shot.base64,
+          stderr,
+          timedOut: false,
+          computer: {
+            ok: true,
+            widthPx: shot.widthPx,
+            heightPx: shot.heightPx,
+            scale: scaleFromGeometry(this.shotGeometry),
+          },
+        };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        const hint = uiPrep.sessionLocked ? LOCKED_ACTION_HINT : '';
+        const tcc = /accessib|not permitted|not authorized|Screen Recording/i.test(message)
+          ? ' Jian Remote needs Screen Recording (and Accessibility for input) in System Settings.'
+          : '';
+        const full = `${message}${tcc}${hint}`.trim();
+        return {
+          exitCode: 1,
+          stdout: '',
+          stderr: full,
+          timedOut: false,
+          computer: { ok: false, error: full },
+        };
+      }
     }
 
     await this.ensure();
 
     try {
-      if (
-        action === 'mouse_move' ||
-        action === 'left_click' ||
-        action === 'right_click' ||
-        action === 'double_click'
-      ) {
+      if (action === 'mouse_move') {
         if (!computer.coordinate) throw new Error('coordinate is required for this action.');
         const { pointX, pointY } = this.mapCoordinate(computer.coordinate);
         const moved = await this.send({ cmd: 'move', pointX, pointY });
         if (!moved.ok) throw new Error(moved.error || 'Move failed.');
-        if (action !== 'mouse_move') {
-          const button = action === 'right_click' ? 'right' : 'left';
-          const count = action === 'double_click' ? 2 : 1;
-          const clicked = await this.send({ cmd: 'click', button, count });
-          if (!clicked.ok) throw new Error(clicked.error || 'Click failed.');
-        }
+      } else if (action === 'left_click' || action === 'right_click' || action === 'double_click') {
+        if (!computer.coordinate) throw new Error('coordinate is required for this action.');
+        const { pointX, pointY } = this.mapCoordinate(computer.coordinate);
+        const button = action === 'right_click' ? 'right' : 'left';
+        const count = action === 'double_click' ? 2 : 1;
+        // One RPC: animate overlay, click, restore user cursor (helper-side).
+        const clicked = await this.send({ cmd: 'clickAt', pointX, pointY, button, count });
+        if (!clicked.ok) throw new Error(clicked.error || 'Click failed.');
       } else if (action === 'left_click_drag') {
         if (!computer.startCoordinate || !computer.coordinate) {
           throw new Error('startCoordinate and coordinate are required for drag.');
@@ -468,12 +491,14 @@ export class MacComputerSession {
       const tcc = /accessib|not permitted|not authorized/i.test(message)
         ? ' Jian Remote needs Accessibility (and Screen Recording for screenshots) in System Settings.'
         : '';
+      const hint = uiPrep.sessionLocked ? LOCKED_ACTION_HINT : '';
+      const full = `${message}${tcc}${hint}`.trim();
       return {
         exitCode: 1,
         stdout: '',
-        stderr: `${message}${tcc}`,
+        stderr: full,
         timedOut: false,
-        computer: { ok: false, error: `${message}${tcc}`.trim() },
+        computer: { ok: false, error: full },
       };
     }
 
@@ -535,11 +560,129 @@ export async function probeComputerCapability() {
     await ensureHelperBinary();
     return true;
   } catch {
-    // Still advertise: screenshot-only path can work; input fails with TCC guidance.
-    // Prefer advertising when swiftc exists so gateway can send jobs after first grant.
     return (
       Boolean(spawnSync('which', ['swiftc'], { encoding: 'utf8' }).stdout?.trim()) ||
       existsSync(helperPaths().binary)
     );
   }
 }
+
+function normalizeUiState(value) {
+  return MAC_UI_STATES.includes(value) ? value : 'unknown';
+}
+
+/**
+ * Lock screen vs display sleep vs unlocked. Prefer the helper binary (CGSession +
+ * CGDisplayIsAsleep); fall back to unknown so we never invent "offline".
+ */
+export function probeMacUiState({ force = false } = {}) {
+  if (process.platform !== 'darwin') return 'unknown';
+  const now = Date.now();
+  if (!force && now - cachedUiState.at < UI_STATE_CACHE_MS) return cachedUiState.state;
+
+  let state = 'unknown';
+  const binary = helperPaths().binary;
+  if (existsSync(binary)) {
+    const probed = spawnSync(binary, ['--session-status'], {
+      encoding: 'utf8',
+      timeout: 4_000,
+      maxBuffer: 16_000,
+    });
+    if (probed.status === 0) {
+      try {
+        const line = (probed.stdout || '').trim().split('\n').pop();
+        const parsed = JSON.parse(line || '{}');
+        if (parsed?.ok) state = normalizeUiState(parsed.state);
+      } catch {
+        state = 'unknown';
+      }
+    }
+  }
+
+  cachedUiState = { at: now, state };
+  return state;
+}
+
+/**
+ * Prevent idle system sleep for the worker process so HTTPS polls keep firing when the
+ * screen is locked. Does not claim lid-closed / clamshell immunity (see docs/remote.md).
+ */
+export function startWorkerIdleAssertion(pid = process.pid) {
+  if (process.platform !== 'darwin') return null;
+  stopWorkerIdleAssertion();
+  try {
+    workerIdleCaffeinate = spawn('/usr/bin/caffeinate', ['-i', '-w', String(pid)], {
+      stdio: 'ignore',
+      detached: false,
+    });
+    workerIdleCaffeinate.on('error', () => {
+      workerIdleCaffeinate = null;
+    });
+    workerIdleCaffeinate.unref?.();
+    return workerIdleCaffeinate;
+  } catch {
+    workerIdleCaffeinate = null;
+    return null;
+  }
+}
+
+export function stopWorkerIdleAssertion() {
+  if (!workerIdleCaffeinate) return;
+  try {
+    workerIdleCaffeinate.kill('SIGTERM');
+  } catch {
+    // already gone
+  }
+  workerIdleCaffeinate = null;
+}
+
+/** While a computer-use session is open, also keep the display from idling off. */
+function startSessionDisplayAssertion(pid = process.pid) {
+  if (process.platform !== 'darwin') return;
+  stopSessionDisplayAssertion();
+  try {
+    sessionDisplayCaffeinate = spawn('/usr/bin/caffeinate', ['-dimsu', '-w', String(pid)], {
+      stdio: 'ignore',
+      detached: false,
+    });
+    sessionDisplayCaffeinate.on('error', () => {
+      sessionDisplayCaffeinate = null;
+    });
+    sessionDisplayCaffeinate.unref?.();
+  } catch {
+    sessionDisplayCaffeinate = null;
+  }
+}
+
+function stopSessionDisplayAssertion() {
+  if (!sessionDisplayCaffeinate) return;
+  try {
+    sessionDisplayCaffeinate.kill('SIGTERM');
+  } catch {
+    // already gone
+  }
+  sessionDisplayCaffeinate = null;
+}
+
+/** User-activity assertion: nudge a sleeping (unlocked) display awake. */
+export function wakeMacDisplay(seconds = 3) {
+  if (process.platform !== 'darwin') return;
+  const secs = Math.max(1, Math.min(30, Math.round(seconds)));
+  spawnSync('/usr/bin/caffeinate', ['-u', '-t', String(secs)], {
+    encoding: 'utf8',
+    timeout: (secs + 2) * 1000,
+  });
+}
+
+/**
+ * Wake the display when locked or asleep (lock often coincides with display sleep).
+ * Never soft-blocks: callers always attempt the OS API next.
+ */
+export function prepareComputerUi() {
+  let state = probeMacUiState({ force: true });
+  if (state === 'display_asleep' || state === 'locked' || state === 'unknown') {
+    wakeMacDisplay(3);
+    state = probeMacUiState({ force: true });
+  }
+  return { sessionLocked: state === 'locked', state };
+}

package/bin/computer-shared.mjs

@@ -0,0 +1,61 @@
+/**
+ * Shared computer-use helpers: coords, capabilities, motion easing.
+ * Platform backends live in computer-{macos,linux,windows}.mjs.
+ */
+
+export const COMPUTER_CAPABILITIES = Object.freeze({
+  darwin: 'computer-macos-v1',
+  linux: 'computer-linux-v1',
+  win32: 'computer-windows-v1',
+});
+
+const CAPABILITY_SET = new Set(Object.values(COMPUTER_CAPABILITIES));
+
+/** True when a worker capability string means remote_computer is supported. */
+export function isComputerCapability(value) {
+  return CAPABILITY_SET.has(value);
+}
+
+export function capabilityForPlatform(platform = process.platform) {
+  return COMPUTER_CAPABILITIES[platform] ?? null;
+}
+
+/** Map screenshot image pixels → display points (primary). Half-pixel centers the click. */
+export function imagePixelToPoint(x, y, geometry) {
+  const { originX, originY, pointWidth, pointHeight, widthPx, heightPx } = geometry;
+  if (!(widthPx > 0 && heightPx > 0 && pointWidth > 0 && pointHeight > 0)) {
+    throw new Error('Invalid screenshot geometry for coordinate mapping');
+  }
+  return {
+    pointX: originX + (x + 0.5) * (pointWidth / widthPx),
+    pointY: originY + (y + 0.5) * (pointHeight / heightPx),
+  };
+}
+
+export function scaleFromGeometry(geometry) {
+  return {
+    imageToPointX: geometry.pointWidth / geometry.widthPx,
+    imageToPointY: geometry.pointHeight / geometry.heightPx,
+  };
+}
+
+/** Ease-out cubic — snappy agent loops, still readable as motion. */
+export function easeOutCubic(t) {
+  const x = Math.min(1, Math.max(0, t));
+  return 1 - (1 - x) ** 3;
+}
+
+/**
+ * Duration (ms) for overlay travel. Caps keep agent loops fast on long jumps.
+ * ~0.25 ms/px with floor 28 ms / ceiling 110 ms.
+ */
+export function motionDurationMs(distancePx) {
+  const d = Math.max(0, distancePx);
+  return Math.round(Math.min(110, Math.max(28, d * 0.25)));
+}
+
+export function pointDistance(a, b) {
+  const dx = (b.pointX ?? b.x) - (a.pointX ?? a.x);
+  const dy = (b.pointY ?? b.y) - (a.pointY ?? a.y);
+  return Math.hypot(dx, dy);
+}