@jian-ai/remote-control 0.8.0 → 0.8.2

package/bin/computer-macos.mjs

@@ -1,6 +1,6 @@
 /**
  * macOS computer-use: primary-display screenshots, coord mapping, Jian overlay helper.
- * English-only messages. Imported by jian-remote.mjs on darwin.
+ * English-only messages. Imported by computer.mjs on darwin.
  */
 import { spawn, spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
@@ -9,40 +9,38 @@ import { chmod, copyFile, mkdir, readFile, writeFile } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { COMPUTER_CAPABILITIES, imagePixelToPoint, scaleFromGeometry } from './computer-shared.mjs';
 
-export const COMPUTER_CAPABILITY = 'computer-macos-v1';
+export { imagePixelToPoint, scaleFromGeometry } from './computer-shared.mjs';
+
+export const COMPUTER_CAPABILITY = COMPUTER_CAPABILITIES.darwin;
 const IDLE_MS = 30_000;
 const HELPER_NAME = 'computer-helper';
+/** How long we cache lock / display-asleep probes between HTTPS polls. */
+const UI_STATE_CACHE_MS = 5_000;
+export const MAC_UI_STATES = Object.freeze(['unlocked', 'locked', 'display_asleep', 'unknown']);
 
-/** Map screenshot image pixels → display points (primary). Half-pixel centers the click. */
-export function imagePixelToPoint(x, y, geometry) {
-  const { originX, originY, pointWidth, pointHeight, widthPx, heightPx } = geometry;
-  if (!(widthPx > 0 && heightPx > 0 && pointWidth > 0 && pointHeight > 0)) {
-    throw new Error('Invalid screenshot geometry for coordinate mapping');
-  }
-  return {
-    pointX: originX + (x + 0.5) * (pointWidth / widthPx),
-    pointY: originY + (y + 0.5) * (pointHeight / heightPx),
-  };
-}
+const LOCKED_COMPUTER_MESSAGE =
+  'Mac is locked — unlock the screen to use Computer Use. Shell and file tools may still work while the worker is online.';
 
-export function scaleFromGeometry(geometry) {
-  return {
-    imageToPointX: geometry.pointWidth / geometry.widthPx,
-    imageToPointY: geometry.pointHeight / geometry.heightPx,
-  };
-}
+let cachedUiState = { at: 0, state: 'unknown' };
+let workerIdleCaffeinate = null;
+let sessionDisplayCaffeinate = null;
 
-/** Capture primary display and compress like the gateway’s former screenshotCommand. */
+/**
+ * Capture primary display and compress for the worker channel.
+ * Single-pass ladder starts at 1600/42 (skips a full-res 1920@50 encode that often
+ * exceeded the budget and forced a second sips pass — common waste on Retina).
+ */
 export function capturePrimaryScreenshotBase64() {
   const script = [
     'tmp=$(mktemp /tmp/jian-shot.XXXXXX.jpg);',
     'screencapture -x -t jpg -D 1 "$tmp" || { rm -f "$tmp"; exit 1; };',
-    'sips -Z 1920 -s format jpeg -s formatOptions 50 "$tmp" >/dev/null 2>&1 || true;',
+    'sips -Z 1600 -s format jpeg -s formatOptions 42 "$tmp" >/dev/null 2>&1 || true;',
     'size=$(wc -c < "$tmp" | tr -d " ");',
-    'if [ "$size" -gt 280000 ]; then sips -Z 1440 -s format jpeg -s formatOptions 40 "$tmp" >/dev/null 2>&1 || true; fi;',
+    'if [ "$size" -gt 280000 ]; then sips -Z 1440 -s format jpeg -s formatOptions 38 "$tmp" >/dev/null 2>&1 || true; fi;',
     'size=$(wc -c < "$tmp" | tr -d " ");',
-    'if [ "$size" -gt 280000 ]; then sips -Z 1280 -s format jpeg -s formatOptions 35 "$tmp" >/dev/null 2>&1 || true; fi;',
+    'if [ "$size" -gt 280000 ]; then sips -Z 1280 -s format jpeg -s formatOptions 34 "$tmp" >/dev/null 2>&1 || true; fi;',
     'w=$(sips -g pixelWidth "$tmp" 2>/dev/null | awk "/pixelWidth/{print \\$2}");',
     'h=$(sips -g pixelHeight "$tmp" 2>/dev/null | awk "/pixelHeight/{print \\$2}");',
     'echo "$w $h";',
@@ -75,22 +73,77 @@ export function capturePrimaryScreenshotBase64() {
   return { base64, widthPx, heightPx };
 }
 
-function helperPaths() {
-  const support = join(homedir(), 'Library', 'Application Support', 'Jian Remote');
+function helperPaths(home = homedir()) {
+  const support = join(home, 'Library', 'Application Support', 'Jian Remote');
   return {
     support,
     source: join(support, `${HELPER_NAME}.swift`),
     binary: join(support, HELPER_NAME),
+    stamp: join(support, `${HELPER_NAME}.sha256`),
+  };
+}
+
+/** Paths inside the installed npm package (or a checkout). */
+export function packageHelperPaths(
+  packageRoot = join(dirname(fileURLToPath(import.meta.url)), '..'),
+) {
+  return {
+    swift: join(packageRoot, 'macos', `${HELPER_NAME}.swift`),
+    binary: join(packageRoot, 'macos', HELPER_NAME),
   };
 }
 
 function packageSwiftPath() {
-  return join(dirname(fileURLToPath(import.meta.url)), '..', 'macos', `${HELPER_NAME}.swift`);
+  return packageHelperPaths().swift;
 }
 
+function packageBinaryPath() {
+  return packageHelperPaths().binary;
+}
+
+function stampForBytes(bytes) {
+  return createHash('sha256').update(bytes).digest('hex').slice(0, 16);
+}
+
+/** Drop download quarantine so a helper from npm can exec under the LaunchAgent. */
+function clearQuarantine(path) {
+  spawnSync('xattr', ['-d', 'com.apple.quarantine', path], { encoding: 'utf8' });
+}
+
+/**
+ * Prefer a Publish-built universal binary from the npm package; otherwise compile with swiftc.
+ * Stamp is over the installed artifact bytes so updates replace a stale helper.
+ */
 async function ensureHelperBinary() {
   const paths = helperPaths();
   await mkdir(paths.support, { recursive: true, mode: 0o755 });
+
+  let previous = '';
+  try {
+    previous = (await readFile(paths.stamp, 'utf8')).trim();
+  } catch {
+    previous = '';
+  }
+
+  const packagedBinary = packageBinaryPath();
+  if (existsSync(packagedBinary)) {
+    const bytes = await readFile(packagedBinary);
+    const hash = `bin:${stampForBytes(bytes)}`;
+    if (previous !== hash || !existsSync(paths.binary)) {
+      await copyFile(packagedBinary, paths.binary);
+      await chmod(paths.binary, 0o755);
+      clearQuarantine(paths.binary);
+      await writeFile(paths.stamp, `${hash}\n`, { mode: 0o644 });
+    }
+    try {
+      const swift = packageSwiftPath();
+      if (existsSync(swift)) await copyFile(swift, paths.source);
+    } catch {
+      // optional
+    }
+    return paths.binary;
+  }
+
   let sourceText;
   try {
     sourceText = await readFile(packageSwiftPath(), 'utf8');
@@ -99,18 +152,11 @@ async function ensureHelperBinary() {
       sourceText = await readFile(paths.source, 'utf8');
     } catch {
       throw new Error(
-        'Computer-use helper source is missing. Reinstall or update @jian-ai/remote-control.',
+        'Computer-use helper is missing from this install. Reinstall or update @jian-ai/remote-control.',
       );
     }
   }
-  const hash = createHash('sha256').update(sourceText).digest('hex').slice(0, 16);
-  const stamp = join(paths.support, `${HELPER_NAME}.sha256`);
-  let previous = '';
-  try {
-    previous = (await readFile(stamp, 'utf8')).trim();
-  } catch {
-    previous = '';
-  }
+  const hash = `src:${stampForBytes(Buffer.from(sourceText, 'utf8'))}`;
   const needsBuild = previous !== hash || !existsSync(paths.binary);
   await writeFile(paths.source, sourceText, { mode: 0o644 });
   if (needsBuild) {
@@ -122,27 +168,44 @@ async function ensureHelperBinary() {
     if (compiled.status !== 0) {
       throw new Error(
         (compiled.stderr || compiled.stdout || 'swiftc failed').trim() ||
-          'Could not build the Jian computer-use helper. Install Xcode Command Line Tools.',
+          'Could not build the Jian computer-use helper. Published packages ship a prebuilt helper; for source builds install Xcode Command Line Tools, or reinstall @jian-ai/remote-control.',
       );
     }
     await chmod(paths.binary, 0o755);
-    await writeFile(stamp, `${hash}\n`, { mode: 0o644 });
+    clearQuarantine(paths.binary);
+    await writeFile(paths.stamp, `${hash}\n`, { mode: 0o644 });
   }
   return paths.binary;
 }
 
-/** Copy Swift sources next to the installed worker when present in the npm package. */
+/** Copy Swift source and any prebuilt helper next to the installed worker. */
 export async function materializeComputerHelperSources(configDir) {
   if (process.platform !== 'darwin') return;
   try {
-    const from = packageSwiftPath();
-    if (!existsSync(from)) return;
+    const pkg = packageHelperPaths();
+    if (!existsSync(pkg.swift) && !existsSync(pkg.binary)) return;
     const support = join(homedir(), 'Library', 'Application Support', 'Jian Remote');
     await mkdir(support, { recursive: true, mode: 0o755 });
-    await copyFile(from, join(support, `${HELPER_NAME}.swift`));
     const localMacos = join(configDir, 'macos');
     await mkdir(localMacos, { recursive: true, mode: 0o755 });
-    await copyFile(from, join(localMacos, `${HELPER_NAME}.swift`));
+    if (existsSync(pkg.swift)) {
+      await copyFile(pkg.swift, join(support, `${HELPER_NAME}.swift`));
+      await copyFile(pkg.swift, join(localMacos, `${HELPER_NAME}.swift`));
+    }
+    if (existsSync(pkg.binary)) {
+      const destSupport = join(support, HELPER_NAME);
+      const destLocal = join(localMacos, HELPER_NAME);
+      await copyFile(pkg.binary, destSupport);
+      await copyFile(pkg.binary, destLocal);
+      await chmod(destSupport, 0o755);
+      await chmod(destLocal, 0o755);
+      clearQuarantine(destSupport);
+      clearQuarantine(destLocal);
+      const bytes = await readFile(pkg.binary);
+      await writeFile(join(support, `${HELPER_NAME}.sha256`), `bin:${stampForBytes(bytes)}\n`, {
+        mode: 0o644,
+      });
+    }
   } catch {
     // Best-effort on install; computer session will surface a clear error.
   }
@@ -161,8 +224,11 @@ export class MacComputerSession {
 
   async ensure() {
     if (this.child && !this.child.killed) {
-      await this.send({ cmd: 'ensureSession' });
-      this.active = true;
+      if (!this.active) {
+        await this.send({ cmd: 'ensureSession' });
+        this.active = true;
+      }
+      startSessionDisplayAssertion(this.child.pid || process.pid);
       this.bumpIdle();
       return;
     }
@@ -181,6 +247,7 @@ export class MacComputerSession {
       this.failAll('Computer-use helper exited unexpectedly.');
       this.child = null;
       this.active = false;
+      stopSessionDisplayAssertion();
     });
     const ping = await this.send({ cmd: 'ping' });
     if (!ping.ok) throw new Error(ping.error || 'Computer-use helper did not respond.');
@@ -197,6 +264,7 @@ export class MacComputerSession {
     };
     await this.send({ cmd: 'ensureSession' });
     this.active = true;
+    startSessionDisplayAssertion(this.child.pid || process.pid);
     this.bumpIdle();
   }
 
@@ -277,6 +345,7 @@ export class MacComputerSession {
   async end() {
     if (this.idleTimer) clearTimeout(this.idleTimer);
     this.idleTimer = null;
+    stopSessionDisplayAssertion();
     if (this.child) {
       try {
         await this.send({ cmd: 'endSession' });
@@ -305,7 +374,7 @@ export class MacComputerSession {
     }
 
     if (action === 'wait') {
-      const ms = Math.round((computer.duration ?? 0.5) * 1000);
+      const ms = Math.round((computer.duration ?? 0.35) * 1000);
       await new Promise((r) => setTimeout(r, ms));
       if (this.active) this.bumpIdle();
       return {
@@ -317,6 +386,17 @@ export class MacComputerSession {
       };
     }
 
+    const uiBlocked = assertComputerUiAllowed();
+    if (uiBlocked) {
+      return {
+        exitCode: 1,
+        stdout: '',
+        stderr: uiBlocked,
+        timedOut: false,
+        computer: { ok: false, error: uiBlocked },
+      };
+    }
+
     if (action === 'screenshot') {
       await this.ensure();
       const shot = capturePrimaryScreenshotBase64();
@@ -344,22 +424,19 @@ export class MacComputerSession {
     await this.ensure();
 
     try {
-      if (
-        action === 'mouse_move' ||
-        action === 'left_click' ||
-        action === 'right_click' ||
-        action === 'double_click'
-      ) {
+      if (action === 'mouse_move') {
         if (!computer.coordinate) throw new Error('coordinate is required for this action.');
         const { pointX, pointY } = this.mapCoordinate(computer.coordinate);
         const moved = await this.send({ cmd: 'move', pointX, pointY });
         if (!moved.ok) throw new Error(moved.error || 'Move failed.');
-        if (action !== 'mouse_move') {
-          const button = action === 'right_click' ? 'right' : 'left';
-          const count = action === 'double_click' ? 2 : 1;
-          const clicked = await this.send({ cmd: 'click', button, count });
-          if (!clicked.ok) throw new Error(clicked.error || 'Click failed.');
-        }
+      } else if (action === 'left_click' || action === 'right_click' || action === 'double_click') {
+        if (!computer.coordinate) throw new Error('coordinate is required for this action.');
+        const { pointX, pointY } = this.mapCoordinate(computer.coordinate);
+        const button = action === 'right_click' ? 'right' : 'left';
+        const count = action === 'double_click' ? 2 : 1;
+        // One RPC: animate overlay, click, restore user cursor (helper-side).
+        const clicked = await this.send({ cmd: 'clickAt', pointX, pointY, button, count });
+        if (!clicked.ok) throw new Error(clicked.error || 'Click failed.');
       } else if (action === 'left_click_drag') {
         if (!computer.startCoordinate || !computer.coordinate) {
           throw new Error('startCoordinate and coordinate are required for drag.');
@@ -469,11 +546,130 @@ export async function probeComputerCapability() {
     await ensureHelperBinary();
     return true;
   } catch {
-    // Still advertise: screenshot-only path can work; input fails with TCC guidance.
-    // Prefer advertising when swiftc exists so gateway can send jobs after first grant.
     return (
       Boolean(spawnSync('which', ['swiftc'], { encoding: 'utf8' }).stdout?.trim()) ||
       existsSync(helperPaths().binary)
     );
   }
 }
+
+function normalizeUiState(value) {
+  return MAC_UI_STATES.includes(value) ? value : 'unknown';
+}
+
+/**
+ * Lock screen vs display sleep vs unlocked. Prefer the helper binary (CGSession +
+ * CGDisplayIsAsleep); fall back to unknown so we never invent "offline".
+ */
+export function probeMacUiState({ force = false } = {}) {
+  if (process.platform !== 'darwin') return 'unknown';
+  const now = Date.now();
+  if (!force && now - cachedUiState.at < UI_STATE_CACHE_MS) return cachedUiState.state;
+
+  let state = 'unknown';
+  const binary = helperPaths().binary;
+  if (existsSync(binary)) {
+    const probed = spawnSync(binary, ['--session-status'], {
+      encoding: 'utf8',
+      timeout: 4_000,
+      maxBuffer: 16_000,
+    });
+    if (probed.status === 0) {
+      try {
+        const line = (probed.stdout || '').trim().split('\n').pop();
+        const parsed = JSON.parse(line || '{}');
+        if (parsed?.ok) state = normalizeUiState(parsed.state);
+      } catch {
+        state = 'unknown';
+      }
+    }
+  }
+
+  cachedUiState = { at: now, state };
+  return state;
+}
+
+/**
+ * Prevent idle system sleep for the worker process so HTTPS polls keep firing when the
+ * screen is locked. Does not claim lid-closed / clamshell immunity (see docs/remote.md).
+ */
+export function startWorkerIdleAssertion(pid = process.pid) {
+  if (process.platform !== 'darwin') return null;
+  stopWorkerIdleAssertion();
+  try {
+    workerIdleCaffeinate = spawn('/usr/bin/caffeinate', ['-i', '-w', String(pid)], {
+      stdio: 'ignore',
+      detached: false,
+    });
+    workerIdleCaffeinate.on('error', () => {
+      workerIdleCaffeinate = null;
+    });
+    workerIdleCaffeinate.unref?.();
+    return workerIdleCaffeinate;
+  } catch {
+    workerIdleCaffeinate = null;
+    return null;
+  }
+}
+
+export function stopWorkerIdleAssertion() {
+  if (!workerIdleCaffeinate) return;
+  try {
+    workerIdleCaffeinate.kill('SIGTERM');
+  } catch {
+    // already gone
+  }
+  workerIdleCaffeinate = null;
+}
+
+/** While a computer-use session is open, also keep the display from idling off. */
+function startSessionDisplayAssertion(pid = process.pid) {
+  if (process.platform !== 'darwin') return;
+  stopSessionDisplayAssertion();
+  try {
+    sessionDisplayCaffeinate = spawn('/usr/bin/caffeinate', ['-dimsu', '-w', String(pid)], {
+      stdio: 'ignore',
+      detached: false,
+    });
+    sessionDisplayCaffeinate.on('error', () => {
+      sessionDisplayCaffeinate = null;
+    });
+    sessionDisplayCaffeinate.unref?.();
+  } catch {
+    sessionDisplayCaffeinate = null;
+  }
+}
+
+function stopSessionDisplayAssertion() {
+  if (!sessionDisplayCaffeinate) return;
+  try {
+    sessionDisplayCaffeinate.kill('SIGTERM');
+  } catch {
+    // already gone
+  }
+  sessionDisplayCaffeinate = null;
+}
+
+/** User-activity assertion: nudge a sleeping (unlocked) display awake. */
+export function wakeMacDisplay(seconds = 3) {
+  if (process.platform !== 'darwin') return;
+  const secs = Math.max(1, Math.min(30, Math.round(seconds)));
+  spawnSync('/usr/bin/caffeinate', ['-u', '-t', String(secs)], {
+    encoding: 'utf8',
+    timeout: (secs + 2) * 1000,
+  });
+}
+
+/**
+ * Before GUI actions: wake a sleeping display; refuse clearly when the session is locked.
+ * Returns null when Computer Use may proceed, or an English error string.
+ */
+export function assertComputerUiAllowed() {
+  let state = probeMacUiState({ force: true });
+  if (state === 'display_asleep' || state === 'unknown') {
+    wakeMacDisplay(3);
+    state = probeMacUiState({ force: true });
+  }
+  if (state === 'locked') return LOCKED_COMPUTER_MESSAGE;
+  return null;
+}

package/bin/computer-shared.mjs

@@ -0,0 +1,61 @@
+/**
+ * Shared computer-use helpers: coords, capabilities, motion easing.
+ * Platform backends live in computer-{macos,linux,windows}.mjs.
+ */
+
+export const COMPUTER_CAPABILITIES = Object.freeze({
+  darwin: 'computer-macos-v1',
+  linux: 'computer-linux-v1',
+  win32: 'computer-windows-v1',
+});
+
+const CAPABILITY_SET = new Set(Object.values(COMPUTER_CAPABILITIES));
+
+/** True when a worker capability string means remote_computer is supported. */
+export function isComputerCapability(value) {
+  return CAPABILITY_SET.has(value);
+}
+
+export function capabilityForPlatform(platform = process.platform) {
+  return COMPUTER_CAPABILITIES[platform] ?? null;
+}
+
+/** Map screenshot image pixels → display points (primary). Half-pixel centers the click. */
+export function imagePixelToPoint(x, y, geometry) {
+  const { originX, originY, pointWidth, pointHeight, widthPx, heightPx } = geometry;
+  if (!(widthPx > 0 && heightPx > 0 && pointWidth > 0 && pointHeight > 0)) {
+    throw new Error('Invalid screenshot geometry for coordinate mapping');
+  }
+  return {
+    pointX: originX + (x + 0.5) * (pointWidth / widthPx),
+    pointY: originY + (y + 0.5) * (pointHeight / heightPx),
+  };
+}
+
+export function scaleFromGeometry(geometry) {
+  return {
+    imageToPointX: geometry.pointWidth / geometry.widthPx,
+    imageToPointY: geometry.pointHeight / geometry.heightPx,
+  };
+}
+
+/** Ease-out cubic — snappy agent loops, still readable as motion. */
+export function easeOutCubic(t) {
+  const x = Math.min(1, Math.max(0, t));
+  return 1 - (1 - x) ** 3;
+}
+
+/**
+ * Duration (ms) for overlay travel. Caps keep agent loops fast on long jumps.
+ * ~0.25 ms/px with floor 28 ms / ceiling 110 ms.
+ */
+export function motionDurationMs(distancePx) {
+  const d = Math.max(0, distancePx);
+  return Math.round(Math.min(110, Math.max(28, d * 0.25)));
+}
+
+export function pointDistance(a, b) {
+  const dx = (b.pointX ?? b.x) - (a.pointX ?? a.x);
+  const dy = (b.pointY ?? b.y) - (a.pointY ?? a.y);
+  return Math.hypot(dx, dy);
+}