onbuzz 4.7.2 → 4.8.0

package/src/tools/desktop/osController.js

@@ -0,0 +1,491 @@
+/**
+ * OS-level desktop primitives — screenshot, mouse, keyboard, windows.
+ *
+ * Wraps `@nut-tree-fork/nut-js` (cross-platform input + screen capture).
+ * Two design choices worth knowing about:
+ *
+ *   1. LAZY NATIVE LOADING. `nut.js` is an OPTIONAL dependency (~30MB
+ *      native binary, needs build tools per OS). It's loaded on first
+ *      use via dynamic `import()`. Headless server CLI installs that
+ *      never call DesktopTool never pay the cost. Build failures on
+ *      Wayland / minimal containers don't break the rest of Loxia —
+ *      the controller surfaces a clear `code: 'NATIVE_UNAVAILABLE'`
+ *      error and the agent falls back gracefully.
+ *
+ *   2. CAPABILITY-AWARE. Every method consults osCapabilities and
+ *      refuses operations the OS won't support (e.g. mouseInput on
+ *      Wayland). Capabilities are probed once at boot.
+ *
+ * Coordinates are CSS-style pixels, top-left origin. The controller
+ * normalises across DPI / Retina via nut.js's built-in scaling. The
+ * caller (DesktopTool / model) never sees raw device pixels.
+ */
+
+import { detectOSCapabilities, PlatformId } from './osCapabilities.js';
+import { makeError } from '../../services/grounding/GroundingModel.js';
+
+// ─── stable error codes ─────────────────────────────────────────────
+
+export const OSError = Object.freeze({
+  /** nut.js failed to load (missing dep, build error, etc.). */
+  NATIVE_UNAVAILABLE: 'OS_NATIVE_UNAVAILABLE',
+  /** The current OS / display server doesn't support this op. */
+  UNSUPPORTED:        'OS_UNSUPPORTED',
+  /** Coords / args are outside the screen or otherwise invalid. */
+  INVALID_INPUT:      'OS_INVALID_INPUT',
+  /** Underlying op (nut.js call) threw. */
+  OP_FAILED:          'OS_OP_FAILED',
+  /** Permission missing (macOS Accessibility / Screen Recording). */
+  PERMISSION_DENIED:  'OS_PERMISSION_DENIED',
+});
+
+// ─── tunable defaults ───────────────────────────────────────────────
+
+const Defaults = Object.freeze({
+  /** Per-keystroke delay when typing — keeps under most app input caps. */
+  TYPE_DELAY_MS: 20,
+  /** Inter-click delay for double-clicks. */
+  DOUBLE_CLICK_GAP_MS: 80,
+  /** Drag step duration; quick enough to feel snappy. */
+  DRAG_DURATION_MS: 200,
+});
+
+/** Mouse button names mapped to nut.js Button enum keys. */
+const MouseButton = Object.freeze({
+  LEFT:   'LEFT',
+  RIGHT:  'RIGHT',
+  MIDDLE: 'MIDDLE',
+});
+
+/**
+ * Cross-platform key-name aliases → nut.js Key enum names. Maps the
+ * names agents naturally try (Meta, Command, Cmd, Windows, Win, Super,
+ * Ctrl, Esc, Return, etc.) onto the actual nut.js names. Lookups in
+ * keyPress check this map BEFORE the strict Key[name] lookup, so
+ * "Meta" works on Windows (→ LeftWin), "Command" works on macOS
+ * (→ LeftCmd), and friends-of-friends spellings resolve too.
+ *
+ * Keep this list curated — adding every conceivable spelling makes
+ * debugging "why did this map" harder. Only add aliases an agent
+ * actually emits in practice.
+ */
+const KEY_ALIASES = Object.freeze({
+  // OS / Modifier keys
+  Meta:    'LeftWin',  meta:    'LeftWin',
+  Command: 'LeftCmd',  command: 'LeftCmd',
+  Cmd:     'LeftCmd',  cmd:     'LeftCmd',
+  Windows: 'LeftWin',  windows: 'LeftWin',
+  Win:     'LeftWin',  win:     'LeftWin',
+  Super:   'LeftSuper', super:  'LeftSuper',
+  Ctrl:    'LeftControl', ctrl: 'LeftControl', Control: 'LeftControl', control: 'LeftControl',
+  Alt:     'LeftAlt',  alt:     'LeftAlt',
+  Shift:   'LeftShift', shift:  'LeftShift',
+  // Common punctuation / whitespace aliases
+  Esc:     'Escape',   esc:     'Escape',
+  Return:  'Enter',    return:  'Enter',
+  Del:     'Delete',   del:     'Delete',
+  Ins:     'Insert',   ins:     'Insert',
+  PgUp:    'PageUp',   pgup:    'PageUp',
+  PgDown:  'PageDown', pgdown:  'PageDown',
+  Space:   'Space',    space:   'Space',
+});
+
+/**
+ * Construct + return a controller. Caller passes a capabilities
+ * snapshot (or null → we'll detect). Lazy-import of nut.js happens
+ * inside; constructor is sync.
+ *
+ * @param {Object} [opts]
+ * @param {import('./osCapabilities.js').OSCapabilities} [opts.capabilities]
+ * @param {Object} [opts.nutImpl]    - injected for tests; defaults to lazy import
+ * @returns {OSController}
+ */
+export function createOSController(opts = {}) {
+  return new OSController(opts);
+}
+
+export class OSController {
+  constructor({ capabilities = null, nutImpl = null } = {}) {
+    this._caps = capabilities;
+    this._nutImpl = nutImpl;
+    this._loadPromise = null;
+  }
+
+  // ─── lifecycle ────────────────────────────────────────────────────
+
+  /**
+   * Resolves capabilities + lazy-loads nut.js. Idempotent — safe to
+   * call before each op without re-incurring the load cost.
+   */
+  async _ensureReady() {
+    if (!this._caps) this._caps = await detectOSCapabilities();
+    if (this._nutImpl) return;
+    if (!this._loadPromise) this._loadPromise = this._loadNative();
+    await this._loadPromise;
+  }
+
+  async _loadNative() {
+    try {
+      // Dynamic import keeps this an OPTIONAL dependency. Wrapped in a
+      // try so a missing-or-broken nut.js doesn't crash the whole CLI.
+      this._nutImpl = await import('@nut-tree-fork/nut-js');
+    } catch (err) {
+      throw makeError(OSError.NATIVE_UNAVAILABLE,
+        'DesktopTool requires "@nut-tree-fork/nut-js" — install it (it is in optionalDependencies) ' +
+        'or check the platform build logs. Original error: ' + err.message);
+    }
+  }
+
+  /**
+   * Returns a frozen snapshot of OS capabilities + load status.
+   * Useful for the admin UI's "OS capability status" panel.
+   */
+  async describeCapabilities() {
+    await this._ensureReady().catch(() => {/* still report caps */});
+    return {
+      ...this._caps,
+      nativeLoaded: this._nutImpl != null,
+    };
+  }
+
+  // ─── primitives ───────────────────────────────────────────────────
+
+  /**
+   * Capture the screen (or a region/monitor) as PNG bytes.
+   * @param {Object} [opts]
+   * @param {number} [opts.monitor]   - 0-indexed monitor; default 0
+   * @param {{x:number,y:number,width:number,height:number}} [opts.region]
+   * @returns {Promise<Buffer>}
+   */
+  async screenshot(opts = {}) {
+    await this._ensureReady();
+    this._requireCapability('screenshot');
+    try {
+      const { screen, Region } = this._nutImpl;
+      if (opts.region) {
+        validateRegion(opts.region);
+        const r = new Region(opts.region.x, opts.region.y, opts.region.width, opts.region.height);
+        return await screen.grabRegion(r).then(toPngBuffer);
+      }
+      return await screen.grab().then(toPngBuffer);
+    } catch (err) {
+      throw wrapOpError('screenshot', err);
+    }
+  }
+
+  /** Returns the primary screen size in CSS pixels + DPI ratio. */
+  async screenSize() {
+    await this._ensureReady();
+    try {
+      const { screen } = this._nutImpl;
+      const width  = await screen.width();
+      const height = await screen.height();
+      return { width, height, dpr: 1 }; // nut.js normalizes to CSS px
+    } catch (err) {
+      throw wrapOpError('screenSize', err);
+    }
+  }
+
+  /**
+   * Move the mouse cursor to (x, y).
+   * @param {number} x
+   * @param {number} y
+   */
+  async mouseMove(x, y) {
+    await this._ensureReady();
+    this._requireCapability('mouseInput');
+    validateCoord(x, y);
+    try {
+      const { mouse, Point } = this._nutImpl;
+      await mouse.move([new Point(x, y)]);
+    } catch (err) {
+      throw wrapOpError('mouseMove', err);
+    }
+  }
+
+  /**
+   * Click at (x, y). Move-then-click; single op so DesktopTool's
+   * mutex only needs to lock once per logical click.
+   *
+   * @param {number} x
+   * @param {number} y
+   * @param {Object} [opts]
+   * @param {'left'|'right'|'middle'} [opts.button='left']
+   * @param {1|2} [opts.count=1]
+   */
+  async mouseClick(x, y, opts = {}) {
+    await this._ensureReady();
+    this._requireCapability('mouseInput');
+    validateCoord(x, y);
+    const button = (opts.button || 'left').toUpperCase();
+    if (!MouseButton[button]) {
+      throw makeError(OSError.INVALID_INPUT, `unknown button "${opts.button}"`);
+    }
+    const count = opts.count === 2 ? 2 : 1;
+    try {
+      const { mouse, Point, Button } = this._nutImpl;
+      await mouse.move([new Point(x, y)]);
+      const btn = Button[button];
+      if (count === 2) {
+        await mouse.doubleClick(btn);
+      } else {
+        await mouse.click(btn);
+      }
+    } catch (err) {
+      throw wrapOpError('mouseClick', err);
+    }
+  }
+
+  /**
+   * Scroll at (x, y).
+   * @param {number} x
+   * @param {number} y
+   * @param {number} dy   - positive = scroll down
+   */
+  async mouseScroll(x, y, dy) {
+    await this._ensureReady();
+    this._requireCapability('mouseInput');
+    validateCoord(x, y);
+    if (!Number.isFinite(dy)) throw makeError(OSError.INVALID_INPUT, 'dy must be a number');
+    try {
+      const { mouse, Point } = this._nutImpl;
+      await mouse.move([new Point(x, y)]);
+      if (dy > 0) await mouse.scrollDown(Math.abs(dy));
+      else if (dy < 0) await mouse.scrollUp(Math.abs(dy));
+    } catch (err) {
+      throw wrapOpError('mouseScroll', err);
+    }
+  }
+
+  /**
+   * Drag from one point to another.
+   * @param {{x:number,y:number}} from
+   * @param {{x:number,y:number}} to
+   * @param {Object} [opts]
+   * @param {'left'|'right'|'middle'} [opts.button='left']
+   */
+  async mouseDrag(from, to, opts = {}) {
+    await this._ensureReady();
+    this._requireCapability('mouseInput');
+    validateCoord(from.x, from.y);
+    validateCoord(to.x, to.y);
+    const button = (opts.button || 'left').toUpperCase();
+    if (!MouseButton[button]) {
+      throw makeError(OSError.INVALID_INPUT, `unknown button "${opts.button}"`);
+    }
+    try {
+      const { mouse, Point, Button } = this._nutImpl;
+      await mouse.move([new Point(from.x, from.y)]);
+      await mouse.pressButton(Button[button]);
+      await mouse.move([new Point(to.x, to.y)]);
+      await mouse.releaseButton(Button[button]);
+    } catch (err) {
+      throw wrapOpError('mouseDrag', err);
+    }
+  }
+
+  /**
+   * Type a literal string into the focused app.
+   * @param {string} text
+   * @param {Object} [opts]
+   * @param {number} [opts.delayMs=Defaults.TYPE_DELAY_MS]
+   */
+  async typeText(text, opts = {}) {
+    await this._ensureReady();
+    this._requireCapability('keyboardInput');
+    if (typeof text !== 'string' || text.length === 0) {
+      throw makeError(OSError.INVALID_INPUT, 'text must be a non-empty string');
+    }
+    const delayMs = Number.isFinite(opts.delayMs) ? opts.delayMs : Defaults.TYPE_DELAY_MS;
+    try {
+      const { keyboard } = this._nutImpl;
+      keyboard.config.autoDelayMs = delayMs;
+      await keyboard.type(text);
+    } catch (err) {
+      throw wrapOpError('typeText', err);
+    }
+  }
+
+  /**
+   * Send a chord (e.g. Ctrl+S, Alt+Tab). Keys are nut.js-style names
+   * (Control, Shift, S, F5, Enter, etc.).
+   * @param {string[]} keys
+   */
+  async keyPress(keys) {
+    await this._ensureReady();
+    this._requireCapability('keyboardInput');
+    if (!Array.isArray(keys) || keys.length === 0) {
+      throw makeError(OSError.INVALID_INPUT, 'keys must be a non-empty array');
+    }
+    try {
+      const { keyboard, Key } = this._nutImpl;
+      const resolved = keys.map(k => {
+        // 1. Cross-platform aliases — agents naturally try "Meta",
+        //    "Command", "Cmd", "Super", "Windows", "Win". Map those to
+        //    nut.js's actual key names (LeftWin / LeftCmd) before
+        //    looking up. Without this, a perfectly reasonable
+        //    "press Windows key" call dies with OS_INVALID_INPUT.
+        const normalized = KEY_ALIASES[k] ?? KEY_ALIASES[k.toLowerCase()] ?? k;
+        const upper = normalized.length === 1 ? normalized.toUpperCase() : normalized;
+        if (Key[upper] === undefined) {
+          // Best-effort hint: include 4 nearby key names from the enum
+          // so the agent can self-correct on the next attempt.
+          const nearby = Object.keys(Key)
+            .filter(n => n.toLowerCase().includes(upper.toLowerCase()) || upper.toLowerCase().includes(n.toLowerCase()))
+            .slice(0, 4);
+          const hint = nearby.length ? ` (did you mean: ${nearby.join(', ')}?)` : '';
+          throw makeError(OSError.INVALID_INPUT, `unknown key "${k}"${hint}`);
+        }
+        return Key[upper];
+      });
+      // pressKey takes the chord as varargs; nut.js holds modifiers down,
+      // presses the last key, releases all.
+      await keyboard.pressKey(...resolved);
+      await keyboard.releaseKey(...resolved);
+    } catch (err) {
+      throw wrapOpError('keyPress', err);
+    }
+  }
+
+  // ─── window management (best-effort, OS-dependent) ────────────────
+
+  /**
+   * List currently-open windows.
+   * @returns {Promise<Array<{title:string, pid?:number, bounds?:object}>>}
+   */
+  async listWindows() {
+    await this._ensureReady();
+    this._requireCapability('windowEnumeration');
+    try {
+      const { getWindows } = this._nutImpl;
+      const winRefs = await getWindows();
+      return Promise.all(winRefs.map(async w => ({
+        title: await w.getTitle(),
+        // bounds + pid are best-effort; nut.js varies by OS.
+        bounds: await safeCall(() => w.getRegion()),
+      })));
+    } catch (err) {
+      throw wrapOpError('listWindows', err);
+    }
+  }
+
+  /**
+   * Focus a window matched by title regex or substring.
+   * @param {Object} criteria
+   * @param {RegExp|string} [criteria.titleMatch]
+   */
+  async focusWindow(criteria) {
+    await this._ensureReady();
+    this._requireCapability('windowFocus');
+    if (!criteria || !criteria.titleMatch) {
+      throw makeError(OSError.INVALID_INPUT, 'criteria.titleMatch is required');
+    }
+    try {
+      const { getWindows } = this._nutImpl;
+      const wins = await getWindows();
+      for (const w of wins) {
+        const title = await w.getTitle();
+        if (matchesCriteria(title, criteria.titleMatch)) {
+          await w.focus();
+          return { focused: true, title };
+        }
+      }
+      return { focused: false, title: null };
+    } catch (err) {
+      throw wrapOpError('focusWindow', err);
+    }
+  }
+
+  // ─── internal ─────────────────────────────────────────────────────
+
+  /**
+   * Refuse the op if the OS doesn't support it (e.g. Wayland mouseInput).
+   * Provides the operator-facing remediation string from osCapabilities.
+   */
+  _requireCapability(name) {
+    const ok = this._caps?.[name];
+    if (!ok) {
+      throw makeError(OSError.UNSUPPORTED,
+        `${name} not available on this OS/session. ` +
+        (this._caps?.degradedReason || 'Run on a supported configuration.'));
+    }
+  }
+}
+
+// ─── helpers (module-private) ───────────────────────────────────────
+
+function validateCoord(x, y) {
+  if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0) {
+    throw makeError(OSError.INVALID_INPUT,
+      `coordinates must be non-negative finite numbers; got (${x}, ${y})`);
+  }
+}
+
+function validateRegion(r) {
+  if (!r || !Number.isFinite(r.x) || !Number.isFinite(r.y) ||
+      !Number.isFinite(r.width) || !Number.isFinite(r.height) ||
+      r.x < 0 || r.y < 0 || r.width < 1 || r.height < 1) {
+    throw makeError(OSError.INVALID_INPUT, `invalid region ${JSON.stringify(r)}`);
+  }
+}
+
+function matchesCriteria(title, m) {
+  if (m instanceof RegExp) return m.test(title);
+  if (typeof m === 'string') return title.toLowerCase().includes(m.toLowerCase());
+  return false;
+}
+
+async function safeCall(fn) {
+  try { return await fn(); }
+  catch { return null; }
+}
+
+async function toPngBuffer(nutImage) {
+  // Forward-compat fast paths if a future nut.js ships a PNG encoder.
+  if (typeof nutImage.toPng === 'function')    return nutImage.toPng();
+  if (typeof nutImage.toBuffer === 'function') return nutImage.toBuffer();
+
+  // Current shape (nut-tree-fork@4.x): Image exposes raw pixel data in
+  // `nutImage.data` with `channels` (3 or 4), `colorMode` (0=BGR,1=RGB
+  // per nut.js convention), and dimensions. We encode to PNG with sharp
+  // which is already a dep (bench harness uses it).
+  if (nutImage.data && nutImage.width && nutImage.height) {
+    const sharp = (await import('sharp')).default;
+    const { width, height } = nutImage;
+    const channels = nutImage.channels || 4;
+    // nut.js's pixel buffer is BGR(A) on Windows by default; sharp wants
+    // RGB(A). Swap red/blue if colorMode signals BGR. Channel-4 with
+    // alpha is preserved.
+    const buf = Buffer.from(nutImage.data);
+    const isBgr = nutImage.colorMode === 0;
+    if (isBgr) swapRedBlueInPlace(buf, channels);
+    return sharp(buf, { raw: { width, height, channels } }).png().toBuffer();
+  }
+  throw makeError(OSError.OP_FAILED, 'nut.js Image has no recognisable pixel buffer');
+}
+
+function swapRedBlueInPlace(buf, channels) {
+  for (let i = 0; i < buf.length; i += channels) {
+    const r = buf[i];
+    buf[i]     = buf[i + 2];
+    buf[i + 2] = r;
+  }
+}
+
+function wrapOpError(opName, err) {
+  // Detect permission-denied flavours from OS error messages.
+  const msg = (err.message || String(err)).toLowerCase();
+  if (msg.includes('not authorized') || msg.includes('permission')) {
+    return makeError(OSError.PERMISSION_DENIED,
+      `${opName} failed: ${err.message}. Check OS permission grants.`);
+  }
+  // Already-typed errors (validation, unsupported, native-missing) pass through;
+  // we only wrap raw native errors.
+  if (err.code === OSError.NATIVE_UNAVAILABLE ||
+      err.code === OSError.UNSUPPORTED        ||
+      err.code === OSError.INVALID_INPUT      ||
+      err.code === OSError.PERMISSION_DENIED) {
+    return err;
+  }
+  return makeError(OSError.OP_FAILED, `${opName} failed: ${err.message}`, { cause: err });
+}