@hienlh/ppm 0.18.15 → 0.19.0

package/src/services/remote-desktop/remote-desktop-input.ts

@@ -1,217 +1,67 @@
 /**
- * Mouse/keyboard injection via Bun FFI straight into `user32.dll` `SendInput` — no separate
- * helper process (drops the Rust-binary build step + unsigned-exe/SmartScreen risk the phase
- * plan flagged for a bundled injector; phase 02's SYSTEM-session helper is a different
- * process/token anyway, so nothing here is meant to be reused as-is).
+ * Platform-neutral entry point for input injection. Picks the backend for `process.platform`
+ * from the registry below; the session and routes never import a platform module directly.
  *
- * Normal desktop only, current interactive session, no SYSTEM/UAC/lock-screen support.
- *
- * `bun:ffi` is only ever `import()`-ed and `dlopen`-ed lazily from inside `loadUser32()`, so
- * merely importing this module (routing, types) on Linux/macOS test runners never touches
- * `user32.dll`.
+ * Every backend module is safe to *import* anywhere (FFI is loaded lazily inside each), so the
+ * registry can hold them all statically — only the selected one ever `dlopen`s. Adding Linux =
+ * `remote-desktop-input-linux.ts` + one line in `BACKENDS`.
  */
-import { codeToVk, MODIFIER_VK_CODES } from "./remote-desktop-vk-map.ts";
-
-export class RemoteInputUnavailableError extends Error {
-  constructor(msg = "Remote input injection is only available on Windows") {
-    super(msg);
-    this.name = "RemoteInputUnavailableError";
-  }
-}
+import { win32InputBackend } from "./remote-desktop-input-win32.ts";
+import { darwinInputBackend } from "./remote-desktop-input-darwin.ts";
+import { RemoteInputUnavailableError, type InputTargetRect, type RemoteInputBackend } from "./remote-desktop-input-backend.ts";
 
-const INPUT_MOUSE = 0;
-const INPUT_KEYBOARD = 1;
-const MOUSEEVENTF_MOVE = 0x0001;
-const MOUSEEVENTF_LEFTDOWN = 0x0002;
-const MOUSEEVENTF_LEFTUP = 0x0004;
-const MOUSEEVENTF_RIGHTDOWN = 0x0008;
-const MOUSEEVENTF_RIGHTUP = 0x0010;
-const MOUSEEVENTF_ABSOLUTE = 0x8000;
-const MOUSEEVENTF_VIRTUALDESK = 0x4000;
-const MOUSEEVENTF_WHEEL = 0x0800;
-/** One notch of a real mouse wheel — `mouseData` for `MOUSEEVENTF_WHEEL` is a signed multiple
- *  of this constant, not a raw pixel delta. */
-const WHEEL_DELTA = 120;
-const KEYEVENTF_KEYUP = 0x0002;
-/** `((DPI_AWARENESS_CONTEXT)-4)` — PER_MONITOR_AWARE_V2. Not a real pointer; the x64 ABI
- *  passes it in the same register/slot as a pointer arg would, bit pattern only. */
-const DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = -4n;
-/** Each `INPUT` struct is 40 bytes on x64 (8-byte `type` slot + 32-byte union). */
-const INPUT_STRUCT_SIZE = 40;
-/** Desktop access rights for input injection. `SendInput` silently no-ops when the calling
- *  thread is not attached to the *input* desktop; attaching needs the READ/WRITE/SWITCH rights
- *  plus JOURNALPLAYBACK so the desktop accepts injected input. These MUST be the real DESKTOP_*
- *  bit values — an invalid mask (e.g. an out-of-range bit) makes `OpenInputDesktop` return NULL
- *  and the attach silently fails, leaving SendInput pointed at the wrong desktop. */
-const DESKTOP_READOBJECTS = 0x0001;
-const DESKTOP_JOURNALPLAYBACK = 0x0020;
-const DESKTOP_WRITEOBJECTS = 0x0080;
-const DESKTOP_SWITCHDESKTOP = 0x0100;
-const INPUT_DESKTOP_ACCESS =
-  DESKTOP_READOBJECTS | DESKTOP_JOURNALPLAYBACK | DESKTOP_WRITEOBJECTS | DESKTOP_SWITCHDESKTOP; // 0x1A1
+export { RemoteInputUnavailableError, type InputTargetRect, type RemoteInputBackend };
 
-/** WINSTA_ALL_ACCESS — enough to open the interactive window station and make it this process's
- *  station so its input desktop becomes reachable. */
-const WINSTA_ALL_ACCESS = 0x37f;
-
-type FfiModule = typeof import("bun:ffi");
-type User32Symbols = {
-  SendInput: (count: number, ptr: number | bigint, size: number) => number;
-  SetProcessDpiAwarenessContext: (ctx: bigint) => boolean;
-  OpenInputDesktop: (flags: number, inherit: boolean, access: number) => number | bigint;
-  SetThreadDesktop: (hdesk: number | bigint) => boolean;
-  CloseDesktop: (hdesk: number | bigint) => boolean;
-  OpenWindowStationW: (name: number | bigint, inherit: boolean, access: number) => number | bigint;
-  SetProcessWindowStation: (hwinsta: number | bigint) => boolean;
+const BACKENDS: Partial<Record<NodeJS.Platform, RemoteInputBackend>> = {
+  win32: win32InputBackend,
+  darwin: darwinInputBackend,
 };
 
-let ffiModule: FfiModule | null = null;
-let user32: User32Symbols | null = null;
-let dpiAwarenessAttempted = false;
-
-async function loadUser32(): Promise<{ ffi: FfiModule; lib: User32Symbols }> {
-  if (process.platform !== "win32") throw new RemoteInputUnavailableError();
-  if (!ffiModule) ffiModule = await import("bun:ffi");
-  const { dlopen, FFIType } = ffiModule;
-  if (!user32) {
-    user32 = dlopen("user32.dll", {
-      SendInput: { args: [FFIType.u32, FFIType.ptr, FFIType.i32], returns: FFIType.u32 },
-      SetProcessDpiAwarenessContext: { args: [FFIType.i64], returns: FFIType.bool },
-      OpenInputDesktop: { args: [FFIType.u32, FFIType.bool, FFIType.u32], returns: FFIType.ptr },
-      SetThreadDesktop: { args: [FFIType.ptr], returns: FFIType.bool },
-      CloseDesktop: { args: [FFIType.ptr], returns: FFIType.bool },
-      OpenWindowStationW: { args: [FFIType.ptr, FFIType.bool, FFIType.u32], returns: FFIType.ptr },
-      SetProcessWindowStation: { args: [FFIType.ptr], returns: FFIType.bool },
-    }).symbols as unknown as User32Symbols;
-  }
-  if (!dpiAwarenessAttempted) {
-    dpiAwarenessAttempted = true;
-    // A second call to this in the same process returns E_ACCESSDENIED — swallow it, it just
-    // means something else (or a previous session) already set it.
-    try { user32.SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2); } catch { /* already set */ }
-  }
-  return { ffi: ffiModule, lib: user32 };
-}
-
-function writeMouseInput(dv: DataView, offset: number, xAbs: number, yAbs: number, flags: number, mouseData = 0): void {
-  dv.setUint32(offset, INPUT_MOUSE, true);
-  dv.setInt32(offset + 8, xAbs, true);
-  dv.setInt32(offset + 12, yAbs, true);
-  dv.setInt32(offset + 16, mouseData, true); // wheel delta for MOUSEEVENTF_WHEEL, unused otherwise
-  dv.setUint32(offset + 20, flags >>> 0, true);
-  dv.setUint32(offset + 24, 0, true); // time: 0 lets the system supply it
-}
-
-function writeKeyInput(dv: DataView, offset: number, vk: number, keyUp: boolean): void {
-  dv.setUint32(offset, INPUT_KEYBOARD, true);
-  dv.setUint16(offset + 8, vk, true);
-  dv.setUint16(offset + 10, 0, true); // wScan — unused, we inject by VK not scan code
-  dv.setUint32(offset + 12, keyUp ? KEYEVENTF_KEYUP : 0, true);
-  dv.setUint32(offset + 16, 0, true); // time
+/** The backend for this host, or null when the platform has none. */
+export function getInputBackend(platform: NodeJS.Platform = process.platform): RemoteInputBackend | null {
+  return BACKENDS[platform] ?? null;
 }
 
-/** Handle of the desktop this thread is currently attached to, so we can close the previous one
- *  when the active input desktop changes (lock/unlock/UAC switches it). One handle in flight. */
-let attachedDesktop: number | bigint | null = null;
-
-/** Attach the calling thread to the *current* input desktop before injecting. Re-run every send
- *  because the input desktop changes on lock/unlock; `CreateProcessAsUser`/process start only
- *  fixes the initial desktop, not later switches. Best-effort: on failure we still try SendInput
- *  (it just no-ops), and log once rather than per-event spam. */
-/** Attach the PROCESS to the interactive window station ("winsta0") once, so its input desktop
- *  is reachable. Without this, a PPM process launched from a non-interactive station (a service,
- *  or some spawners) can't OpenInputDesktop at all — SendInput then goes nowhere. No-op when the
- *  process is already on winsta0 (the normal case for PPM started inside the user's session). */
-let stationEnsured = false;
-function ensureInteractiveStation(ffi: FfiModule, lib: User32Symbols): void {
-  if (stationEnsured) return;
-  stationEnsured = true;
-  const name = "winsta0";
-  const wide = new Uint16Array(name.length + 1);
-  for (let i = 0; i < name.length; i++) wide[i] = name.charCodeAt(i);
-  const hwinsta = lib.OpenWindowStationW(ffi.ptr(wide), false, WINSTA_ALL_ACCESS);
-  if (!hwinsta) return; // already on it, or no access — OpenInputDesktop below will tell us
-  lib.SetProcessWindowStation(hwinsta);
-}
-
-function attachToInputDesktop(ffi: FfiModule, lib: User32Symbols): void {
-  ensureInteractiveStation(ffi, lib);
-  const hdesk = lib.OpenInputDesktop(0, false, INPUT_DESKTOP_ACCESS);
-  if (!hdesk) return; // couldn't open the input desktop (e.g. secure/lock desktop); leave thread as-is
-  const ok = lib.SetThreadDesktop(hdesk);
-  if (ok) {
-    if (attachedDesktop && attachedDesktop !== hdesk) {
-      try { lib.CloseDesktop(attachedDesktop); } catch { /* ignore */ }
-    }
-    attachedDesktop = hdesk;
-  } else {
-    // SetThreadDesktop fails if this thread owns windows/hooks — shouldn't for the server thread.
-    try { lib.CloseDesktop(hdesk); } catch { /* ignore */ }
-  }
+/** Sync, cheap: does this platform have an injector at all? Used on the per-event hot path to
+ *  drop input messages early. "Would it actually work" (OS permissions) is answered by
+ *  `remote-desktop-requirements.ts`. */
+export function isInputAvailable(): boolean {
+  return getInputBackend() !== null;
 }
 
-async function sendRaw(buf: Uint8Array, count: number): Promise<void> {
-  const { ffi, lib } = await loadUser32();
-  attachToInputDesktop(ffi, lib);
-  const sent = lib.SendInput(count, ffi.ptr(buf), INPUT_STRUCT_SIZE);
-  if (sent !== count) console.warn(`[remote-desktop] SendInput accepted ${sent}/${count} events`);
+function required(): RemoteInputBackend {
+  const backend = getInputBackend();
+  if (!backend) throw new RemoteInputUnavailableError();
+  return backend;
 }
 
-/** Move the cursor and, if this is a click, press/release the button — one absolute move
- *  per call so the button acts at the position the client actually clicked. `xFrac`/`yFrac`
- *  are 0..1 fractions of the capture (client already dropped devicePixelRatio; only the
- *  canvas-relative fraction is meaningful for host coordinates). */
-export async function injectPointer(
-  xFrac: number,
-  yFrac: number,
-  button: "left" | "right" | null,
-  down: boolean | null,
+export function injectPointer(
+  xFrac: number, yFrac: number, button: "left" | "right" | null, down: boolean | null, target: InputTargetRect | null = null,
 ): Promise<void> {
-  const xAbs = Math.round(Math.min(Math.max(xFrac, 0), 1) * 65535);
-  const yAbs = Math.round(Math.min(Math.max(yFrac, 0), 1) * 65535);
-  let flags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK;
-  if (button === "left") flags |= down ? MOUSEEVENTF_LEFTDOWN : MOUSEEVENTF_LEFTUP;
-  else if (button === "right") flags |= down ? MOUSEEVENTF_RIGHTDOWN : MOUSEEVENTF_RIGHTUP;
+  return required().pointer(xFrac, yFrac, button, down, target);
+}
 
-  const buf = new Uint8Array(INPUT_STRUCT_SIZE);
-  writeMouseInput(new DataView(buf.buffer), 0, xAbs, yAbs, flags);
-  await sendRaw(buf, 1);
+export function injectWheel(deltaY: number): Promise<void> {
+  // A zero delta is a no-op on every OS — settle it here so no backend is even consulted.
+  if (Math.round(deltaY) === 0) return Promise.resolve();
+  return required().wheel(deltaY);
 }
 
-/** Scroll the wheel. `deltaY` is in `WHEEL_DELTA` (120) units — positive rotates the wheel
- *  forward (scrolls up/away from the user), negative rotates backward (scrolls down). The
- *  client accumulates raw two-finger-drag pixels into this unit before sending, so this layer
- *  stays a dumb relay: round + clamp to a safe i16 range so a runaway client value can't wrap
- *  into a huge or negative-looking mouseData field. */
-export async function injectWheel(deltaY: number): Promise<void> {
-  const clamped = Math.max(-32768, Math.min(32767, Math.round(deltaY)));
-  if (clamped === 0) return;
-  const buf = new Uint8Array(INPUT_STRUCT_SIZE);
-  writeMouseInput(new DataView(buf.buffer), 0, 0, 0, MOUSEEVENTF_WHEEL, clamped);
-  await sendRaw(buf, 1);
+export function injectKey(code: string, down: boolean): Promise<boolean> {
+  return required().key(code, down);
 }
 
-/** Send a keydown/keyup by physical key code. Returns false for an unmapped code (caller
- *  should just drop it rather than injecting a wrong key). */
-export async function injectKey(code: string, down: boolean): Promise<boolean> {
-  const vk = codeToVk(code);
-  if (vk === null) return false;
-  const buf = new Uint8Array(INPUT_STRUCT_SIZE);
-  writeKeyInput(new DataView(buf.buffer), 0, vk, !down);
-  await sendRaw(buf, 1);
+/** Type text as-is. Resolves false when this platform's backend has no text path (the caller
+ *  may fall back to per-key events or tell the user). */
+export async function injectText(text: string): Promise<boolean> {
+  const backend = required();
+  if (!backend.text) return false;
+  await backend.text(text);
   return true;
 }
 
-/** Force keyup for every modifier VK, regardless of tracked state — the backstop for a lost
- *  keyup (blur, WS drop) leaving Shift/Ctrl/Alt/Win logically held on the host. */
 export async function releaseAllModifiers(): Promise<void> {
-  if (process.platform !== "win32") return;
-  const buf = new Uint8Array(INPUT_STRUCT_SIZE * MODIFIER_VK_CODES.length);
-  const dv = new DataView(buf.buffer);
-  MODIFIER_VK_CODES.forEach((vk, i) => writeKeyInput(dv, i * INPUT_STRUCT_SIZE, vk, true));
-  try { await sendRaw(buf, MODIFIER_VK_CODES.length); } catch { /* best-effort */ }
-}
-
-export function isInputAvailable(): boolean {
-  return process.platform === "win32";
+  const backend = getInputBackend();
+  if (!backend) return;
+  await backend.releaseAllModifiers();
 }

package/src/services/remote-desktop/remote-desktop-macos-permissions.ts

@@ -0,0 +1,62 @@
+/**
+ * macOS TCC pre-flight for the two grants remote desktop needs, via Bun FFI. Both checks are
+ * cheap and side-effect free; the `request*` calls make macOS show its own prompt (once per
+ * grant — a denied prompt never re-appears, the user then has to use System Settings).
+ *
+ * Why pre-flight instead of reacting to failures: without Screen Recording, avfoundation still
+ * runs and hands ffmpeg **black frames**; without Accessibility, `CGEventPost` is a silent
+ * no-op. Neither path produces an error the session could surface.
+ *
+ * TCC keys command-line tools by executable — the grant lands on `bun` (the PPM daemon's
+ * binary), not on a terminal or on PPM. Frameworks are `dlopen`-ed lazily and only on darwin.
+ */
+const CG = "/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics";
+const APP_SERVICES = "/System/Library/Frameworks/ApplicationServices.framework/ApplicationServices";
+
+type Symbols = {
+  CGPreflightScreenCaptureAccess: () => boolean;
+  CGRequestScreenCaptureAccess: () => boolean;
+  AXIsProcessTrusted: () => boolean;
+};
+
+let loaded: { ffi: typeof import("bun:ffi"); lib: Symbols } | null = null;
+
+async function load() {
+  if (loaded) return loaded;
+  const ffi = await import("bun:ffi");
+  const { dlopen, FFIType: T } = ffi;
+  const cg = dlopen(CG, {
+    CGPreflightScreenCaptureAccess: { args: [], returns: T.bool },
+    CGRequestScreenCaptureAccess: { args: [], returns: T.bool },
+  }).symbols;
+  const ax = dlopen(APP_SERVICES, { AXIsProcessTrusted: { args: [], returns: T.bool } }).symbols;
+  loaded = { ffi, lib: { ...cg, ...ax } as unknown as Symbols };
+  return loaded;
+}
+
+export type MacPermissionId = "screen-recording" | "accessibility";
+
+/** System Settings deep links (macOS 13+ pane ids). */
+export const MAC_PERMISSION_SETTINGS_URL: Record<MacPermissionId, string> = {
+  "screen-recording": "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture",
+  accessibility: "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility",
+};
+
+/** Current grant state. Off darwin both are reported as granted so callers need no OS branch. */
+export async function macPermissionStatus(): Promise<Record<MacPermissionId, boolean>> {
+  if (process.platform !== "darwin") return { "screen-recording": true, accessibility: true };
+  const { lib } = await load();
+  return { "screen-recording": lib.CGPreflightScreenCaptureAccess(), accessibility: lib.AXIsProcessTrusted() };
+}
+
+/** Ask macOS to show the grant prompt. Returns the state *after* the call — for Accessibility
+ *  the prompt is asynchronous, so the caller should keep polling `macPermissionStatus()`. */
+export async function requestMacPermission(id: MacPermissionId): Promise<boolean> {
+  if (process.platform !== "darwin") return true;
+  const { lib } = await load();
+  if (id === "screen-recording") return lib.CGRequestScreenCaptureAccess();
+  // Accessibility has no prompt API without a CFDictionary (`AXIsProcessTrustedWithOptions` +
+  // the `kAXTrustedCheckOptionPrompt` data symbol, which bun:ffi cannot dlsym). The panel
+  // deep-links to the Settings pane instead, where the user adds the process by hand.
+  return lib.AXIsProcessTrusted();
+}

package/src/services/remote-desktop/remote-desktop-requirements.ts

@@ -0,0 +1,124 @@
+/**
+ * What this host still needs before remote desktop works, as a flat checklist the UI renders
+ * generically. Each platform contributes its own items; the client never branches on OS — it
+ * only knows `gates` (which half of the feature an item blocks) and the `actions` it can offer.
+ *
+ * Adding a requirement = push one more item here (and, for `host` actions, handle its id in
+ * `runHostAction`). The UI needs no change.
+ */
+import { getFfmpegCapabilities } from "../media-transcode/ffmpeg-capabilities.ts";
+import { captureInputForPlatform } from "./remote-desktop-capture-input.ts";
+import { getInputBackend } from "./remote-desktop-input.ts";
+import {
+  MAC_PERMISSION_SETTINGS_URL,
+  macPermissionStatus,
+  requestMacPermission,
+  type MacPermissionId,
+} from "./remote-desktop-macos-permissions.ts";
+
+export type HostAction = "request" | "open-settings";
+
+export type RequirementAction =
+  /** Type `command` into a PPM terminal — that shell runs on the host, so this works from a phone. */
+  | { kind: "terminal"; label: string; command: string }
+  /** A web page the *client* opens in its own browser (docs, downloads). */
+  | { kind: "link"; label: string; url: string }
+  /** Something only the host can do — show an OS permission prompt, open its Settings pane.
+   *  `POST /api/remote-desktop/requirements/:id/:action`. Never a client-side URL: a
+   *  `x-apple.systempreferences:` link means nothing on the phone driving the session. */
+  | { kind: "host"; label: string; action: HostAction };
+
+export interface RemoteDesktopRequirement {
+  id: string;
+  ok: boolean;
+  /** Which part of the feature this blocks: no video = nothing to show; no input = view-only. */
+  gates: "video" | "input";
+  title: string;
+  /** One sentence: what it is for and what to do. Shown only while `ok` is false. */
+  detail: string;
+  actions: RequirementAction[];
+}
+
+export interface RemoteDesktopReadiness {
+  platform: NodeJS.Platform;
+  /** There is a capture path for this OS at all. Gates the UI entry; everything else is a
+   *  requirement the user can satisfy in place. */
+  platformSupported: boolean;
+  requirements: RemoteDesktopRequirement[];
+  /** Every `video` requirement is met. */
+  videoReady: boolean;
+  /** Every `input` requirement is met (and the platform has an injector). */
+  inputReady: boolean;
+}
+
+const FFMPEG_INSTALL: Partial<Record<NodeJS.Platform, RequirementAction>> = {
+  darwin: { kind: "terminal", label: "Install with Homebrew", command: "brew install ffmpeg" },
+  win32: { kind: "terminal", label: "Install with winget", command: "winget install --id Gyan.FFmpeg -e" },
+  linux: { kind: "terminal", label: "Install with apt", command: "sudo apt install ffmpeg" },
+};
+
+function ffmpegRequirement(platform: NodeJS.Platform, present: boolean): RemoteDesktopRequirement {
+  const install = FFMPEG_INSTALL[platform];
+  return {
+    id: "ffmpeg",
+    ok: present,
+    gates: "video",
+    title: "ffmpeg",
+    detail: "Captures and encodes the screen. Install it, then come back — PPM re-checks automatically.",
+    actions: [
+      ...(install ? [install] : []),
+      { kind: "link", label: "Download page", url: "https://ffmpeg.org/download.html" },
+    ],
+  };
+}
+
+function macPermissionRequirement(id: MacPermissionId, granted: boolean): RemoteDesktopRequirement {
+  const screen = id === "screen-recording";
+  return {
+    id,
+    ok: granted,
+    gates: screen ? "video" : "input",
+    title: screen ? "Screen Recording permission" : "Accessibility permission",
+    detail: screen
+      ? "Without it macOS hands PPM a black screen. Allow the PPM process (bun) under Screen & System Audio Recording."
+      : "Needed to move the mouse and type. Add the PPM process (bun) under Accessibility; until then the view is read-only.",
+    actions: [
+      ...(screen ? [{ kind: "host", label: "Ask macOS now", action: "request" } as RequirementAction] : []),
+      { kind: "host", label: "Open System Settings on the host", action: "open-settings" },
+    ],
+  };
+}
+
+export async function remoteDesktopReadiness(platform: NodeJS.Platform = process.platform): Promise<RemoteDesktopReadiness> {
+  const platformSupported = captureInputForPlatform(platform) !== null;
+  const requirements: RemoteDesktopRequirement[] = [];
+  if (platformSupported) {
+    const caps = await getFfmpegCapabilities();
+    requirements.push(ffmpegRequirement(platform, !!caps.ffmpeg));
+    if (platform === "darwin") {
+      const status = await macPermissionStatus();
+      requirements.push(macPermissionRequirement("screen-recording", status["screen-recording"]));
+      requirements.push(macPermissionRequirement("accessibility", status.accessibility));
+    }
+  }
+  const inputSupported = getInputBackend(platform) !== null;
+  return {
+    platform,
+    platformSupported,
+    requirements,
+    videoReady: platformSupported && requirements.filter((r) => r.gates === "video").every((r) => r.ok),
+    inputReady: inputSupported && requirements.filter((r) => r.gates === "input").every((r) => r.ok),
+  };
+}
+
+/** Run a `host` action. Returns the requirement's `ok` afterwards, or null when the pair has
+ *  no host path (caller answers 404). `open-settings` returns the current state — the grant
+ *  lands later, the UI keeps polling. */
+export async function runHostAction(id: string, action: HostAction): Promise<boolean | null> {
+  if (id !== "screen-recording" && id !== "accessibility") return null;
+  if (action === "request") return requestMacPermission(id);
+  if (process.platform === "darwin") {
+    Bun.spawn(["open", MAC_PERMISSION_SETTINGS_URL[id]], { stdout: "ignore", stderr: "ignore" });
+  }
+  return (await macPermissionStatus())[id];
+}

package/src/services/remote-desktop/remote-desktop-session.ts

@@ -7,7 +7,8 @@
 import { startCapture, type CaptureHandle } from "./remote-desktop-capture.ts";
 import { avc1CodecString } from "./avc1-codec-string.ts";
 import type { AccessUnit } from "./access-unit-assembler.ts";
-import { injectPointer, injectKey, injectWheel, releaseAllModifiers, isInputAvailable } from "./remote-desktop-input.ts";
+import { injectPointer, injectKey, injectWheel, injectText, releaseAllModifiers, isInputAvailable } from "./remote-desktop-input.ts";
+import { resolveDisplay, type RemoteDisplay } from "./remote-desktop-displays.ts";
 
 /** Minimal socket surface this module needs — matches Bun's `ServerWebSocket` shape closely
  *  enough to be faked in a unit test without a real connection. */
@@ -28,6 +29,8 @@ const HEARTBEAT_TIMEOUT_MS = 30_000;
  *  queueing forever — a slow WAN/tunnel link must degrade to "waits ~2s for a keyframe",
  *  never to unbounded memory growth or an ever-growing latency queue. */
 const BACKPRESSURE_THRESHOLD_BYTES = 512 * 1024;
+/** Longest `text` message injected in one go — a paste, not a file; anything bigger is dropped. */
+const MAX_TEXT_CHARS = 1024;
 
 export class RemoteDesktopSession {
   private capture: CaptureHandle | null = null;
@@ -41,12 +44,13 @@ export class RemoteDesktopSession {
   /** Resolves once the underlying ffmpeg process has actually exited (not merely asked to). */
   readonly exited: Promise<void>;
 
-  constructor(private readonly ws: RemoteDesktopSocket) {
+  constructor(private readonly ws: RemoteDesktopSocket, private readonly display: RemoteDisplay | null) {
     this.exited = new Promise((resolve) => { this.exitResolve = resolve; });
   }
 
   async start(): Promise<void> {
     this.capture = await startCapture({
+      display: this.display,
       onAccessUnit: (au) => this.handleAccessUnit(au),
       // `reason` is only set when ffmpeg died on its own (crash, access denied, etc) — tell
       // the client *why* before closing instead of leaving it to guess from a bare
@@ -80,7 +84,7 @@ export class RemoteDesktopSession {
       const { xFrac, yFrac, button, down } = msg as { xFrac?: unknown; yFrac?: unknown; button?: unknown; down?: unknown };
       if (typeof xFrac === "number" && typeof yFrac === "number") {
         const btn = button === "left" || button === "right" ? button : null;
-        await injectPointer(xFrac, yFrac, btn, typeof down === "boolean" ? down : null);
+        await injectPointer(xFrac, yFrac, btn, typeof down === "boolean" ? down : null, this.display);
       }
       return;
     }
@@ -97,6 +101,13 @@ export class RemoteDesktopSession {
       }
       return;
     }
+    if (msg.type === "text") {
+      // Layout-independent text entry (mobile keyboards, non-ASCII). Backends without a text
+      // path report false and the message is dropped — the client keeps its per-key fallback.
+      const { text } = msg as { text?: unknown };
+      if (typeof text === "string" && text.length > 0 && text.length <= MAX_TEXT_CHARS) await injectText(text);
+      return;
+    }
     if (msg.type === "releaseAll") {
       await this.releaseHeldKeys();
     }
@@ -164,12 +175,12 @@ const activeSessions = new Set<RemoteDesktopSession>();
 
 /** Evict any previous session (awaiting its ffmpeg exit, capped so a wedged process can't
  *  hang a reconnect) before starting the new one. */
-export async function createRemoteDesktopSession(ws: RemoteDesktopSocket): Promise<RemoteDesktopSession> {
+export async function createRemoteDesktopSession(ws: RemoteDesktopSocket, displayId?: string): Promise<RemoteDesktopSession> {
   for (const existing of [...activeSessions]) {
     existing.close();
     await Promise.race([existing.exited, Bun.sleep(2000)]);
   }
-  const session = new RemoteDesktopSession(ws);
+  const session = new RemoteDesktopSession(ws, await resolveDisplay(displayId));
   await session.start();
   activeSessions.add(session);
   return session;