@moxxy/plugin-browser 0.27.0

package/src/browser-surface.test.ts

@@ -0,0 +1,367 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { PassThrough } from 'node:stream';
+import { buildBrowserSurface } from './browser-surface.js';
+import { closeBrowserSidecar, type SidecarStream } from './browser-session.js';
+
+/**
+ * The browser surface polls the shared sidecar's `frame` method a few times a
+ * second and forwards each JPEG as a `surface.data` payload. These tests drive
+ * a FAKE sidecar (injected via deps.spawnFn) whose per-method behaviour the
+ * test controls, plus fake timers to step the poll interval — so the
+ * inFlight guard, the FAIL_GRACE status logic, snapshot catch-up, and the
+ * input coordinate de-normalization are exercised without Playwright.
+ */
+
+const FRAME_INTERVAL_MS = 300;
+const FAIL_GRACE = 6;
+
+type Reply = { ok: true; result: unknown } | { ok: false; message: string; kind?: string };
+
+interface FakeSidecar {
+  spawn: (path: string) => SidecarStream;
+  /** Set the handler invoked for each incoming method. */
+  setHandler: (fn: (method: string, params: unknown) => Reply) => void;
+  received: Array<{ method: string; params: unknown }>;
+}
+
+function makeControllableSpawn(): FakeSidecar {
+  const received: Array<{ method: string; params: unknown }> = [];
+  let handler: (method: string, params: unknown) => Reply = () => ({ ok: false, message: 'no handler' });
+  const spawn = (_path: string): SidecarStream => {
+    const stdin = new PassThrough();
+    const stdout = new PassThrough();
+    let buf = '';
+    stdin.on('data', (chunk) => {
+      buf += chunk.toString('utf8');
+      let nl: number;
+      while ((nl = buf.indexOf('\n')) !== -1) {
+        const line = buf.slice(0, nl);
+        buf = buf.slice(nl + 1);
+        if (!line.trim()) continue;
+        const reqMsg = JSON.parse(line) as { id: string; method: string; params: unknown };
+        received.push({ method: reqMsg.method, params: reqMsg.params });
+        const r = handler(reqMsg.method, reqMsg.params);
+        const reply = r.ok
+          ? { id: reqMsg.id, ok: true, result: r.result }
+          : { id: reqMsg.id, ok: false, error: { message: r.message, kind: r.kind } };
+        stdout.write(JSON.stringify(reply) + '\n');
+      }
+    });
+    const exitListeners: Array<(code: number | null) => void> = [];
+    return {
+      stdin,
+      stdout,
+      kill: () => {
+        for (const l of exitListeners) l(0);
+        return true;
+      },
+      once: (_event, listener) => {
+        exitListeners.push(listener as (code: number | null) => void);
+      },
+    };
+  };
+  return { spawn, setHandler: (fn) => (handler = fn), received };
+}
+
+const frame = (over: Partial<{ base64: string; mediaType: string; url: string; width: number; height: number }> = {}) => ({
+  base64: 'AAAA',
+  mediaType: 'image/jpeg',
+  url: 'https://example.com/',
+  width: 1000,
+  height: 500,
+  ...over,
+});
+
+/** Flush microtasks so the async tick() round-trips through the fake streams. */
+const flush = async (): Promise<void> => {
+  for (let i = 0; i < 6; i++) await Promise.resolve();
+};
+
+afterEach(async () => {
+  vi.useRealTimers();
+  await closeBrowserSidecar();
+});
+
+describe('browser surface polling lifecycle', () => {
+  it('emits a frame payload and tracks it as the snapshot', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler((method) => (method === 'frame' ? { ok: true, result: frame() } : { ok: false, message: 'x' }));
+
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    const payloads: unknown[] = [];
+    surface.onData((p) => payloads.push(p));
+
+    await flush(); // the immediate kick tick()
+    expect(payloads).toEqual([{ type: 'frame', base64: 'AAAA', mime: 'image/jpeg', url: 'https://example.com/' }]);
+    expect(surface.snapshot()).toEqual({ type: 'frame', base64: 'AAAA', mime: 'image/jpeg', url: 'https://example.com/' });
+
+    surface.close();
+  });
+
+  it('snapshot reports "Starting browser…" before any frame arrives', () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler(() => ({ ok: false, message: 'not yet' }));
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    expect(surface.snapshot()).toEqual({ type: 'status', text: 'Starting browser…' });
+    surface.close();
+  });
+
+  it('does NOT emit a status on the first failures, but DOES on the FAIL_GRACE-th (no prior frame)', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler(() => ({ ok: false, message: 'launch failed' }));
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    const statuses: Array<{ type: string; text?: string }> = [];
+    surface.onData((p) => statuses.push(p as { type: string; text?: string }));
+
+    await flush(); // tick #1 (the kick)
+    expect(statuses).toHaveLength(0);
+    // Advance to the FAIL_GRACE-th failure. Each interval triggers one tick.
+    for (let i = 1; i < FAIL_GRACE; i++) {
+      await vi.advanceTimersByTimeAsync(FRAME_INTERVAL_MS);
+      await flush();
+    }
+    expect(statuses).toEqual([{ type: 'status', text: 'Browser unavailable: launch failed' }]);
+
+    // Further failures do NOT re-emit (status fires once per failure streak).
+    await vi.advanceTimersByTimeAsync(FRAME_INTERVAL_MS);
+    await flush();
+    expect(statuses).toHaveLength(1);
+
+    surface.close();
+  });
+
+  it('a successful frame resets the failure counter', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    let ok = false;
+    sidecar.setHandler(() => (ok ? { ok: true, result: frame() } : { ok: false, message: 'flaky' }));
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    const events: Array<{ type: string }> = [];
+    surface.onData((p) => events.push(p as { type: string }));
+
+    await flush(); // fail #1
+    await vi.advanceTimersByTimeAsync(FRAME_INTERVAL_MS); // fail #2
+    await flush();
+    ok = true; // a good frame lands, resetting fails
+    await vi.advanceTimersByTimeAsync(FRAME_INTERVAL_MS);
+    await flush();
+    ok = false; // start failing again
+
+    // Because the good frame reset `fails` to 0, we must accrue FAIL_GRACE
+    // fresh failures via FAIL_GRACE more ticks before the status fires.
+    for (let i = 0; i < FAIL_GRACE; i++) {
+      await vi.advanceTimersByTimeAsync(FRAME_INTERVAL_MS);
+      await flush();
+    }
+    // The single status here is the "disconnected" one (a prior frame existed).
+    const statuses = events.filter((e) => e.type === 'status');
+    expect(statuses).toHaveLength(1);
+    expect((statuses[0] as { text: string }).text).toMatch(/disconnected/);
+
+    surface.close();
+  });
+
+  it('surfaces a "disconnected" status when the page dies after a frame was shown', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    let alive = true;
+    sidecar.setHandler(() => (alive ? { ok: true, result: frame() } : { ok: false, message: 'page crashed' }));
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    const events: Array<{ type: string; text?: string }> = [];
+    surface.onData((p) => events.push(p as { type: string; text?: string }));
+
+    await flush(); // good frame
+    alive = false;
+    for (let i = 0; i < FAIL_GRACE; i++) {
+      await vi.advanceTimersByTimeAsync(FRAME_INTERVAL_MS);
+      await flush();
+    }
+    const statuses = events.filter((e) => e.type === 'status');
+    expect(statuses).toEqual([{ type: 'status', text: 'Browser disconnected: page crashed' }]);
+    // The stale frame is still available via snapshot.
+    expect(surface.snapshot()).toMatchObject({ type: 'frame' });
+
+    surface.close();
+  });
+
+  it('a needs-install error pauses polling and emits an install prompt', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler(() => ({ ok: false, message: 'Playwright is not installed.', kind: 'needs-install' }));
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    const events: Array<{ type: string; needsInstall?: boolean; text?: string }> = [];
+    surface.onData((p) => events.push(p as { type: string; needsInstall?: boolean }));
+
+    await flush(); // the kick tick() hits the needs-install reply immediately
+    expect(events).toEqual([
+      expect.objectContaining({ type: 'status', needsInstall: true }),
+    ]);
+    // Snapshot for a late viewer also reports the install affordance.
+    expect(surface.snapshot()).toMatchObject({ type: 'status', needsInstall: true });
+
+    // Polling is paused: no further `frame` calls reach the sidecar (we don't
+    // spin on an import that will keep failing).
+    const before = sidecar.received.length;
+    await vi.advanceTimersByTimeAsync(FRAME_INTERVAL_MS * 4);
+    await flush();
+    expect(sidecar.received.length).toBe(before);
+
+    surface.close();
+  });
+
+  it('treats a malformed frame reply as a failed tick (no blank frame emitted)', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    // The sidecar replies ok, but with a frame missing base64/url — a partial or
+    // altered reply. The surface must NOT stream {base64: undefined} to subs.
+    sidecar.setHandler((method) =>
+      method === 'frame' ? { ok: true, result: { mediaType: 'image/jpeg' } } : { ok: false, message: 'x' },
+    );
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    const events: Array<{ type: string }> = [];
+    surface.onData((p) => events.push(p as { type: string }));
+
+    await flush();
+    // No frame payload streamed (it was treated as a failure).
+    expect(events.filter((e) => e.type === 'frame')).toHaveLength(0);
+    // Drive past FAIL_GRACE: a status surfaces instead of silent blanks.
+    for (let i = 1; i < FAIL_GRACE; i++) {
+      await vi.advanceTimersByTimeAsync(FRAME_INTERVAL_MS);
+      await flush();
+    }
+    expect(events.some((e) => e.type === 'status')).toBe(true);
+
+    surface.close();
+  });
+
+  it('close() stops the poll — no further frame calls reach the sidecar', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler(() => ({ ok: true, result: frame() }));
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    await flush();
+    const before = sidecar.received.length;
+    surface.close();
+    await vi.advanceTimersByTimeAsync(FRAME_INTERVAL_MS * 4);
+    await flush();
+    expect(sidecar.received.length).toBe(before);
+  });
+});
+
+describe('browser surface input mapping', () => {
+  it('click maps normalized fx/fy to viewport pixel coords from the last frame', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler((method) =>
+      method === 'frame' ? { ok: true, result: frame({ width: 1000, height: 500 }) } : { ok: true, result: { url: 'x' } },
+    );
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    await flush(); // establish last frame (1000x500)
+
+    await surface.input({ type: 'click', fx: 0.5, fy: 0.2 });
+    await flush();
+    const mouseCall = sidecar.received.find((r) => r.method === 'mouse');
+    expect(mouseCall?.params).toEqual({ x: 500, y: 100, count: 1 });
+
+    surface.close();
+  });
+
+  it('click before any frame uses the 1280x720 fallback viewport', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    // Frame fails so `last` stays null, but mouse succeeds.
+    sidecar.setHandler((method) => (method === 'mouse' ? { ok: true, result: { url: 'x' } } : { ok: false, message: 'no frame' }));
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    await flush();
+
+    await surface.input({ type: 'click', fx: 1, fy: 1 });
+    await flush();
+    const mouseCall = sidecar.received.find((r) => r.method === 'mouse');
+    expect(mouseCall?.params).toEqual({ x: 1280, y: 720, count: 1 });
+
+    surface.close();
+  });
+
+  it('resize sets the page viewport so the view fills the pane', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler(() => ({ ok: true, result: frame() }));
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    await flush();
+    await surface.resize?.({ width: 900, height: 600 });
+    await flush();
+    const vp = sidecar.received.find((r) => r.method === 'setviewport');
+    expect(vp?.params).toEqual({ width: 900, height: 600 });
+    surface.close();
+  });
+
+  it('dblclick forwards clickCount 2; hover + nav reach their sidecar methods', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler((method) => (method === 'frame' ? { ok: true, result: frame({ width: 1000, height: 500 }) } : { ok: true, result: {} }));
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    await flush();
+
+    await surface.input({ type: 'dblclick', fx: 0.5, fy: 0.5 });
+    await surface.input({ type: 'move', fx: 0.1, fy: 0.2 });
+    await surface.input({ type: 'reload' });
+    await flush();
+
+    expect(sidecar.received.find((r) => r.method === 'mouse')?.params).toEqual({ x: 500, y: 250, count: 2 });
+    expect(sidecar.received.find((r) => r.method === 'mousemove')?.params).toEqual({ x: 100, y: 100 });
+    expect(sidecar.received.some((r) => r.method === 'reload')).toBe(true);
+    surface.close();
+  });
+
+  it('zoom forwards the factor; capture maps the region and emits the PNG', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler((method) => {
+      if (method === 'frame') return { ok: true, result: frame({ width: 1000, height: 500 }) };
+      if (method === 'capture') return { ok: true, result: { mediaType: 'image/png', base64: 'PNGDATA' } };
+      return { ok: true, result: {} };
+    });
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    const events: Array<{ type: string; base64?: string }> = [];
+    surface.onData((p) => events.push(p as { type: string }));
+    await flush();
+
+    await surface.input({ type: 'zoom', factor: 1.5 });
+    await surface.input({ type: 'capture', fx: 0.1, fy: 0.2, fw: 0.5, fh: 0.4 });
+    await flush();
+
+    expect(sidecar.received.find((r) => r.method === 'zoom')?.params).toEqual({ factor: 1.5 });
+    // normalized region × viewport (1000×500) → CSS px clip.
+    expect(sidecar.received.find((r) => r.method === 'capture')?.params).toEqual({
+      x: 100,
+      y: 100,
+      width: 500,
+      height: 200,
+    });
+    expect(events.find((e) => e.type === 'captured')?.base64).toBe('PNGDATA');
+    surface.close();
+  });
+
+  it('navigate proxies goto and surfaces a goto rejection as a status line', async () => {
+    vi.useFakeTimers();
+    const sidecar = makeControllableSpawn();
+    sidecar.setHandler((method) => {
+      if (method === 'goto') return { ok: false, message: 'SSRF blocked' };
+      return { ok: false, message: 'no frame' };
+    });
+    const surface = buildBrowserSurface({ sidecarPath: '/fake.js', spawnFn: sidecar.spawn }).open();
+    const events: Array<{ type: string; text?: string }> = [];
+    surface.onData((p) => events.push(p as { type: string; text?: string }));
+    await flush();
+
+    await surface.input({ type: 'navigate', url: 'http://10.0.0.1/' });
+    await flush();
+    expect(events.some((e) => e.type === 'status' && e.text === 'SSRF blocked')).toBe(true);
+    expect(sidecar.received.some((r) => r.method === 'goto')).toBe(true);
+
+    surface.close();
+  });
+});

package/src/browser-surface.ts

@@ -0,0 +1,275 @@
+import { defineSurface, type SurfaceInstance } from '@moxxy/sdk';
+import {
+  browserSidecarCall,
+  closeBrowserSidecar,
+  resolveBrowserInstallRoot,
+  sidecarErrorKind,
+  type BrowserSessionDeps,
+} from './browser-session.js';
+import { installPlaywrightPackage } from './sidecar/install.js';
+
+/**
+ * The `browser` surface: a live, in-window view of the SAME Playwright page the
+ * `browser_session` tool drives. We "stream" the page by polling a JPEG frame
+ * (`frame` sidecar method) a few times a second and forwarding it as a
+ * `surface.data` payload; the user's clicks/keys/scroll/navigate are proxied
+ * back onto the page via the sidecar's coordinate-based input methods. So the
+ * agent and the user operate ONE shared page — agent navigations show up in the
+ * pane, and the user can take over.
+ *
+ * Why polling and not a CDP `Page.startScreencast` push: the screencast only
+ * emits on visual change, so a freshly-opened (blank / static / headless) page
+ * produces no frames at all — the pane sat on "Loading…" forever with the
+ * underlying error swallowed. A screenshot poll always yields a frame (even a
+ * blank one), so the view comes up reliably and a real launch/install failure
+ * surfaces as a status line instead of an indefinite spinner. Polling rides the
+ * existing sidecar RPC surface and works on every Playwright browser, not just
+ * Chromium.
+ */
+
+const FRAME_INTERVAL_MS = 300;
+/** Consecutive `frame` failures (with no frame ever seen) before we stop
+ *  assuming "still launching" and show the error. ~6 × 300ms ≈ 1.8s grace. */
+const FAIL_GRACE = 6;
+
+interface Frame {
+  mediaType: string;
+  base64: string;
+  url: string;
+  width: number;
+  height: number;
+}
+
+/** Validate a sidecar `frame` reply before emitting it: a partial / altered
+ *  reply must be treated as a failed tick (so FAIL_GRACE eventually surfaces a
+ *  status) rather than streamed to subscribers as `{base64: undefined, …}`. */
+function isFrame(v: unknown): v is Frame {
+  if (!v || typeof v !== 'object') return false;
+  const f = v as Record<string, unknown>;
+  return typeof f.base64 === 'string' && typeof f.mediaType === 'string' && typeof f.url === 'string';
+}
+
+export function buildBrowserSurface(deps?: BrowserSessionDeps) {
+  return defineSurface({
+    kind: 'browser',
+    description: "A live view of the agent's browser; click, type, and navigate.",
+    open: (): SurfaceInstance => {
+      const dataSubs = new Set<(payload: unknown) => void>();
+      let last: Frame | null = null;
+      let timer: ReturnType<typeof setInterval> | null = null;
+      let inFlight = false;
+      let fails = 0;
+      // Set once we detect the `playwright` npm package isn't installed. Polling
+      // pauses (no point retrying an import that will keep failing) and the pane
+      // shows an Install affordance; an `install` input clears it.
+      let needsInstall = false;
+      let installing = false;
+
+      const emit = (payload: unknown): void => {
+        for (const cb of dataSubs) cb(payload);
+      };
+
+      const stopPolling = (): void => {
+        if (timer) clearInterval(timer);
+        timer = null;
+      };
+      const startPolling = (): void => {
+        if (timer) return;
+        void tick();
+        timer = setInterval(() => void tick(), FRAME_INTERVAL_MS);
+      };
+
+      const tick = async (): Promise<void> => {
+        if (inFlight || needsInstall || installing) return; // don't pile up / retry a known-missing dep
+        inFlight = true;
+        try {
+          const reply = await browserSidecarCall('frame', {}, deps);
+          if (!isFrame(reply)) throw new Error('malformed frame reply from sidecar');
+          last = reply;
+          fails = 0;
+          emit({ type: 'frame', base64: reply.base64, mime: reply.mediaType, url: reply.url });
+        } catch (err) {
+          // The `playwright` npm package is simply absent — recoverable. Pause
+          // polling and ask the user (the download is ~200MB) rather than spin on
+          // a failing import or dump a raw "not installed" error.
+          if (sidecarErrorKind(err) === 'needs-install') {
+            needsInstall = true;
+            stopPolling();
+            emit({
+              type: 'status',
+              needsInstall: true,
+              text: 'The browser engine (Playwright) is not installed. It is a one-time ~200MB download.',
+            });
+            return;
+          }
+          // The first failures are usually the browser still launching (or a
+          // one-time binary install). Only surface a hard error once it's
+          // clearly not transient, so the user isn't left on a silent spinner.
+          // Fire EXACTLY on the FAIL_GRACE-th consecutive failure (a successful
+          // frame resets `fails` to 0), so the status is emitted once per
+          // failure streak rather than every tick. We surface it whether or not
+          // a prior frame exists: with no frame yet the page never came up;
+          // with a prior frame the page later died and the pane would otherwise
+          // sit frozen on a stale screenshot with no hint the live view is dead.
+          if (++fails === FAIL_GRACE) {
+            const message = err instanceof Error ? err.message : String(err);
+            emit({
+              type: 'status',
+              text: last ? `Browser disconnected: ${message}` : `Browser unavailable: ${message}`,
+            });
+          }
+        } finally {
+          inFlight = false;
+        }
+      };
+
+      // After an interaction, grab a frame now and once more shortly after, so a
+      // click/keypress that kicks off async work (a navigation, a menu opening,
+      // an animation) shows up promptly instead of waiting for the next poll.
+      const bump = (): void => {
+        void tick();
+        setTimeout(() => void tick(), 140);
+      };
+
+      // Kick an immediate frame (launches the browser), then poll.
+      startPolling();
+
+      const runInstall = async (): Promise<void> => {
+        if (installing) return;
+        installing = true;
+        stopPolling();
+        emit({ type: 'status', text: 'Installing browser engine… (one-time, ~200MB)' });
+        try {
+          await installPlaywrightPackage({
+            rootDir: resolveBrowserInstallRoot(deps),
+            onProgress: (line) => emit({ type: 'status', text: line }),
+          });
+          // The sidecar cached its failed `import('playwright')`; drop it so the
+          // next frame call respawns a fresh sidecar that imports the now-present
+          // package.
+          await closeBrowserSidecar();
+          needsInstall = false;
+          fails = 0;
+          emit({ type: 'status', text: 'Installed. Starting browser…' });
+          startPolling();
+        } catch (err) {
+          const message = err instanceof Error ? err.message : String(err);
+          emit({ type: 'status', needsInstall: true, text: `Install failed: ${message}` });
+        } finally {
+          installing = false;
+        }
+      };
+
+      return {
+        id: 'browser',
+        kind: 'browser',
+        onData: (cb) => {
+          dataSubs.add(cb);
+          return () => dataSubs.delete(cb);
+        },
+        snapshot: () =>
+          needsInstall
+            ? {
+                type: 'status',
+                needsInstall: true,
+                text: 'The browser engine (Playwright) is not installed. It is a one-time ~200MB download.',
+              }
+            : last
+              ? { type: 'frame', base64: last.base64, mime: last.mediaType, url: last.url }
+              : { type: 'status', text: 'Starting browser…' },
+        resize: async (size) => {
+          // Match the page viewport to the pane so the live view fills the
+          // container (no letterboxing) and click coords map 1:1.
+          if (!size.width || !size.height) return;
+          await browserSidecarCall('setviewport', { width: size.width, height: size.height }, deps).catch(
+            () => undefined,
+          );
+          void tick();
+        },
+        input: async (msg) => {
+          if (msg.type === 'install') {
+            await runInstall();
+            return;
+          }
+          const vw = last?.width ?? 1280;
+          const vh = last?.height ?? 720;
+          if (msg.type === 'navigate' && typeof msg.url === 'string') {
+            // The sidecar's goto re-runs the SSRF guard (loopback/private/
+            // metadata blocked), so a hostile URL never navigates. Unlike the
+            // pointer/key proxies below, a navigate rejection is user-meaningful
+            // (bad URL format, SSRF block) — surface it as a status line instead
+            // of swallowing it, so the user knows why the address bar did nothing.
+            try {
+              await browserSidecarCall('goto', { url: msg.url }, deps);
+            } catch (err) {
+              const text = err instanceof Error ? err.message : String(err);
+              emit({ type: 'status', text });
+            }
+            bump();
+          } else if (
+            (msg.type === 'click' || msg.type === 'dblclick') &&
+            typeof msg.fx === 'number' &&
+            typeof msg.fy === 'number'
+          ) {
+            const count = msg.type === 'dblclick' ? 2 : 1;
+            await browserSidecarCall('mouse', { x: msg.fx * vw, y: msg.fy * vh, count }, deps).catch(
+              () => undefined,
+            );
+            bump();
+          } else if (msg.type === 'move' && typeof msg.fx === 'number' && typeof msg.fy === 'number') {
+            // Hover — drive the pointer so :hover styles/tooltips render. A single
+            // follow-up frame (not a bump) keeps the cost down at move frequency.
+            await browserSidecarCall('mousemove', { x: msg.fx * vw, y: msg.fy * vh }, deps).catch(
+              () => undefined,
+            );
+            void tick();
+          } else if (
+            msg.type === 'capture' &&
+            typeof msg.fx === 'number' &&
+            typeof msg.fy === 'number' &&
+            typeof msg.fw === 'number' &&
+            typeof msg.fh === 'number'
+          ) {
+            // Sharp PNG of the dragged region → handed back to the pane, which
+            // attaches it to the chat composer (the user then describes the change).
+            const shot = (await browserSidecarCall(
+              'capture',
+              { x: msg.fx * vw, y: msg.fy * vh, width: msg.fw * vw, height: msg.fh * vh },
+              deps,
+            ).catch(() => null)) as unknown;
+            // Validate before emitting so a partial/altered reply doesn't hand
+            // the composer a {base64: undefined} attachment.
+            if (
+              shot &&
+              typeof shot === 'object' &&
+              typeof (shot as { base64?: unknown }).base64 === 'string' &&
+              typeof (shot as { mediaType?: unknown }).mediaType === 'string'
+            ) {
+              const s = shot as { base64: string; mediaType: string };
+              emit({ type: 'captured', base64: s.base64, mediaType: s.mediaType });
+            }
+          } else if (msg.type === 'zoom' && typeof msg.factor === 'number') {
+            await browserSidecarCall('zoom', { factor: msg.factor }, deps).catch(() => undefined);
+            bump();
+          } else if (msg.type === 'key' && typeof msg.key === 'string') {
+            await browserSidecarCall('key', { key: msg.key }, deps).catch(() => undefined);
+            bump();
+          } else if (msg.type === 'scroll' && typeof msg.dy === 'number') {
+            await browserSidecarCall('scroll', { dy: msg.dy }, deps).catch(() => undefined);
+            bump();
+          } else if (msg.type === 'back' || msg.type === 'forward' || msg.type === 'reload') {
+            await browserSidecarCall(msg.type, {}, deps).catch(() => undefined);
+            bump();
+          }
+        },
+        close: () => {
+          if (timer) clearInterval(timer);
+          timer = null;
+          dataSubs.clear();
+          // The underlying page stays alive for the agent's browser_session tool;
+          // it's torn down on session shutdown (closeBrowserSidecar).
+        },
+      };
+    },
+  });
+}