@moxxy/plugin-computer-control 0.41.2 → 0.42.0

package/src/macos/profile.ts

@@ -0,0 +1,19 @@
+import { fileURLToPath } from 'node:url';
+import type { PlatformProfile } from '../backend/backend.js';
+import { CONTRACT_PROTOCOL_VERSION } from '../backend/rpc.js';
+import { verifyHelperArtifact } from '../helper/artifact.js';
+
+/** The universal helper built by `native/macos/build.sh`. */
+export const macosHelperPath = fileURLToPath(new URL('../../bin/darwin-universal/moxxy-computer', import.meta.url));
+
+export const macosProfile: PlatformProfile = {
+  platform: 'darwin',
+  protocolVersion: CONTRACT_PROTOCOL_VERSION,
+  helperPath: macosHelperPath,
+  helperArgs: [],
+  verifyHelper: () => verifyHelperArtifact(macosHelperPath, CONTRACT_PROTOCOL_VERSION),
+  unavailableMessage: 'The macOS Computer Use helper is missing or does not match this version. Reinstall Moxxy; chat remains available.',
+  // Launching an app and letting it settle can take several seconds on its own.
+  timeoutMs: 30_000,
+  previewCodecs: ['h264', 'jpeg'],
+};

package/src/preview/controller.test.ts

@@ -0,0 +1,308 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { PreviewController, type PreviewChunk, type PreviewCodec, type PreviewImage, type PreviewMessage, type PreviewSource } from './controller.js';
+
+const image = (label: string): PreviewImage => ({ mediaType: 'image/jpeg', base64: label, width: 640, height: 400 });
+
+/** A producer that records what it was asked to do; `start` can be made to fail. */
+function source(fail?: string, codecs: readonly PreviewCodec[] = ['jpeg']): PreviewSource & { calls: string[] } {
+  const calls: string[] = [];
+  return {
+    calls,
+    codecs,
+    start: async (fps, codec) => { calls.push(codec === 'jpeg' ? `start ${fps}` : `start ${fps} ${codec}`); if (fail) throw new Error(fail); },
+    stop: async () => { calls.push('stop'); },
+    keyframe: async () => { calls.push('keyframe'); },
+  };
+}
+const video = () => source(undefined, ['h264', 'jpeg']);
+const chunk = (seq: number, key = false): PreviewChunk => ({ seq, key, codec: 'avc1.4d001f', data: `chunk${seq}`, timestamp: seq * 200_000, width: 640, height: 400 });
+const chunks = (messages: PreviewMessage[]) => messages.flatMap((message) => (message.type === 'chunk' ? [message.data] : []));
+
+function viewer(controller: PreviewController): { messages: PreviewMessage[]; leave: () => void } {
+  const messages: PreviewMessage[] = [];
+  return { messages, leave: controller.subscribe((message) => messages.push(message)) };
+}
+const frames = (messages: PreviewMessage[]) => messages.flatMap((message) => (message.type === 'frame' ? [message.image.base64] : []));
+const states = (messages: PreviewMessage[]) => messages.flatMap((message) => (message.type === 'state' ? [message.state] : []));
+
+beforeEach(() => { vi.useFakeTimers(); });
+afterEach(() => { vi.useRealTimers(); });
+
+describe('PreviewController', () => {
+  it('produces frames only while someone watches a running turn', async () => {
+    const controller = new PreviewController();
+    const helper = source();
+    controller.attach(helper);
+    expect(helper.calls).toEqual([]);
+    const first = viewer(controller);
+    const second = viewer(controller);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 2']);
+    first.leave();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 2']);
+    second.leave();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 2', 'stop']);
+  });
+
+  it('starts the producer when a turn begins while a viewer is already waiting', async () => {
+    const controller = new PreviewController();
+    const watching = viewer(controller);
+    expect(states(watching.messages)).toEqual(['stopped']);
+    const helper = source();
+    controller.attach(helper);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 2']);
+    controller.frame(helper, image('a'));
+    expect(states(watching.messages)).toEqual(['stopped', 'live']);
+    expect(frames(watching.messages)).toEqual(['a']);
+  });
+
+  it('keeps only the latest frame when frames arrive faster than the viewer rate', async () => {
+    const controller = new PreviewController({ fps: 2 });
+    const helper = source();
+    controller.attach(helper);
+    const watching = viewer(controller);
+    for (const label of ['a', 'b', 'c', 'd']) controller.frame(helper, image(label));
+    expect(frames(watching.messages)).toEqual(['a']);
+    await vi.advanceTimersByTimeAsync(500);
+    expect(frames(watching.messages)).toEqual(['a', 'd']);
+    await vi.advanceTimersByTimeAsync(2000);
+    expect(frames(watching.messages)).toEqual(['a', 'd']);
+  });
+
+  it('turns stale when the producer goes quiet and live again with its next sign of life', async () => {
+    const controller = new PreviewController({ staleAfterMs: 3000 });
+    const helper = source();
+    controller.attach(helper);
+    const watching = viewer(controller);
+    controller.frame(helper, image('a'));
+    await vi.advanceTimersByTimeAsync(2000);
+    controller.frame(helper, undefined);
+    await vi.advanceTimersByTimeAsync(2000);
+    expect(states(watching.messages)).toEqual(['stopped', 'live']);
+    await vi.advanceTimersByTimeAsync(1500);
+    expect(states(watching.messages)).toEqual(['stopped', 'live', 'stale']);
+    controller.frame(helper, undefined);
+    expect(states(watching.messages)).toEqual(['stopped', 'live', 'stale', 'live']);
+    expect(frames(watching.messages)).toEqual(['a']);
+  });
+
+  it('says why there is no picture when the producer cannot start or fails', async () => {
+    const controller = new PreviewController();
+    const watching = viewer(controller);
+    controller.attach(source('Screen Recording is not allowed'));
+    await vi.advanceTimersByTimeAsync(0);
+    expect(watching.messages.at(-1)).toEqual({ type: 'state', state: 'unavailable', reason: 'Screen Recording is not allowed' });
+    const helper = source();
+    controller.attach(helper);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.failed(helper, 'The window closed');
+    expect(watching.messages.at(-1)).toEqual({ type: 'state', state: 'unavailable', reason: 'The window closed' });
+  });
+
+  it('stops and forgets the picture when the turn ends, so nothing of it is shown afterwards', async () => {
+    const controller = new PreviewController();
+    const helper = source();
+    const detach = controller.attach(helper);
+    const watching = viewer(controller);
+    controller.frame(helper, image('a'));
+    detach();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(watching.messages.at(-1)).toEqual({ type: 'state', state: 'stopped' });
+    expect(controller.snapshot()).toEqual({ state: 'stopped' });
+    // The helper is gone with its turn; it is not asked to stop, and its late frames are dropped.
+    expect(helper.calls).toEqual(['start 2']);
+    controller.frame(helper, image('late'));
+    await vi.advanceTimersByTimeAsync(5000);
+    expect(frames(watching.messages)).toEqual(['a']);
+    expect(states(watching.messages).at(-1)).toBe('stopped');
+  });
+
+  it('gives a late viewer the current state and the latest frame at once', async () => {
+    const controller = new PreviewController();
+    const helper = source();
+    controller.attach(helper);
+    viewer(controller);
+    controller.frame(helper, image('a'));
+    const late = viewer(controller);
+    expect(late.messages).toEqual([{ type: 'state', state: 'live' }, { type: 'frame', seq: 1, image: image('a') }]);
+    expect(controller.snapshot()).toEqual({ state: 'live', frame: { seq: 1, image: image('a') } });
+  });
+
+  it('follows the newest turn and ignores frames of an older one', async () => {
+    const controller = new PreviewController();
+    const older = source();
+    const newer = source();
+    controller.attach(older);
+    const watching = viewer(controller);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.attach(newer);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(older.calls).toEqual(['start 2', 'stop']);
+    expect(newer.calls).toEqual(['start 2']);
+    controller.frame(older, image('old'));
+    controller.frame(newer, image('new'));
+    expect(frames(watching.messages)).toEqual(['new']);
+  });
+
+  it('runs video at 30 pictures a second, up to 30 when the viewer chooses', async () => {
+    const controller = new PreviewController();
+    const helper = video();
+    controller.attach(helper);
+    controller.subscribe(() => undefined, ['h264', 'jpeg']);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.setFps(10);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.setFps(120);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 30 h264', 'start 10 h264', 'start 30 h264']);
+  });
+
+  it('keeps the rate of single pictures between 1 and 5 and restarts a running producer at the new rate', async () => {
+    const controller = new PreviewController();
+    const helper = source();
+    controller.attach(helper);
+    viewer(controller);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.setFps(30);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.setFps(0);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.setFps(1);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 2', 'start 5', 'start 1']);
+  });
+});
+
+describe('PreviewController video', () => {
+  const watch = (controller: PreviewController, codecs?: readonly PreviewCodec[]) => {
+    const messages: PreviewMessage[] = [];
+    const listener = (message: PreviewMessage) => { messages.push(message); };
+    return { messages, listener, leave: controller.subscribe(listener, codecs) };
+  };
+
+  it('uses video only when the helper makes it and every viewer can show it', async () => {
+    const controller = new PreviewController();
+    const helper = video();
+    controller.attach(helper);
+    const modern = watch(controller, ['h264', 'jpeg']);
+    await vi.advanceTimersByTimeAsync(0);
+    // A new encoder starts with a key frame by itself.
+    expect(helper.calls).toEqual(['start 30 h264']);
+    // A viewer that only shows pictures joins: everyone gets pictures.
+    const plain = watch(controller);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls.at(-1)).toBe('start 2');
+    plain.leave();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 30 h264', 'start 2', 'start 30 h264']);
+    modern.leave();
+  });
+
+  it('keeps pictures when the helper has no video', async () => {
+    const controller = new PreviewController();
+    const helper = source();
+    controller.attach(helper);
+    watch(controller, ['h264', 'jpeg']);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 2']);
+  });
+
+  it('lets a viewer say later what it can show', async () => {
+    const controller = new PreviewController();
+    const helper = video();
+    controller.attach(helper);
+    const late = watch(controller);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.accept(late.listener, ['h264', 'jpeg']);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 2', 'start 30 h264']);
+  });
+
+  it('gives a viewer nothing until a key frame, then every chunk in order', async () => {
+    const controller = new PreviewController();
+    const helper = video();
+    controller.attach(helper);
+    const first = watch(controller, ['h264']);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.chunk(helper, chunk(1));
+    expect(chunks(first.messages)).toEqual([]);
+    controller.chunk(helper, chunk(2, true));
+    controller.chunk(helper, chunk(3));
+    expect(chunks(first.messages)).toEqual(['chunk2', 'chunk3']);
+    expect(states(first.messages).at(-1)).toBe('live');
+    // A second viewer joins in the middle of the stream: it waits for its own key frame.
+    const second = watch(controller, ['h264']);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls.at(-1)).toBe('keyframe');
+    controller.chunk(helper, chunk(4));
+    expect(chunks(first.messages)).toEqual(['chunk2', 'chunk3', 'chunk4']);
+    expect(chunks(second.messages)).toEqual([]);
+    controller.chunk(helper, chunk(5, true));
+    expect(chunks(second.messages)).toEqual(['chunk5']);
+    expect(controller.snapshot()).toEqual({ state: 'live' });
+  });
+
+  it('shows the pictures of a producer that was asked for video and cannot make it', async () => {
+    const controller = new PreviewController();
+    const helper = video();
+    controller.attach(helper);
+    const only = watch(controller, ['h264', 'jpeg']);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(helper.calls).toEqual(['start 30 h264']);
+
+    controller.frame(helper, image('still'));
+
+    expect(frames(only.messages)).toEqual(['still']);
+    expect(states(only.messages).at(-1)).toBe('live');
+  });
+
+  it('drops deltas after a lost chunk until the next key frame and asks for one', async () => {
+    const controller = new PreviewController();
+    const helper = video();
+    controller.attach(helper);
+    const only = watch(controller, ['h264']);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.chunk(helper, chunk(1, true));
+    controller.chunk(helper, chunk(2));
+    helper.calls.length = 0;
+    controller.chunk(helper, chunk(4));
+    controller.chunk(helper, chunk(5));
+    expect(chunks(only.messages)).toEqual(['chunk1', 'chunk2']);
+    expect(helper.calls).toEqual(['keyframe']);
+    controller.chunk(helper, chunk(6, true));
+    controller.chunk(helper, chunk(7));
+    expect(chunks(only.messages)).toEqual(['chunk1', 'chunk2', 'chunk6', 'chunk7']);
+  });
+
+  it('starts a viewer that fell behind again from a key frame', async () => {
+    const controller = new PreviewController();
+    const helper = video();
+    controller.attach(helper);
+    const slow = watch(controller, ['h264']);
+    const other = watch(controller, ['h264']);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.chunk(helper, chunk(1, true));
+    helper.calls.length = 0;
+    controller.keyframe(slow.listener);
+    expect(helper.calls).toEqual(['keyframe']);
+    controller.chunk(helper, chunk(2));
+    expect(chunks(slow.messages)).toEqual(['chunk1']);
+    expect(chunks(other.messages)).toEqual(['chunk1', 'chunk2']);
+    controller.chunk(helper, chunk(3, true));
+    expect(chunks(slow.messages)).toEqual(['chunk1', 'chunk3']);
+  });
+
+  it('goes stale when chunks stop, like pictures do', async () => {
+    const controller = new PreviewController({ staleAfterMs: 500 });
+    const helper = video();
+    controller.attach(helper);
+    const only = watch(controller, ['h264']);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.chunk(helper, chunk(1, true));
+    await vi.advanceTimersByTimeAsync(600);
+    expect(states(only.messages).at(-1)).toBe('stale');
+  });
+});

package/src/preview/controller.ts

@@ -0,0 +1,262 @@
+/** What the human sees of the app the agent works in. Never shown to the model or written to the session log. */
+export interface PreviewImage {
+  readonly mediaType: 'image/jpeg';
+  readonly base64: string;
+  readonly width: number;
+  readonly height: number;
+}
+
+/**
+ * `live`: frames are current. `stale`: the producer went quiet. `unavailable`:
+ * it cannot produce (with the reason). `stopped`: no turn is using the computer.
+ */
+export type PreviewState = 'live' | 'stale' | 'unavailable' | 'stopped';
+
+/** How the picture travels: single JPEG frames, or an H.264 stream for viewers that can decode one. */
+export type PreviewCodec = 'jpeg' | 'h264';
+const isCodec = (value: unknown): value is PreviewCodec => value === 'jpeg' || value === 'h264';
+/** The codecs in `value` this controller knows; anything else in the list is ignored. */
+export const previewCodecs = (value: unknown): PreviewCodec[] => (Array.isArray(value) ? value.filter(isCodec) : []);
+
+/** One access unit of the video, Annex B, base64. A key chunk decodes without anything before it. */
+export interface PreviewChunk {
+  readonly seq: number;
+  readonly key: boolean;
+  /** RFC 6381 codec string, e.g. `avc1.4d001f`. */
+  readonly codec: string;
+  readonly data: string;
+  /** Microseconds. */
+  readonly timestamp: number;
+  readonly width: number;
+  readonly height: number;
+}
+
+export type PreviewMessage =
+  | { readonly type: 'frame'; readonly seq: number; readonly image: PreviewImage }
+  | ({ readonly type: 'chunk' } & PreviewChunk)
+  | { readonly type: 'state'; readonly state: PreviewState; readonly reason?: string };
+
+export interface PreviewSnapshot {
+  readonly state: PreviewState;
+  readonly reason?: string;
+  readonly frame?: { readonly seq: number; readonly image: PreviewImage };
+}
+
+/** One turn's helper, as far as the preview is concerned. `start` on a running producer changes its rate. */
+export interface PreviewSource {
+  /** What this helper can produce. */
+  readonly codecs: readonly PreviewCodec[];
+  start(fps: number, codec: PreviewCodec): Promise<void>;
+  stop(): Promise<void>;
+  /** Make the next chunk a key frame. */
+  keyframe(): Promise<void>;
+}
+
+/** Pictures a second. Video is smooth; single pictures are each a whole JPEG, so they stay slow. */
+export const PREVIEW_FPS = { h264: { default: 30, max: 30 }, jpeg: { default: 2, max: 5 }, min: 1 } as const;
+
+type Listener = (message: PreviewMessage) => void;
+interface Viewer {
+  /** What this viewer can show. */
+  codecs: readonly PreviewCodec[];
+  /** In a video stream: nothing is sent to it until the next key frame. */
+  waiting: boolean;
+}
+
+/**
+ * Connects the viewers of the preview to the helper of the turn that is using
+ * the computer. The helper captures only while someone watches; viewers get the
+ * latest frame at the chosen rate, never a backlog. Video is used when the
+ * helper makes it and every viewer can decode it; a viewer that cannot follow
+ * the stream gets nothing until the next key frame.
+ */
+export class PreviewController {
+  private readonly listeners = new Map<Listener, Viewer>();
+  private readonly staleAfterMs: number;
+  /** What the viewer asked for; without a wish each kind of stream runs at its own default. */
+  private wish?: number;
+  /** The newest attached turn; older ones are ignored. */
+  private active?: PreviewSource;
+  /** What the active producer was last asked to do. */
+  private running?: { readonly source: PreviewSource; readonly fps: number; readonly codec: PreviewCodec };
+  /** Sequence number of the last chunk the producer sent. */
+  private lastChunk?: number;
+  private state: PreviewState = 'stopped';
+  private reason?: string;
+  private seq = 0;
+  private latest?: { seq: number; image: PreviewImage };
+  private deliveredAt = Number.NEGATIVE_INFINITY;
+  private flush?: ReturnType<typeof setTimeout>;
+  private quiet?: ReturnType<typeof setTimeout>;
+
+  constructor(options: { fps?: number; staleAfterMs?: number } = {}) {
+    this.wish = options.fps;
+    this.staleAfterMs = options.staleAfterMs ?? 3000;
+  }
+
+  /** A viewer joins: it gets the current state and latest frame at once, then every change. */
+  subscribe(listener: Listener, codecs: readonly PreviewCodec[] = ['jpeg']): () => void {
+    this.listeners.set(listener, { codecs, waiting: true });
+    listener(this.stateMessage());
+    if (this.latest) listener({ type: 'frame', ...this.latest });
+    this.sync();
+    return () => {
+      if (!this.listeners.delete(listener)) return;
+      this.sync();
+    };
+  }
+
+  /** A viewer says what it can show, after it joined. */
+  accept(listener: Listener, codecs: readonly PreviewCodec[]): void {
+    const viewer = this.listeners.get(listener);
+    if (!viewer) return;
+    viewer.codecs = codecs;
+    this.sync();
+  }
+
+  /** A viewer lost its place in the video (it dropped chunks or its decoder failed). */
+  keyframe(listener: Listener): void {
+    const viewer = this.listeners.get(listener);
+    if (!viewer || this.running?.codec !== 'h264') return;
+    viewer.waiting = true;
+    this.askForKey(this.running.source);
+  }
+
+  snapshot(): PreviewSnapshot {
+    return { state: this.state, ...(this.reason ? { reason: this.reason } : {}), ...(this.latest ? { frame: this.latest } : {}) };
+  }
+
+  setFps(fps: number): void {
+    this.wish = fps;
+    this.sync();
+  }
+
+  /** A turn started using the computer. Returns the detach to call when its helper is gone. */
+  attach(source: PreviewSource): () => void {
+    this.active = source;
+    this.forget();
+    this.sync();
+    return () => {
+      if (this.active !== source) return;
+      this.active = undefined;
+      // The helper leaves with its turn, so there is nothing left to stop.
+      this.running = undefined;
+      this.forget();
+      this.setState('stopped');
+    };
+  }
+
+  /** A frame from a producer, or `undefined` for "still running, nothing changed". */
+  frame(source: PreviewSource, image: PreviewImage | undefined): void {
+    if (source !== this.active || this.listeners.size === 0) return;
+    this.alive();
+    // A producer asked for video that sends a picture cannot make video: the picture is shown.
+    if (!image) return;
+    this.latest = { seq: ++this.seq, image };
+    const wait = this.deliveredAt + 1000 / this.rate('jpeg') - Date.now();
+    if (wait <= 0) this.deliver();
+    else this.flush ??= setTimeout(() => this.deliver(), wait);
+  }
+
+  /** A piece of the video from a producer. */
+  chunk(source: PreviewSource, chunk: PreviewChunk): void {
+    if (source !== this.active || this.running?.codec !== 'h264' || this.listeners.size === 0) return;
+    this.alive();
+    // A chunk went missing: what follows cannot be decoded until the stream starts again.
+    const lost = !chunk.key && this.lastChunk !== undefined && chunk.seq !== this.lastChunk + 1;
+    this.lastChunk = chunk.seq;
+    if (lost) {
+      for (const viewer of this.listeners.values()) viewer.waiting = true;
+      this.askForKey(source);
+    }
+    for (const [listener, viewer] of this.listeners) {
+      if (chunk.key) viewer.waiting = false;
+      if (!viewer.waiting) listener({ type: 'chunk', ...chunk });
+    }
+  }
+
+  failed(source: PreviewSource, reason: string): void {
+    if (source !== this.active) return;
+    clearTimeout(this.quiet);
+    this.setState('unavailable', reason);
+  }
+
+  private alive(): void {
+    this.setState('live');
+    clearTimeout(this.quiet);
+    this.quiet = setTimeout(() => this.setState('stale'), this.staleAfterMs);
+  }
+
+  private askForKey(source: PreviewSource): void {
+    void source.keyframe().catch(() => undefined);
+  }
+
+  private codecFor(source: PreviewSource): PreviewCodec {
+    const everyone = [...this.listeners.values()].every((viewer) => viewer.codecs.includes('h264'));
+    return everyone && source.codecs.includes('h264') ? 'h264' : 'jpeg';
+  }
+
+  private rate(codec: PreviewCodec): number {
+    const { default: usual, max } = PREVIEW_FPS[codec];
+    return Math.min(max, Math.max(PREVIEW_FPS.min, Math.round(this.wish ?? usual)));
+  }
+
+  private deliver(): void {
+    clearTimeout(this.flush);
+    this.flush = undefined;
+    if (!this.latest) return;
+    this.deliveredAt = Date.now();
+    this.emit({ type: 'frame', ...this.latest });
+  }
+
+  /** Makes the producer match what is wanted: running at the current rate and codec only while watched. */
+  private sync(): void {
+    const codec = this.active && this.codecFor(this.active);
+    const wanted = this.active && codec && this.listeners.size > 0 ? { source: this.active, fps: this.rate(codec), codec } : undefined;
+    const running = this.running;
+    if (running && running.source !== wanted?.source) {
+      this.running = undefined;
+      clearTimeout(this.quiet);
+      clearTimeout(this.flush);
+      this.flush = undefined;
+      void running.source.stop().catch(() => undefined);
+    }
+    if (!wanted) return;
+    const same = this.running?.source === wanted.source && this.running.fps === wanted.fps && this.running.codec === wanted.codec;
+    if (!same) {
+      if (this.running?.codec !== wanted.codec) {
+        // The other kind of stream starts from nothing: no old picture, no place in the old video.
+        this.forget();
+        for (const viewer of this.listeners.values()) viewer.waiting = true;
+      }
+      this.running = wanted;
+      wanted.source.start(wanted.fps, wanted.codec).catch((error: unknown) => this.failed(wanted.source, error instanceof Error ? error.message : String(error)));
+    }
+    // Whoever is waiting in a video stream (a new viewer, or all of them after a switch) needs a key frame.
+    if (wanted.codec === 'h264' && [...this.listeners.values()].some((viewer) => viewer.waiting) && this.lastChunk !== undefined) this.askForKey(wanted.source);
+  }
+
+  private forget(): void {
+    clearTimeout(this.quiet);
+    clearTimeout(this.flush);
+    this.flush = undefined;
+    this.latest = undefined;
+    this.lastChunk = undefined;
+    this.deliveredAt = Number.NEGATIVE_INFINITY;
+  }
+
+  private setState(state: PreviewState, reason?: string): void {
+    if (this.state === state && this.reason === reason) return;
+    this.state = state;
+    this.reason = reason;
+    this.emit(this.stateMessage());
+  }
+
+  private stateMessage(): PreviewMessage {
+    return { type: 'state', state: this.state, ...(this.reason ? { reason: this.reason } : {}) };
+  }
+
+  private emit(message: PreviewMessage): void {
+    for (const listener of this.listeners.keys()) listener(message);
+  }
+}

package/src/preview/surface.test.ts

@@ -0,0 +1,68 @@
+import { describe, expect, it } from 'vitest';
+import { PreviewController, type PreviewSource } from './controller.js';
+import { COMPUTER_PREVIEW_SURFACE, buildComputerPreviewSurface } from './surface.js';
+
+const helper = (calls: string[], codecs: PreviewSource['codecs'] = ['jpeg']): PreviewSource => ({
+  codecs,
+  start: async (fps, codec) => { calls.push(codec === 'jpeg' ? `start ${fps}` : `start ${fps} ${codec}`); },
+  stop: async () => { calls.push('stop'); },
+  keyframe: async () => { calls.push('keyframe'); },
+});
+const settle = () => new Promise((resolve) => setImmediate(resolve));
+
+describe('computer-preview surface', () => {
+  it('is the viewer of the preview: opening it starts frames, closing it stops them', async () => {
+    const controller = new PreviewController();
+    const calls: string[] = [];
+    const source = helper(calls);
+    controller.attach(source);
+    const surface = buildComputerPreviewSurface(controller);
+    expect(surface.kind).toBe(COMPUTER_PREVIEW_SURFACE);
+    const instance = await surface.open({ cwd: process.cwd() });
+    const payloads: unknown[] = [];
+    const off = instance.onData((payload) => payloads.push(payload));
+    await settle();
+    expect(calls).toEqual(['start 2']);
+    controller.frame(source, { mediaType: 'image/jpeg', base64: 'a', width: 2, height: 1 });
+    expect(payloads.at(-1)).toEqual({ type: 'frame', seq: 1, image: { mediaType: 'image/jpeg', base64: 'a', width: 2, height: 1 } });
+    expect(instance.snapshot?.()).toEqual({ state: 'live', frame: { seq: 1, image: { mediaType: 'image/jpeg', base64: 'a', width: 2, height: 1 } } });
+    off();
+    await instance.close();
+    await settle();
+    expect(calls).toEqual(['start 2', 'stop']);
+  });
+
+  it('lets the viewer choose the frame rate and ignores anything else it sends', async () => {
+    const controller = new PreviewController();
+    const calls: string[] = [];
+    controller.attach(helper(calls));
+    const instance = await buildComputerPreviewSurface(controller).open({ cwd: process.cwd() });
+    instance.onData(() => undefined);
+    await settle();
+    await instance.input({ type: 'configure', fps: 4 });
+    await instance.input({ type: 'configure', fps: 'fast' });
+    await instance.input({ type: 'click', x: 1, y: 1 });
+    await settle();
+    expect(calls).toEqual(['start 2', 'start 4']);
+  });
+
+  it('switches to video for a viewer that can decode it and passes on its request for a key frame', async () => {
+    const controller = new PreviewController();
+    const calls: string[] = [];
+    const source = helper(calls, ['h264', 'jpeg']);
+    controller.attach(source);
+    const instance = await buildComputerPreviewSurface(controller).open({ cwd: process.cwd() });
+    const payloads: unknown[] = [];
+    instance.onData((payload) => payloads.push(payload));
+    await settle();
+    await instance.input({ type: 'configure', codecs: ['h264', 'jpeg'] });
+    await settle();
+    expect(calls).toEqual(['start 2', 'start 30 h264']);
+    controller.chunk(source, { seq: 1, key: true, codec: 'avc1.4d001f', data: 'AAAA', timestamp: 0, width: 640, height: 400 });
+    expect(payloads.at(-1)).toEqual({ type: 'chunk', seq: 1, key: true, codec: 'avc1.4d001f', data: 'AAAA', timestamp: 0, width: 640, height: 400 });
+    await instance.input({ type: 'keyframe' });
+    await instance.input({ type: 'configure', codecs: ['vp9', 7] });
+    await settle();
+    expect(calls).toEqual(['start 2', 'start 30 h264', 'keyframe', 'start 2']);
+  });
+});