@frockbot/plugin-computer 0.0.0 → 0.1.0

package/src/protocol.ts

@@ -0,0 +1,405 @@
+// The Computer presence protocol crosses the hosted client, gateway and Bot
+// Durable Object seams. It contains projections and commands only: notably,
+// the viewer URL is present solely on a read projection and never on a command
+// receipt, so an idempotency record cannot become durable secret storage.
+
+export const COMPUTER_COMMAND_TYPES = [
+  "connect",
+  "takeControl",
+  "releaseControl",
+  "refreshControl",
+  "refreshViewer",
+  "runDoctor",
+] as const;
+
+export type ComputerCommandTypeV1 = (typeof COMPUTER_COMMAND_TYPES)[number];
+
+export interface ComputerCommandV1 {
+  version: 1;
+  commandId: string;
+  botId: string;
+  type: ComputerCommandTypeV1;
+}
+
+export interface ComputerViewerSessionViewV1 {
+  version: 1;
+  id: string;
+  /**
+   * A bearer secret for the VNC transport. It crosses this one projection,
+   * lives only in client memory, and must never enter storage or a log.
+   */
+  url: string;
+  expiresAt: string;
+}
+
+export interface ComputerControlLeaseViewV1 {
+  version: 1;
+  ownerId: string;
+  acquiredAt: string;
+  expiresAt: string;
+}
+
+export interface ComputerScreenshotViewV1 {
+  version: 1;
+  path: string;
+  capturedAt: string;
+  contentHash: string;
+  url: string;
+}
+
+export interface ComputerDoctorCheckViewV1 {
+  version: 1;
+  name: string;
+  status: "pass" | "fail";
+  detail: string;
+}
+
+export interface ComputerDoctorViewV1 {
+  version: 1;
+  capturedAt: string;
+  summary: string;
+  checks: ComputerDoctorCheckViewV1[];
+}
+
+export const COMPUTER_PHASES = [
+  "unconfigured",
+  "idle",
+  "provisioning",
+  "updating",
+  "ready",
+  "taking-control",
+  "human-control",
+  "disconnected",
+  "error",
+] as const;
+
+export type ComputerPhase = (typeof COMPUTER_PHASES)[number];
+
+export const COMPUTER_UPDATE_MESSAGE_PREFIX = "Updating the Computer: ";
+
+/** Extracts the provider's update phase label without coupling to a provider. */
+export function computerUpdateLabelV1(
+  message: string | undefined,
+): string | undefined {
+  if (!message?.startsWith(COMPUTER_UPDATE_MESSAGE_PREFIX)) return undefined;
+  const label = message.slice(COMPUTER_UPDATE_MESSAGE_PREFIX.length).trim();
+  return label || undefined;
+}
+
+export interface ComputerProjectionV1 {
+  version: 1;
+  botId: string;
+  providerLabel: string;
+  phase: ComputerPhase;
+  message: string;
+  viewerSession?: ComputerViewerSessionViewV1;
+  controlLease?: ComputerControlLeaseViewV1;
+  screenshots: ComputerScreenshotViewV1[];
+  doctor?: ComputerDoctorViewV1;
+}
+
+export type ComputerCommandReceiptV1 =
+  | {
+      version: 1;
+      commandId: string;
+      type: ComputerCommandTypeV1;
+      status: "applied";
+      completedAt: string;
+    }
+  | {
+      version: 1;
+      commandId: string;
+      type: ComputerCommandTypeV1;
+      status: "rejected";
+      completedAt: string;
+      failure: string;
+    };
+
+export class ComputerProtocolDecodeError extends Error {
+  override readonly name = "ComputerProtocolDecodeError";
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function record(value: unknown, label: string): Record<string, unknown> {
+  if (!isRecord(value)) {
+    throw new ComputerProtocolDecodeError(`${label} must be an object`);
+  }
+  return value;
+}
+
+function exactKeys(
+  value: Record<string, unknown>,
+  required: readonly string[],
+  optional: readonly string[],
+  label: string,
+): void {
+  const allowed = new Set([...required, ...optional]);
+  if (
+    !required.every((key) => Object.hasOwn(value, key)) ||
+    Object.keys(value).some((key) => !allowed.has(key))
+  ) {
+    throw new ComputerProtocolDecodeError(`${label} has unexpected fields`);
+  }
+}
+
+function text(value: unknown, label: string): string {
+  if (typeof value !== "string" || value.length === 0 || value.length > 4096) {
+    throw new ComputerProtocolDecodeError(`${label} is invalid`);
+  }
+  return value;
+}
+
+function timestamp(value: unknown, label: string): string {
+  const decoded = text(value, label);
+  if (!Number.isFinite(Date.parse(decoded))) {
+    throw new ComputerProtocolDecodeError(`${label} is invalid`);
+  }
+  return decoded;
+}
+
+function commandType(value: unknown): ComputerCommandTypeV1 {
+  const decoded = COMPUTER_COMMAND_TYPES.find((known) => known === value);
+  if (!decoded) {
+    throw new ComputerProtocolDecodeError("Computer command type is unknown");
+  }
+  return decoded;
+}
+
+function phase(value: unknown): ComputerPhase {
+  const decoded = COMPUTER_PHASES.find((known) => known === value);
+  if (!decoded) {
+    throw new ComputerProtocolDecodeError("Computer phase is unknown");
+  }
+  return decoded;
+}
+
+export function decodeComputerCommandV1(value: unknown): ComputerCommandV1 {
+  const candidate = record(value, "Computer command");
+  exactKeys(
+    candidate,
+    ["version", "commandId", "botId", "type"],
+    [],
+    "Computer command",
+  );
+  if (candidate.version !== 1) {
+    throw new ComputerProtocolDecodeError(
+      "Computer command version is unsupported",
+    );
+  }
+  return {
+    version: 1,
+    commandId: text(candidate.commandId, "Computer commandId"),
+    botId: text(candidate.botId, "Computer botId"),
+    type: commandType(candidate.type),
+  };
+}
+
+function decodeViewerSessionV1(value: unknown): ComputerViewerSessionViewV1 {
+  const candidate = record(value, "Computer viewer session");
+  exactKeys(
+    candidate,
+    ["version", "id", "url", "expiresAt"],
+    [],
+    "Computer viewer session",
+  );
+  if (candidate.version !== 1) {
+    throw new ComputerProtocolDecodeError(
+      "Computer viewer session version is unsupported",
+    );
+  }
+  return {
+    version: 1,
+    id: text(candidate.id, "Computer viewer session id"),
+    url: text(candidate.url, "Computer viewer session URL"),
+    expiresAt: timestamp(
+      candidate.expiresAt,
+      "Computer viewer session expiresAt",
+    ),
+  };
+}
+
+function decodeControlLeaseV1(value: unknown): ComputerControlLeaseViewV1 {
+  const candidate = record(value, "Computer control lease");
+  exactKeys(
+    candidate,
+    ["version", "ownerId", "acquiredAt", "expiresAt"],
+    [],
+    "Computer control lease",
+  );
+  if (candidate.version !== 1) {
+    throw new ComputerProtocolDecodeError(
+      "Computer control lease version is unsupported",
+    );
+  }
+  return {
+    version: 1,
+    ownerId: text(candidate.ownerId, "Computer control lease ownerId"),
+    acquiredAt: timestamp(
+      candidate.acquiredAt,
+      "Computer control lease acquiredAt",
+    ),
+    expiresAt: timestamp(
+      candidate.expiresAt,
+      "Computer control lease expiresAt",
+    ),
+  };
+}
+
+function decodeScreenshotV1(value: unknown): ComputerScreenshotViewV1 {
+  const candidate = record(value, "Computer screenshot");
+  exactKeys(
+    candidate,
+    ["version", "path", "capturedAt", "contentHash", "url"],
+    [],
+    "Computer screenshot",
+  );
+  if (candidate.version !== 1) {
+    throw new ComputerProtocolDecodeError(
+      "Computer screenshot version is unsupported",
+    );
+  }
+  return {
+    version: 1,
+    path: text(candidate.path, "Computer screenshot path"),
+    capturedAt: timestamp(
+      candidate.capturedAt,
+      "Computer screenshot capturedAt",
+    ),
+    contentHash: text(candidate.contentHash, "Computer screenshot contentHash"),
+    url: text(candidate.url, "Computer screenshot URL"),
+  };
+}
+
+function decodeDoctorCheckV1(value: unknown): ComputerDoctorCheckViewV1 {
+  const candidate = record(value, "Computer doctor check");
+  exactKeys(
+    candidate,
+    ["version", "name", "status", "detail"],
+    [],
+    "Computer doctor check",
+  );
+  if (candidate.version !== 1) {
+    throw new ComputerProtocolDecodeError(
+      "Computer doctor check version is unsupported",
+    );
+  }
+  if (candidate.status !== "pass" && candidate.status !== "fail") {
+    throw new ComputerProtocolDecodeError(
+      "Computer doctor check status is invalid",
+    );
+  }
+  return {
+    version: 1,
+    name: text(candidate.name, "Computer doctor check name"),
+    status: candidate.status,
+    detail: text(candidate.detail, "Computer doctor check detail"),
+  };
+}
+
+function decodeDoctorV1(value: unknown): ComputerDoctorViewV1 {
+  const candidate = record(value, "Computer doctor report");
+  exactKeys(
+    candidate,
+    ["version", "capturedAt", "summary", "checks"],
+    [],
+    "Computer doctor report",
+  );
+  if (candidate.version !== 1 || !Array.isArray(candidate.checks)) {
+    throw new ComputerProtocolDecodeError("Computer doctor report is invalid");
+  }
+  return {
+    version: 1,
+    capturedAt: timestamp(
+      candidate.capturedAt,
+      "Computer doctor report capturedAt",
+    ),
+    summary: text(candidate.summary, "Computer doctor report summary"),
+    checks: candidate.checks.map(decodeDoctorCheckV1),
+  };
+}
+
+export function decodeComputerProjectionV1(
+  value: unknown,
+): ComputerProjectionV1 {
+  const candidate = record(value, "Computer projection");
+  exactKeys(
+    candidate,
+    ["version", "botId", "providerLabel", "phase", "message", "screenshots"],
+    ["viewerSession", "controlLease", "doctor"],
+    "Computer projection",
+  );
+  if (candidate.version !== 1 || !Array.isArray(candidate.screenshots)) {
+    throw new ComputerProtocolDecodeError("Computer projection is invalid");
+  }
+  return {
+    version: 1,
+    botId: text(candidate.botId, "Computer projection botId"),
+    providerLabel: text(
+      candidate.providerLabel,
+      "Computer projection providerLabel",
+    ),
+    phase: phase(candidate.phase),
+    message: text(candidate.message, "Computer projection message"),
+    ...(candidate.viewerSession === undefined
+      ? {}
+      : { viewerSession: decodeViewerSessionV1(candidate.viewerSession) }),
+    ...(candidate.controlLease === undefined
+      ? {}
+      : { controlLease: decodeControlLeaseV1(candidate.controlLease) }),
+    screenshots: candidate.screenshots.map(decodeScreenshotV1),
+    ...(candidate.doctor === undefined
+      ? {}
+      : { doctor: decodeDoctorV1(candidate.doctor) }),
+  };
+}
+
+export function decodeComputerCommandReceiptV1(
+  value: unknown,
+): ComputerCommandReceiptV1 {
+  const candidate = record(value, "Computer command receipt");
+  const rejected = candidate.status === "rejected";
+  exactKeys(
+    candidate,
+    ["version", "commandId", "type", "status", "completedAt"],
+    rejected ? ["failure"] : [],
+    "Computer command receipt",
+  );
+  if (candidate.version !== 1) {
+    throw new ComputerProtocolDecodeError(
+      "Computer command receipt version is unsupported",
+    );
+  }
+  const common = {
+    version: 1 as const,
+    commandId: text(candidate.commandId, "Computer command receipt commandId"),
+    type: commandType(candidate.type),
+    completedAt: timestamp(
+      candidate.completedAt,
+      "Computer command receipt completedAt",
+    ),
+  };
+  if (candidate.status === "applied") return { ...common, status: "applied" };
+  if (candidate.status === "rejected") {
+    return {
+      ...common,
+      status: "rejected",
+      failure: text(candidate.failure, "Computer command receipt failure"),
+    };
+  }
+  throw new ComputerProtocolDecodeError(
+    "Computer command receipt status is invalid",
+  );
+}
+
+export function computerCommandFingerprintV1(
+  command: ComputerCommandV1,
+): string {
+  return JSON.stringify({
+    version: 1,
+    botId: command.botId,
+    type: command.type,
+  });
+}

package/src/roots.ts

@@ -0,0 +1,6 @@
+/** The Package-declared durable root screenshots are written to. */
+export const COMPUTER_SCREENSHOTS_ROOT_ID = "screenshots";
+/** Screenshots kept per Bot. Older captures are pruned on the next capture. */
+export const COMPUTER_SCREENSHOT_RETENTION = 20;
+/** The Package-declared durable root a self-check report is filed in. */
+export const COMPUTER_DOCTOR_ROOT_ID = "doctor";

package/src/screenshot.test.ts

@@ -0,0 +1,253 @@
+// `computer_screenshot`: parity row 25.
+//
+// The subject is what the tool *files*, not what `scrot` produced. Three rules
+// are asserted here because they are the ones a future change could quietly
+// break: the bytes go through `ComputerWorkspace.write` so the Bot is recorded
+// as their writer, the root is bounded, and the model gets a reference it can
+// resolve rather than a picture of a path.
+import { describe, expect, test } from "bun:test";
+import { SystemPromptRegistry } from "@frockbot/plugin-prompt";
+import { ToolRegistry } from "@frockbot/plugin-tools";
+import {
+  ComputerRegistry,
+  computerBotPathKeyV1,
+  type ComputerHandle,
+  type ComputerProvider,
+} from "@frockbot/computer-core";
+import { createPluginHarness } from "@frockbot/plugin-testkit";
+import { SessionStore } from "@frockbot/kernel-contracts";
+import { createComputerAgentPlugin, pngDimensionsV1 } from "./agent.js";
+import { FakeWorkspace } from "./workspace-fixture.js";
+
+/** A 4x3 PNG: a real signature and a real IHDR, and nothing after it. */
+function png(width = 4, height = 3): Uint8Array {
+  const bytes = new Uint8Array(32);
+  bytes.set([137, 80, 78, 71, 13, 10, 26, 10], 0);
+  const view = new DataView(bytes.buffer);
+  view.setUint32(16, width);
+  view.setUint32(20, height);
+  return bytes;
+}
+
+function providerWith(
+  workspace: FakeWorkspace,
+  capture: () => Promise<{
+    bytes: Uint8Array;
+    mediaType: "image/png";
+    display: string;
+    capturedAt: string;
+  }>,
+): ComputerProvider {
+  return {
+    id: "fixture",
+    open: (identity, tenant, assignment): Promise<ComputerHandle> =>
+      Promise.resolve({
+        assignment,
+        identity,
+        tenant,
+        workspace,
+        screenshot: { capture: () => capture() },
+        close: () => Promise.resolve(),
+      }),
+  };
+}
+
+async function mount(provider: ComputerProvider, writer = true) {
+  const harness = await createPluginHarness([
+    ComputerRegistry,
+    ToolRegistry,
+    SystemPromptRegistry,
+    SessionStore,
+  ]);
+  harness.root.computers.register(provider);
+  await harness.mount(
+    createComputerAgentPlugin({
+      userId: "user-1",
+      defaultProviderId: "fixture",
+      ...(writer
+        ? {
+            writer: {
+              sessionId: "session-1",
+              turnId: "run-9",
+              runId: "run-9",
+            },
+          }
+        : {}),
+    }),
+  );
+  return harness;
+}
+
+async function capture(
+  harness: Awaited<ReturnType<typeof createPluginHarness>>,
+) {
+  const context = {
+    botId: "bot-1",
+    agentId: "run-9",
+    compositionGenerationId: "bootstrap",
+    turnType: "chat" as const,
+    sessionId: "session-1",
+    effectId: "tool:1:1:0",
+    signal: new AbortController().signal,
+  };
+  const prepared = await harness.root.tools.prepare(
+    { id: crypto.randomUUID(), name: "computer_screenshot", input: {} },
+    context,
+  );
+  if (prepared.kind !== "ready") throw new Error(prepared.result.content);
+  return harness.root.tools.executePrepared(prepared, context);
+}
+
+describe("computer_screenshot", () => {
+  test("files the capture through the Workspace with the Bot as its writer", async () => {
+    const workspace = new FakeWorkspace();
+    const harness = await mount(
+      providerWith(workspace, () =>
+        Promise.resolve({
+          bytes: png(1280, 720),
+          mediaType: "image/png" as const,
+          display: ":100",
+          capturedAt: "2026-08-31T00:00:00.000Z",
+        }),
+      ),
+    );
+
+    const result = await capture(harness);
+
+    expect(result.isError).toBe(false);
+    const answer = JSON.parse(result.content) as Record<string, unknown>;
+    const botKey = computerBotPathKeyV1("bot-1");
+    expect(answer).toMatchObject({
+      path: `${botKey}/run-9-1.png`,
+      rootId: "screenshots",
+      width: 1280,
+      height: 720,
+      display: ":100",
+      capturedAt: "2026-08-31T00:00:00.000Z",
+    });
+    // The writer is the point: a file a shell left on the Computer would sync
+    // back `unattributed`, so the tool reads the bytes and writes them here.
+    const written = workspace.files.get(`${botKey}/run-9-1.png`);
+    expect(written?.generation.writer).toEqual({
+      kind: "bot",
+      botId: "bot-1",
+      sessionId: "session-1",
+      turnId: "run-9",
+      runId: "run-9",
+    });
+    expect(result.attachments).toEqual([
+      {
+        kind: "image",
+        mediaType: "image/png",
+        workspacePath: {
+          root: {
+            kind: "package-declared",
+            userId: "user-1",
+            packageId: "computer",
+            rootId: "screenshots",
+          },
+          path: `${botKey}/run-9-1.png`,
+        },
+        contentHash: written!.generation.contentHash,
+        bytes: written!.generation.size,
+      },
+    ]);
+    await harness.dispose();
+  });
+
+  test("prunes to the newest twenty captures for one Bot", async () => {
+    const workspace = new FakeWorkspace();
+    const harness = await mount(
+      providerWith(workspace, () =>
+        Promise.resolve({
+          bytes: png(),
+          mediaType: "image/png" as const,
+          display: ":100",
+          capturedAt: "2026-08-31T00:00:00.000Z",
+        }),
+      ),
+    );
+
+    for (let index = 0; index < 23; index += 1) await capture(harness);
+
+    expect(workspace.files.size).toBe(20);
+    expect(workspace.deleted).toEqual([
+      `${computerBotPathKeyV1("bot-1")}/run-9-1.png`,
+      `${computerBotPathKeyV1("bot-1")}/run-9-2.png`,
+      `${computerBotPathKeyV1("bot-1")}/run-9-3.png`,
+    ]);
+    await harness.dispose();
+  });
+
+  test("reports a Computer with no screenshot capability as a failure", async () => {
+    const workspace = new FakeWorkspace();
+    const harness = await mount({
+      id: "fixture",
+      open: (identity, tenant, assignment) =>
+        Promise.resolve({
+          assignment,
+          identity,
+          tenant,
+          workspace,
+          close: () => Promise.resolve(),
+        }),
+    });
+
+    const result = await capture(harness);
+
+    expect(result).toMatchObject({ isError: true });
+    expect(result.content).toContain("does not support screenshots");
+    await harness.dispose();
+  });
+
+  test("carries a refused capture back as the tool's failure", async () => {
+    const workspace = new FakeWorkspace();
+    const harness = await mount(
+      providerWith(workspace, () =>
+        Promise.reject(
+          new Error("The user is controlling this agent's computer"),
+        ),
+      ),
+    );
+
+    const result = await capture(harness);
+
+    expect(result).toMatchObject({ isError: true });
+    expect(result.content).toContain("controlling this agent's computer");
+    expect(workspace.files.size).toBe(0);
+    await harness.dispose();
+  });
+
+  test("is not offered outside a Turn that can name its writer", async () => {
+    const workspace = new FakeWorkspace();
+    const harness = await mount(
+      providerWith(workspace, () =>
+        Promise.resolve({
+          bytes: png(),
+          mediaType: "image/png" as const,
+          display: ":100",
+          capturedAt: "2026-08-31T00:00:00.000Z",
+        }),
+      ),
+      false,
+    );
+
+    expect(
+      harness.root.tools
+        .schemas({ turnType: "chat" })
+        .map((schema) => schema.name),
+    ).not.toContain("computer_screenshot");
+    await harness.dispose();
+  });
+});
+
+describe("pngDimensionsV1", () => {
+  test("reads the IHDR of a PNG and refuses anything else", () => {
+    expect(pngDimensionsV1(png(1280, 720))).toEqual({
+      width: 1280,
+      height: 720,
+    });
+    expect(pngDimensionsV1(new Uint8Array(8))).toBeUndefined();
+    expect(pngDimensionsV1(png(0, 0))).toBeUndefined();
+  });
+});