@sanoma-ai/workflows 0.11.16 → 0.11.18

package/src/runner-box.ts

@@ -0,0 +1,218 @@
+import { readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import {
+  FILE,
+  frame,
+  RUNNER_HEALTH,
+  type RunnerAnswer,
+  type RunnerHealth,
+  type RunnerRequest,
+  STDERR,
+  STDIN,
+  unframe,
+} from "./runner-protocol.ts";
+import type { Box, Process } from "./sandbox-contract.ts";
+
+// A box whose commands and files go through the runner (`sandbox-runner.ts`), a small server of
+// ours a backend starts inside a remote box, over one WebSocket: so every vendor's backend is
+// "open the box, start the runner, talk to it", whatever the vendor's own exec can or cannot
+// stream. The protocol is in `runner-protocol.ts`. One connection carries many processes; a
+// dropped one ends them here (exit 255) but kills nothing in the box.
+
+/**
+ * The runner as one file (`dist/sandbox-runner.cjs`, its dependencies bundled in), which a
+ * backend writes into a box and starts with the box's `node` (22 or later).
+ */
+export async function runnerScript(): Promise<Uint8Array> {
+  // `../dist/` from `src/` (the tests) and from `dist/` alike.
+  const at = new URL("../dist/sandbox-runner.cjs", import.meta.url);
+  return readFile(at).catch(() => {
+    throw new Error(`${fileURLToPath(at)} is missing: build @sanoma-ai/workflows`);
+  });
+}
+
+/**
+ * What the runner at `url` (asked with `headers`) answers at `RUNNER_HEALTH`, or undefined when nothing
+ * answers as a runner: a backend starts one then, and replaces one of another `protocol`.
+ */
+export const runnerHealth = (url: string, headers?: Record<string, string>): Promise<RunnerHealth | undefined> =>
+  fetch(`${url}${RUNNER_HEALTH}`, { headers, signal: AbortSignal.timeout(5_000) })
+    .then(async (res) => {
+      const health = res.ok ? ((await res.json()) as Partial<RunnerHealth> | null) : null;
+      return typeof health?.protocol === "number" && typeof health.pid === "number"
+        ? { protocol: health.protocol, pid: health.pid }
+        : undefined;
+    })
+    .catch(() => undefined);
+
+const encoder = new TextEncoder();
+
+/** Drops a write or close the reader no longer wants (it cancelled). */
+const ignore = () => {};
+
+/** What an id of the connection waits for: a process's output and exit, or a request's answer. */
+interface Waiter {
+  bytes(stream: number, bytes: Uint8Array): void;
+  answer(answer: RunnerAnswer): void;
+  /** The connection dropped. */
+  lost(): void;
+}
+
+/** An open connection to the runner, and what its ids wait for. */
+interface Connection {
+  ws: WebSocket;
+  waiting: Map<number, Waiter>;
+}
+
+/** How long the runner's WebSocket may take to open, in ms. */
+const OPEN_MS = 30_000;
+
+/**
+ * The box `id` whose runner is at `url` (`http(s)://`, the WebSocket's upgrade sent with
+ * `headers`): its commands, files and `stop` (which kills every command in it and closes the
+ * connection). Connects at the first call, and again at the next call after a drop or a `stop`.
+ * How its ports are reached is the vendor's, so the backend adds `expose`.
+ */
+export function runnerBox({
+  url,
+  headers,
+  id,
+}: {
+  url: string;
+  headers?: Record<string, string>;
+  id: string;
+}): Omit<Box, "expose"> {
+  const where = `the runner at ${new URL(url).host}`;
+  let current: Promise<Connection> | undefined;
+  let last = 0;
+
+  const connect = () => (current ??= open());
+  function open(): Promise<Connection> {
+    const opening = new Promise<Connection>((resolve, reject) => {
+      // Node's WebSocket (undici) takes `headers` beside the standard's protocols.
+      const ws = new WebSocket(url.replace(/^http/, "ws"), { headers } as unknown as string[]);
+      ws.binaryType = "arraybuffer";
+      const connection: Connection = { ws, waiting: new Map() };
+      const late = setTimeout(() => ws.close(), OPEN_MS);
+      ws.addEventListener("open", () => {
+        clearTimeout(late);
+        resolve(connection);
+      });
+      ws.addEventListener("message", ({ data }: { data: string | ArrayBuffer }) => {
+        try {
+          if (typeof data !== "string") {
+            const { id: of, stream, bytes } = unframe(new Uint8Array(data));
+            return connection.waiting.get(of)?.bytes(stream, bytes);
+          }
+          const answer = JSON.parse(data) as RunnerAnswer;
+          if (answer.id !== undefined) connection.waiting.get(answer.id)?.answer(answer);
+        } catch {
+          // A frame that is not the protocol's (another runner's) is dropped.
+        }
+      });
+      ws.addEventListener("close", ({ code, reason }) => {
+        clearTimeout(late);
+        if (current === opening) current = undefined;
+        reject(new Error(`${where} closed the connection (${reason || code})`));
+        for (const waiter of connection.waiting.values()) waiter.lost();
+        connection.waiting.clear();
+      });
+    });
+    return opening;
+  }
+
+  /** Sends `request` (and `bytes` after it): resolves with the file it answers, null for none, or nothing for `ok`. */
+  async function ask(
+    request: { op: "read"; path: string } | { op: "write"; path: string; mode?: number } | { op: "stop" },
+    bytes?: Uint8Array,
+  ): Promise<Uint8Array | null | undefined> {
+    const { ws, waiting } = await connect();
+    const n = ++last;
+    return new Promise((resolve, reject) => {
+      const settle = (done: () => void) => {
+        waiting.delete(n);
+        done();
+      };
+      waiting.set(n, {
+        bytes: (_, file) => settle(() => resolve(file)),
+        answer: (answer) =>
+          settle(() =>
+            answer.op === "error"
+              ? reject(new Error(`${where}: ${answer.message}`))
+              : resolve(answer.op === "absent" ? null : undefined),
+          ),
+        lost: () => reject(new Error(`${where} closed the connection`)),
+      });
+      ws.send(JSON.stringify({ ...request, id: n }));
+      if (bytes) ws.send(frame(n, FILE, bytes));
+    });
+  }
+
+  return {
+    id,
+    async run(command, { cwd, env, stdin = false, signal } = {}): Promise<Process> {
+      const { ws, waiting } = await connect();
+      const n = ++last;
+      const stdout = new TransformStream<Uint8Array, Uint8Array>();
+      const stderr = new TransformStream<Uint8Array, Uint8Array>();
+      const out = stdout.writable.getWriter();
+      const err = stderr.writable.getWriter();
+      const exited = new Promise<{ exitCode: number }>((done) => {
+        const end = (exitCode: number) => {
+          waiting.delete(n);
+          out.close().catch(ignore);
+          err.close().catch(ignore);
+          done({ exitCode });
+        };
+        waiting.set(n, {
+          bytes: (stream, bytes) => void (stream === STDERR ? err : out).write(bytes).catch(ignore),
+          answer(answer) {
+            if (answer.op === "exit") return end(answer.code);
+            if (answer.op !== "error") return;
+            err.write(encoder.encode(`${where}: ${answer.message}\n`)).catch(ignore);
+            end(127);
+          },
+          lost: () => end(255),
+        });
+      });
+      ws.send(JSON.stringify({ op: "run", id: n, command, cwd, env, stdin } satisfies RunnerRequest));
+      const kill = (sig: NodeJS.Signals = "SIGKILL") => {
+        if (waiting.has(n)) ws.send(JSON.stringify({ op: "kill", id: n, signal: sig } satisfies RunnerRequest));
+      };
+      const onAbort = () => kill();
+      if (signal?.aborted) onAbort();
+      signal?.addEventListener("abort", onAbort, { once: true });
+      void exited.then(() => signal?.removeEventListener("abort", onAbort));
+      const endInput = () => void ws.send(JSON.stringify({ op: "stdin-end", id: n } satisfies RunnerRequest));
+      return {
+        stdin: new WritableStream<Uint8Array>({
+          write(bytes) {
+            if (!stdin) throw new Error("its stdin is closed: run it with stdin: true");
+            ws.send(frame(n, STDIN, bytes));
+          },
+          close: endInput,
+          abort: endInput,
+        }),
+        stdout: stdout.readable,
+        stderr: stderr.readable,
+        wait: () => exited,
+        kill,
+      };
+    },
+    read: async (path) => (await ask({ op: "read", path })) ?? null,
+    async write(path, bytes, { mode } = {}) {
+      await ask({ op: "write", path, mode }, bytes);
+    },
+    async stop() {
+      // This connection, not one a call opens meanwhile after a drop.
+      const closing = connect();
+      try {
+        await ask({ op: "stop" });
+      } finally {
+        // The next call opens a new connection, not this closing one.
+        if (current === closing) current = undefined;
+        (await closing.catch(() => undefined))?.ws.close();
+      }
+    },
+  };
+}

package/src/runner-protocol.ts

@@ -0,0 +1,61 @@
+// The runner's protocol, over one WebSocket per box, shared by the runner (`sandbox-runner.ts`)
+// and its client (`runner-box.ts`): text frames for control, as JSON, each carrying the `id` of
+// the process or the request it is about; binary frames for bytes, `frame(id, stream, bytes)`.
+
+/** The port the runner listens on in a box. */
+export const RUNNER_PORT = 7700;
+
+/**
+ * Where the runner answers its health: not `/health`, which Agent37's edge answers itself on
+ * every port, so that path never reaches what listens there.
+ */
+export const RUNNER_HEALTH = "/runner/health";
+
+/**
+ * The protocol's version, which the runner's health answers: a backend replaces a runner
+ * that answers another, one an earlier release started in a box that slept.
+ */
+export const RUNNER_PROTOCOL = 2;
+
+/** What the runner's health (`RUNNER_HEALTH`) answers: its protocol, and its pid, which a backend replacing it ends. */
+export interface RunnerHealth {
+  protocol: number;
+  pid: number;
+}
+
+/** What a binary frame's bytes are: a process's stdin, stdout or stderr, or a file read or written. */
+export const STDIN = 0;
+export const STDOUT = 1;
+export const STDERR = 2;
+export const FILE = 3;
+
+/** What the client says, as text. A `write`'s bytes follow it as the frame `(id, FILE)`. */
+export type RunnerRequest =
+  | { op: "run"; id: number; command: string; cwd?: string; env?: Record<string, string>; stdin: boolean }
+  | { op: "stdin-end"; id: number }
+  | { op: "kill"; id: number; signal: NodeJS.Signals }
+  | { op: "read"; id: number; path: string }
+  | { op: "write"; id: number; path: string; mode?: number }
+  | { op: "stop"; id: number };
+
+/** What the runner answers, as text. A `read` that finds a file answers the frame `(id, FILE)`. */
+export type RunnerAnswer =
+  | { op: "exit"; id: number; code: number }
+  | { op: "absent"; id: number }
+  | { op: "ok"; id: number }
+  | { op: "error"; id?: number; message: string };
+
+/** A binary frame: `id` (4 bytes), `stream` (1 byte), then `bytes`. */
+export function frame(id: number, stream: number, bytes: Uint8Array): Uint8Array<ArrayBuffer> {
+  const framed = new Uint8Array(5 + bytes.length);
+  new DataView(framed.buffer).setUint32(0, id);
+  framed[4] = stream;
+  framed.set(bytes, 5);
+  return framed;
+}
+
+export const unframe = (framed: Uint8Array) => ({
+  id: new DataView(framed.buffer, framed.byteOffset, framed.byteLength).getUint32(0),
+  stream: framed[4]!,
+  bytes: framed.subarray(5),
+});

package/src/sandbox-contract.ts

@@ -0,0 +1,121 @@
+import type { z } from "zod";
+import { hasPath } from "./host.ts";
+import type { LoginContext } from "./op.ts";
+
+// The `Sandbox` contract: what a resource type that runs a coding agent's box (an Agent37
+// instance) declares about itself, beside `Host`, `Database` and `Wakeup`, and the backend its
+// connector attaches (`ConnectorSetup.sandbox`) to open one box per name, so boxes are swappable
+// and one conformance kit decides whether one works. Vendor- and library-neutral: web streams,
+// no AI SDK. (`sandbox.ts` is something else: a scenario's run over the fakes.)
+
+/** What a sandbox can do, for choosing one: declared, never checked at runtime. */
+export interface SandboxCapabilities {
+  /** A box's files, its home's too, survive `stop` and the next `open` of its name. */
+  snapshot: boolean;
+  /** A stopped box keeps its memory and processes, and resumes them on the next `open`. */
+  pause: boolean;
+  /** `expose` makes a port of the box reachable from outside it. */
+  port: boolean;
+}
+
+/** What a sandbox's state says of it. */
+export interface SandboxOutputs {
+  id: string;
+  healthy: boolean;
+}
+
+/**
+ * A resource type's `sandbox`: the fields its boxes' choices are in, by dotted path into its
+ * declared fields, and how its state reads. The vendor's own facts (its region, its type of
+ * machine) are fields of the resource, never read from the environment.
+ */
+export interface SandboxSpec {
+  capabilities: SandboxCapabilities;
+  inputs: {
+    /** The image a box runs. */
+    image: string;
+    /** A box's size, in vCPUs: its memory follows the vendor's shapes. */
+    size: string;
+  };
+  outputs: (state: Record<string, unknown>) => SandboxOutputs;
+}
+
+/** A process `Box.run` started: `bash -c <command>`. */
+export interface Process {
+  /** Its stdin: closed at once unless `run` was given `stdin: true`. */
+  stdin: WritableStream<Uint8Array>;
+  /** Its stdout, as it prints, ended once it has exited. */
+  stdout: ReadableStream<Uint8Array>;
+  stderr: ReadableStream<Uint8Array>;
+  /** Resolves when it has exited: non-zero when it failed or was killed. */
+  wait(): Promise<{ exitCode: number }>;
+  /** Kills it and what it started (default `SIGKILL`). */
+  kill(signal?: NodeJS.Signals): void;
+}
+
+/**
+ * A box: a computer of its own a coding agent works in. A relative `path` or `cwd` is under the
+ * box's home (`$HOME`), which a backend with `snapshot` keeps across `stop`.
+ */
+export interface Box {
+  /** The vendor's id of it. */
+  id: string;
+  /** Runs `bash -c <command>`, in `cwd`, with `env` over the box's own. An abort kills it. */
+  run(
+    command: string,
+    opts?: { cwd?: string; env?: Record<string, string>; stdin?: boolean; signal?: AbortSignal },
+  ): Promise<Process>;
+  /** A file's bytes; null when there is none. */
+  read(path: string): Promise<Uint8Array | null>;
+  /** Writes a file, its directories made. */
+  write(path: string, bytes: Uint8Array, opts?: { mode?: number }): Promise<void>;
+  /** Where its `port` is reached from outside it, with the headers that reach it. */
+  expose(port: number): Promise<{ url: string; headers?: Record<string, string> }>;
+  /** Stops it and every process in it; the next `open` of its name finds what it kept. */
+  stop(): Promise<void>;
+}
+
+/**
+ * A connector's boxes (`ConnectorSetup.sandbox`). A box is keyed by `name`, under which the
+ * vendor keeps its state, so no snapshot handle crosses the contract.
+ */
+export interface SandboxBackend {
+  /**
+   * The box `name`, made from `image` with `size` vCPUs when there is none (left out, the backend's
+   * own), woken or started when stopped.
+   */
+  open(opts: { name: string; image?: string; size?: number }): Promise<Box>;
+  /** Deletes the box `name` and whatever the vendor kept of it: the next `open` finds it empty. */
+  discard(opts: { name: string }): Promise<void>;
+}
+
+/**
+ * What the app gives a connector's sandbox half (`ConnectorSetup.sandbox`): the secrets, as a model
+ * half reads them, and the declared fields of the resource whose boxes it opens, which hold the
+ * vendor's facts (its region, its type of machine) beside the image and size `open` takes.
+ */
+export type SandboxContext = Pick<LoginContext, "secrets"> & { declared: Record<string, unknown> };
+
+/**
+ * Throws what is wrong with a resource type's `sandbox`: a type apply cannot change (not
+ * `writable`), a capability that is not a boolean, an input that names no field, `outputs` that
+ * is not a function.
+ */
+export function checkSandbox(
+  what: string,
+  sandbox: SandboxSpec,
+  type: { schema: z.ZodObject; writable?: boolean },
+): void {
+  const problems: string[] = [];
+  if (!type.writable) problems.push("it must be writable, so apply can create and update it");
+  const capabilities: Partial<SandboxCapabilities> = sandbox.capabilities ?? {};
+  for (const name of ["snapshot", "pause", "port"] as const) {
+    if (typeof capabilities[name] !== "boolean") problems.push(`capabilities.${name} must be true or false`);
+  }
+  for (const input of ["image", "size"] as const) {
+    const path: unknown = sandbox.inputs?.[input];
+    if (typeof path !== "string" || !hasPath(type.schema, path)) problems.push(`${input} names no field (${path})`);
+  }
+  if (typeof sandbox.outputs !== "function") problems.push("outputs must be a function of the state");
+  if (problems.length) throw new Error(`${what}: its sandbox: ${problems.join("; ")}`);
+}

package/src/sandbox-runner.ts

@@ -0,0 +1,187 @@
+import { createServer } from "node:http";
+import type { AddressInfo } from "node:net";
+import { homedir } from "node:os";
+import { setTimeout as delay } from "node:timers/promises";
+import { type WebSocket, WebSocketServer } from "ws";
+import { localBox } from "./local-sandbox.ts";
+import {
+  FILE,
+  frame,
+  RUNNER_HEALTH,
+  RUNNER_PORT,
+  RUNNER_PROTOCOL,
+  type RunnerAnswer,
+  type RunnerHealth,
+  type RunnerRequest,
+  STDERR,
+  STDIN,
+  STDOUT,
+  unframe,
+} from "./runner-protocol.ts";
+import type { Process } from "./sandbox-contract.ts";
+import { errorMessage } from "./shared.ts";
+
+// The runner: the server of ours a backend starts inside a remote box, which `runnerBox` talks to
+// over a WebSocket (the protocol is in `runner-protocol.ts`). It serves this machine as the local
+// backend's box does (`localBox`, whose code it bundles): commands are `bash -c` in process groups
+// of their own, a relative path or `cwd` is under the home. Built into one file,
+// `dist/sandbox-runner.cjs` (`ws` bundled in), run as `node sandbox-runner.cjs`.
+//
+// It listens on `SANOMA_RUNNER_HOST:SANOMA_RUNNER_PORT` (default `0.0.0.0:7700`) and asks no
+// credential: the vendor's edge authenticates every request to the port, the upgrade too
+// (Agent37: the API key, on a port never made public). Anything inside the box already runs as
+// the box's user, so a secret of the runner's own would guard nothing.
+
+const port = Number(process.env.SANOMA_RUNNER_PORT ?? RUNNER_PORT);
+const host = process.env.SANOMA_RUNNER_HOST ?? "0.0.0.0";
+
+/**
+ * How long an exited command's output may go quiet before its exit is sent, in ms: `localBox` has
+ * waited for its pipes already, unless what it left running holds them. Output held back for a
+ * slow connection is not quiet.
+ */
+const LAST_OUTPUT = 50;
+/** How much a connection may hold unsent before a command's output waits for it, in bytes. */
+const UNSENT = 1 << 20;
+/** How often a connection is pinged: Agent37's edge closes one silent for 15 minutes. */
+const BEAT = 30_000;
+
+// Its commands' environment is its own, but for where it listens.
+const { SANOMA_RUNNER_PORT: _, SANOMA_RUNNER_HOST: __, ...own } = process.env;
+const box = localBox("runner", homedir(), own as Record<string, string>);
+
+const server = createServer((req, res) => {
+  if (req.url !== RUNNER_HEALTH) return void res.writeHead(404).end();
+  res
+    .writeHead(200, { "content-type": "application/json" })
+    .end(JSON.stringify({ protocol: RUNNER_PROTOCOL, pid: process.pid } satisfies RunnerHealth));
+});
+new WebSocketServer({ server })
+  .on("connection", serve)
+  .on("error", (error) => console.error(`the runner's WebSocket server: ${error.message}`));
+server.listen(port, host, () => {
+  const { address, port: bound } = server.address() as AddressInfo;
+  console.log(`sanoma runner listening on ${address}:${bound}`);
+});
+// Asked to end (the box shuts down): what it started ends with it.
+process.on("SIGTERM", () => void box.stop().finally(() => process.exit(0)));
+
+/** One client's connection: its commands, by the ids it gave them. */
+function serve(ws: WebSocket) {
+  // Promises: a command's kill and its stdin may come before it has started, and wait for it.
+  const processes = new Map<number, Promise<Process>>();
+  const inputs = new Map<number, Promise<WritableStreamDefaultWriter<Uint8Array>>>();
+  /** The writes whose bytes are yet to come. */
+  const writes = new Map<number, { path: string; mode?: number }>();
+  const send = (data: Uint8Array | string) => ws.readyState === ws.OPEN && ws.send(data);
+  const say = (answer: RunnerAnswer) => send(JSON.stringify(answer));
+  const beat = setInterval(() => ws.ping(), BEAT);
+  // A frame ws refuses (one too large, one malformed) closes this connection, not the runner.
+  ws.on("error", (error) => console.error(`a connection: ${error.message}`));
+  ws.on("close", () => {
+    clearInterval(beat);
+    // A dropped connection kills nothing: its commands go on, their stdin ended, their output dropped.
+    for (const id of inputs.keys()) endInput(id);
+  });
+  ws.on("message", (data: Buffer, binary) => {
+    // A frame that is not the protocol's is answered, never a crash of the runner.
+    if (binary) {
+      if (data.length < 5) return say({ op: "error", message: "a binary frame is an id, a stream and bytes" });
+      return received(unframe(data));
+    }
+    let request: RunnerRequest;
+    try {
+      request = JSON.parse(String(data));
+    } catch {
+      return say({ op: "error", message: "a text frame is JSON" });
+    }
+    if (typeof request !== "object" || request === null) return say({ op: "error", message: "a request is an object" });
+    handle(request).catch((error: unknown) => say({ op: "error", id: request.id, message: errorMessage(error) }));
+  });
+
+  async function handle(request: RunnerRequest): Promise<unknown> {
+    switch (request.op) {
+      case "run": {
+        const { id, command, cwd, env, stdin } = request;
+        const starting = box.run(command, { cwd, env, stdin });
+        processes.set(id, starting);
+        if (stdin) {
+          const input = starting.then((started) => started.stdin.getWriter());
+          // A run that fails to start is answered below; its stdin's rejection is not another.
+          input.catch(() => {});
+          inputs.set(id, input);
+        }
+        try {
+          const started = await starting;
+          const moved = { at: Date.now(), done: false };
+          const pumped = Promise.all([
+            pump(id, STDOUT, started.stdout, moved),
+            pump(id, STDERR, started.stderr, moved),
+          ]).finally(() => (moved.done = true));
+          const { exitCode } = await started.wait();
+          // What it printed last goes out first, unless what it left running holds its pipes.
+          moved.at = Date.now();
+          while (!moved.done && Date.now() - moved.at < LAST_OUTPUT) await Promise.race([pumped, delay(10)]);
+          return say({ op: "exit", id, code: exitCode });
+        } finally {
+          processes.delete(id);
+          inputs.delete(id);
+        }
+      }
+      case "stdin-end":
+        return endInput(request.id);
+      case "kill":
+        return processes.get(request.id)?.then((started) => started.kill(request.signal));
+      case "read": {
+        const file = await box.read(request.path);
+        return file ? send(frame(request.id, FILE, file)) : say({ op: "absent", id: request.id });
+      }
+      case "write":
+        return writes.set(request.id, request);
+      case "stop":
+        await box.stop();
+        return say({ op: "ok", id: request.id });
+      default:
+        throw new Error(`no such request: ${(request as { op?: unknown }).op}`);
+    }
+  }
+
+  function received({ id, stream, bytes }: ReturnType<typeof unframe>) {
+    if (stream === STDIN)
+      return void inputs
+        .get(id)
+        ?.then((writer) => writer.write(bytes))
+        .catch(() => {});
+    const write = writes.get(id);
+    if (stream !== FILE || !write) return;
+    writes.delete(id);
+    box.write(write.path, bytes, { mode: write.mode }).then(
+      () => say({ op: "ok", id }),
+      (error: unknown) => say({ op: "error", id, message: errorMessage(error) }),
+    );
+  }
+
+  /** Ends the stdin of the command `id`, once it has started. */
+  function endInput(id: number) {
+    const input = inputs.get(id);
+    inputs.delete(id);
+    return input?.then((writer) => writer.close()).catch(() => {});
+  }
+
+  /** Sends what `from` prints, until it ends or fails, keeping in `moved` when it last sent or waited to. */
+  async function pump(id: number, stream: number, from: ReadableStream<Uint8Array>, moved: { at: number }) {
+    try {
+      for await (const chunk of from) {
+        // Its output waits for a slow connection, and so the command does, its pipe full.
+        while (ws.readyState === ws.OPEN && ws.bufferedAmount > UNSENT) {
+          moved.at = Date.now();
+          await delay(10);
+        }
+        send(frame(id, stream, chunk));
+        moved.at = Date.now();
+      }
+    } catch {
+      // A pipe that failed has nothing more to say.
+    }
+  }
+}

package/src/scenario.ts

@@ -12,6 +12,7 @@ import {
   type Step as GherkinStep,
 } from "@cucumber/messages";
 import type { z } from "zod";
+import { targetOf } from "./call.ts";
 import type { RunSummary, SanomaClient } from "./client.ts";
 import type { Approver, Principal } from "./define.ts";
 import type { ResolvedConfig } from "./config.ts";
@@ -44,10 +45,12 @@ import {
   DATA_VENDOR,
   declaresPeople,
   DRIFT_WORKFLOW,
+  entryOpsOf,
   errorMessage,
   fitsOp,
   flatten,
   groupNamesOf,
+  isBatch,
   isEnded,
   principalIdsOf,
   problemLine,
@@ -446,6 +449,12 @@ function rulesOf(scope: Scope): { rules: Rule[]; parameterTypes: ParameterTypeSp
     { name: "field", regexp: /\S+/, transformer: (s) => s },
   ];
   const opOf = (id: unknown) => scope.ops.get(id as string)!;
+  /** An operation a `Then` step reads the ledger for: never a batch, which the ledger records as its entries. */
+  const recordedOp = (id: unknown): string => {
+    const op = opOf(id);
+    if (isBatch(op)) throw new Error(`${op.id} is recorded as its entries: name ${entryOpsOf(op).join(" or ")}`);
+    return op.id;
+  };
   const workflowOf = (name: unknown) => scope.workflows.get(name as string)!;
   // The people as the root's data files declare them when a step names one (`readPeople`, read
   // again only once a file changed): while they declare no one, any id and any group go.
@@ -596,33 +605,33 @@ function rulesOf(scope: Scope): { rules: Rule[]; parameterTypes: ParameterTypeSp
       source: "{op} was called with",
       takes: TAKES_OBJECT,
       read: ([op], pickle) =>
-        expecting({ op: op as string, input: fieldsFrom(pickle, opOf(op).input, inputOf(op), true), called: true }),
+        expecting({ op: recordedOp(op), input: fieldsFrom(pickle, opOf(op).input, inputOf(op), true), called: true }),
     },
     ...[true, false].map((called): Rule => ({
       kind: "then",
       source: `{op} was ${called ? "" : "not "}called`,
-      read: ([op]) => expecting({ op: op as string, called }),
+      read: ([op]) => expecting({ op: recordedOp(op), called }),
     })),
     {
       kind: "then",
       source: "{op} was denied",
-      read: ([op]) => expecting({ op: op as string, decision: "denied" }),
+      read: ([op]) => expecting({ op: recordedOp(op), decision: "denied" }),
     },
     {
       kind: "then",
       source: "{op} was denied with {string}",
-      read: ([op, reason]) => expecting({ op: op as string, decision: "denied", reason: reason as string }),
+      read: ([op, reason]) => expecting({ op: recordedOp(op), decision: "denied", reason: reason as string }),
     },
     {
       kind: "then",
       source: "{op} was held for {who}",
-      read: ([op, who]) => expecting({ op: op as string, decision: "held", approver: who as string }),
+      read: ([op, who]) => expecting({ op: recordedOp(op), decision: "held", approver: who as string }),
     },
     {
       kind: "then",
       source: "{op} was held for group {string}",
       read: ([op, group]) =>
-        expecting({ op: op as string, decision: "held", approver: { group: groupNamed(group as string) } }),
+        expecting({ op: recordedOp(op), decision: "held", approver: { group: groupNamed(group as string) } }),
     },
     {
       kind: "then",
@@ -1207,6 +1216,7 @@ function testWorld(built: Built, { driver, fake, live }: FeatureTarget) {
             record({
               type: "op.called",
               op: op.id,
+              ...targetOf(op, input),
               effect: op.effect,
               input,
               decision: { kind: "allow" },