@jr2/orchestrator 0.3.0 → 0.4.1

package/src/http.ts

@@ -14,8 +14,20 @@
 //   # the Agent's Harness, through its Custodian, and nothing else
 //   GET  /agents/:iid/surface          accepts + schemas + semantics             [Sandbox token]
 //   POST /agents/:iid/events           validate + deliver → the turn receipt      [Sandbox token]
+//   GET  /agents/:iid/deliveries/:id   long-poll an outstanding pick's answer    [Sandbox token]
 //   POST /sandboxes/:name/fetch        ask the node cache to fetch one Repo      [Sandbox token]
 //
+//   # named Charts (ADR-0070): a Chart under a name is a Workflow `<wrapper>/<name>`
+//   GET    /workflows/:wrapper/chart-schema           the grammar, narrowed to the Open body    [open]
+//   PUT    /workflows/:wrapper/charts/:name           admit + keep a runtime Chart              [Instance token]
+//   DELETE /workflows/:wrapper/charts/:name           remove a runtime Chart                    [Instance token]
+//   POST   /workflows/:wrapper/charts/:name/runs      start the named Chart                     [Instance token]
+//   GET    /workflows/:name/runs/:runId/machine       the run's bound Machine, Charts compiled  [open]
+//
+//   # `jr2 up`, and nothing else (ADR-0069)
+//   PUT  /assembly                     publish the next Assembly                 [publish token]
+//   GET  /assembly                     the current Assembly and the held ones    [open]
+//
 // The token is not decoration: an Agent has code execution in its Harness container and shares the
 // pod's network namespace, so it can reach these routes. A Sandbox token may deliver ONLY to an
 // agent surface recorded against its own Sandbox — never to a Gate. That is what stops an Agent
@@ -41,16 +53,21 @@ import { Hono } from "hono";
 import type { Context, MiddlewareHandler } from "hono";
 import { accepts } from "hono/accepts";
 import { streamSSE } from "hono/streaming";
-import { EventValidationError, UnknownAddressError } from "./registration.ts";
+import { EventValidationError, NotTakenError, OutstandingError, UnknownAddressError } from "./registration.ts";
 import type { RepoStatus } from "./repos.ts";
 import { UnmountedRepoError, type FetchAnswer } from "./repo-fetch.ts";
 import { mayAskForSandbox, mayDeliverToAgent, type Authenticator, type Principal } from "./tokens.ts";
-import { observe, type RunHost } from "./run-host.ts";
+import { observe, type RunFeedEvent, type RunHost, type RunStatus, type WorkflowFeedEvent } from "./run-host.ts";
 import { KIT_VERSION } from "./config.ts";
+import type { AssemblyDoor } from "./assembly.ts";
+import { ChartRefusal } from "./chart.ts";
+import { chartName, NamedChartError, splitChartName, type NamedCharts } from "./charts.ts";
 
 /** A `POST /runs/:id/events` body: the down-channel event. CANCEL is all that is left of it
- * (ADR-0013): APPROVE and STEER rode the deferred/poll machinery, which is reserved, not built. */
-type RunEventBody = { type?: string };
+ * (ADR-0013): APPROVE and STEER rode the deferred/poll machinery: `deferred`
+ * is retired (ADR-0006), `poll` reserved but not built.
+ * `keep` leaves the cancelled run's Workspaces for inspection instead of destroying them (ADR-0025). */
+type RunEventBody = { type?: string; keep?: unknown };
 
 /** `GET /runs/resolve` floor — a 1-char prefix is a table scan, not a question. The CLI enforces
  * the same floor on the argument; this one guards the scan regardless of who is calling. */
@@ -77,6 +94,15 @@ function errMessage(err: unknown): string {
   return err instanceof Error ? err.message : String(err);
 }
 
+/** The 503 body while a deferred restore is pending (ADR-0026, ADR-0069): ask again. */
+function restorePendingMessage(what: string): string {
+  return `the Orchestrator has not restored its runs yet (no current Assembly) — ask again for ${what}`;
+}
+
+/** Why the publish token is refused everywhere but `PUT /assembly` (ADR-0069): it is not an
+ * Instance principal, so it reads no run, starts none, and speaks for no Agent. */
+const PUBLISHER_ONLY = "the publish token publishes an Assembly (PUT /assembly) and does nothing else";
+
 /** The app's context: `authenticated` resolves the bearer to a `principal` the routes read back. */
 type JR2Env = { Variables: { principal: Principal } };
 
@@ -231,36 +257,141 @@ const PING_MS = 15_000;
  * is the `error: terminated` an attached `jr2 run` hit). The frame is an SSE COMMENT (`:\n\n`):
  * every client ignores it, so it needs no place in the wire vocabulary.
  *
- * It is deliberately a **ping**, not a heartbeat or keepalive — CONTEXT.md puts both on the Lease's
+ * It is deliberately a **ping**, not a heartbeat or keepalive — GLOSSARY.md puts both on the Lease's
  * Avoid list, and this asserts nothing and expects no answer.
  *
  * It is NOT a liveness probe, and must not be mistaken for one: `stream.write` swallows its own
  * errors (hono's `StreamingApi`), so a failed write is indistinguishable from a good one. A peer
  * that goes away is detected by `stream.onAbort`, which is what every handler here wires to its
  * exit. The tick checks `aborted`/`closed` only so a ping that fires between the abort and the
- * teardown does not write into a dead stream. Returns its own clear fn.
+ * teardown does not write into a dead stream.
+ *
+ * The tick is also where a subscriber is judged (ADR-0022): a reader {@link Subscriber.behind} is
+ * closed here, and the ping itself goes out through the subscriber, so a reader stalled on a quiet
+ * feed falls behind too. Returns its own clear fn.
  */
-function pinger(stream: SseStream, done: () => void, everyMs = PING_MS): () => void {
+function pinger(stream: SseStream, sub: Subscriber, done: () => void, everyMs = PING_MS): () => void {
   const timer = setInterval(() => {
     if (stream.aborted || stream.closed) return done();
-    void stream.write(":\n\n");
+    if (sub.behind()) {
+      // Aborted, not drained: the frames it would never read are dropped with it.
+      stream.abort();
+      return done();
+    }
+    void sub.send(":\n\n");
   }, everyMs);
   return () => clearInterval(timer);
 }
 
-/** The slice of hono's `StreamingApi` the ping needs: the raw write (`writeSSE` cannot express a
- *  comment frame) plus the two flags that say the peer is gone. */
-type SseStream = { write: (s: string) => Promise<unknown>; aborted: boolean; closed: boolean };
+/** The slice of hono's `StreamingApi` the feeds need: the raw write (a frame is rendered here, and
+ *  `writeSSE` cannot express a comment frame), the two flags that say the peer is gone, and the
+ *  abort that closes a reader that fell behind. */
+type SseStream = { write: (s: string) => Promise<unknown>; aborted: boolean; closed: boolean; abort: () => void };
+
+/**
+ * How many writes one subscriber may leave unread across a whole ping interval before it is closed
+ * (ADR-0022). A reader this far behind is not catching up, and holding frames for it is a per-client
+ * queue the server does not keep: the feeds are level-triggered, so closing it loses no state — the
+ * reader reconnects (EventSource on its own retry, the CLI on the stream's end) and the opening
+ * frame is the whole current truth.
+ */
+const BACKLOG_FRAMES = 64;
+
+/**
+ * One SSE frame, spelled as hono's `writeSSE` spells it. JSON has no raw newline, so the payload is
+ * always one `data:` line.
+ */
+function sseFrame(event: string, data: unknown, retry?: number): string {
+  return `event: ${event}\ndata: ${JSON.stringify(data)}\n${retry ? `retry: ${retry}\n` : ""}\n`;
+}
+
+/**
+ * A run that left the live set WITHOUT settling — stopped, so parked for the next boot's restore
+ * (ADR-0025) — reads through as `active`, which is the truth about the Machine and says nothing
+ * about the feed. `gone` is a fact, not an inference (ADR-0022): stated here, so a reader that
+ * re-attaches whenever a feed ends before a terminal status (the CLI) knows this end is the run's
+ * own and does not re-attach in a loop.
+ */
+async function sayGoneIfParked(
+  stream: { writeSSE: (m: { event: string; data: string }) => Promise<unknown> },
+  status: RunStatus,
+): Promise<void> {
+  if (status.status !== "active") return;
+  await stream.writeSSE({ event: "gone", data: JSON.stringify({ runId: status.runId }) });
+}
+
+/**
+ * Render each feed event ONCE, however many subscribers share it (ADR-0022). The host hands every
+ * listener of a feed the same event object, so the object is the key, and the frame is released
+ * with it. `undefined` is a rendering too — "this band does not carry it" — and is cached alike.
+ */
+function renderOnce<E extends object, F>(render: (event: E) => F): (event: E) => F {
+  const frames = new WeakMap<E, F>();
+  return (event) => {
+    if (frames.has(event)) return frames.get(event) as F;
+    const frame = render(event);
+    frames.set(event, frame);
+    return frame;
+  };
+}
+
+/** One subscriber's writes, and the one question asked of them at each ping. */
+type Subscriber = {
+  /** Write a shared frame as-is. */
+  send: (frame: string) => Promise<void>;
+  /** Whether {@link BACKLOG_FRAMES} writes sent before the previous ping are still unread. */
+  behind: () => boolean;
+};
+
+/**
+ * One subscriber (ADR-0022): the only state kept per client is a count of its writes. A reader is
+ * judged on LAG, never on burst size — one transition's Emits, or a Pool's statuses, reach every
+ * subscriber in one tick, and a reader that keeps up drains them long before the next ping. What is
+ * still unread a whole interval after it was sent is a reader that fell behind.
+ */
+function subscriber(stream: SseStream): Subscriber {
+  let sent = 0;
+  let read = 0;
+  let sentByLastPing = 0;
+  return {
+    async send(frame) {
+      sent++;
+      try {
+        await stream.write(frame);
+      } finally {
+        read++; // a stream's writes settle in order, so this counts a prefix
+      }
+    },
+    behind() {
+      const unread = sentByLastPing - read;
+      sentByLastPing = sent;
+      return unread >= BACKLOG_FRAMES;
+    },
+  };
+}
 
 export type CreateAppOptions = {
   /** Ping interval for the SSE feeds. Tests shorten it; nothing in production sets it. */
   pingMs?: number;
   /**
-   * Whether this instance has a data plane at all (ADR-0051): a registered Machine composes a
-   * Sandbox AND the process is deployed in a cluster. `false` is an answer — "this instance runs
-   * no Workspace" — and `jr2 status` says so rather than listing nothing.
+   * Whether this instance has a data plane at all (ADR-0051, ADR-0069): the process is deployed in
+   * a cluster, so it wires its kube client whatever the current Assembly composes. `false` is an
+   * answer — "this instance runs no Workspace" — and `jr2 status` says so rather than listing
+   * nothing.
    */
   dataPlane?: boolean;
+  /**
+   * The publish door and the current Assembly (ADR-0069). Absent for the in-process tests that
+   * build an app straight over a host: `GET /assembly` then answers no current Assembly, and
+   * `PUT /assembly` answers 404.
+   */
+  assembly?: AssemblyDoor;
+  /**
+   * The digests of the ConfigMaps this process reads through a mount (ADR-0069), as
+   * `readMountDigests` reads them. `GET /assembly` reports them, so `jr2 up` can wait for the
+   * kubelet before it publishes. Absent when not deployed: nothing is mounted.
+   */
+  mounts?: () => Promise<Record<string, string>>;
   /**
    * The Repo resources as the cluster currently reports them (ADR-0051), read per request off the
    * port — a cache the agent cloned minutes after boot reads as present the next time anyone asks.
@@ -275,6 +406,17 @@ export type CreateAppOptions = {
    * ask, and the route says so rather than waiting on a cluster that is not there.
    */
   fetchRepo?: (sandbox: string, identity: string) => Promise<FetchAnswer>;
+  /** How long one poll of an outstanding pick waits for its answer before it answers `outstanding`
+   * (ADR-0013, ADR-0024). Default 25 s: every request stays far below undici's 300 s and the
+   * Custodian's idle limit. Tests shorten it. */
+  deliveryWaitMs?: number;
+  /**
+   * The named Charts of the Instance (ADR-0070): what `PUT`/`DELETE /workflows/:wrapper/charts/:name`
+   * change, what `GET /workflows/:wrapper/chart-schema` narrows the grammar from, and the
+   * unrunnable ones `GET /workflows` adds to the host's listing. Absent for the in-process tests
+   * that build an app straight over a host: those routes then answer 404.
+   */
+  charts?: NamedCharts;
 };
 
 /**
@@ -294,6 +436,7 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
     if (!auth) return next(); // no authenticator configured: tests only (see the doc comment)
     const principal = auth(bearerOf(c));
     if (!principal) return c.json({ error: "unauthorized" }, 401);
+    if (principal.kind === "publisher") return c.json({ error: PUBLISHER_ONLY }, 403);
     c.set("principal", principal);
     return next();
   };
@@ -312,6 +455,7 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
     if (!auth) return next(); // no authenticator configured: tests only (see the doc comment)
     const principal = auth(bearerOf(c));
     if (!principal) return c.json({ error: "unauthorized" }, 401);
+    if (principal.kind === "publisher") return c.json({ error: PUBLISHER_ONLY }, 403);
     if (principal.kind !== "instance") {
       return c.json({ error: "run state and control need the Instance token — a Sandbox token has neither" }, 403);
     }
@@ -323,16 +467,166 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
   // readiness probe's target and because identity is not run state — it is the same class of thing
   // as the route table, which is public by being served. The CLI probes it to explain a failure it
   // could otherwise only report as a bare status code (version skew reads as a nonsense 404).
-  // `hash` is the image's content address (ADR-0019), absent for a host-booted fixture process,
-  // which has no image to be addressed.
-  app.get("/healthz", (c) => c.json({ ok: true, version: KIT_VERSION, hash: process.env.JR2_CONTENT_HASH }));
+  // `hash` is the current Assembly's (ADR-0069), absent while the process holds none.
+  app.get("/healthz", (c) => c.json({ ok: true, version: KIT_VERSION, hash: opts.assembly?.current()?.hash }));
   app.get("/readyz", (c) => c.json({ ready: true }));
 
   // Structure, not state: the workflow listing, a template's Machine, and the Console shell are
   // unauthenticated. They expose no run, drive nothing, and the Console is a BROWSER page — it
   // loads before any token is entered (ADR-0032). Everything that reads or moves a run is guarded
   // below.
-  app.get("/workflows", (c) => c.json(host.workflows()));
+  // Each Workflow with its origin, and each Chart wrapper's Charts inside its entry (ADR-0070); a
+  // named Chart that cannot run now is listed with why.
+  app.get("/workflows", (c) => {
+    const listing = host.workflowList();
+    return c.json(opts.charts ? opts.charts.listing(listing) : listing);
+  });
+
+  // The current Assembly and the ones live runs hold (ADR-0069). Open like `/workflows`: a hash
+  // and a Kit version are identity, the class of thing `/healthz` already serves.
+  // `mounts` is what this process reads from its mounted ConfigMaps, so a publish can wait until
+  // the kubelet has delivered the data the converge wrote.
+  app.get("/assembly", async (c) => {
+    const view = opts.assembly ? await opts.assembly.view() : { kitVersion: KIT_VERSION, current: null, held: [] };
+    return c.json(opts.mounts ? { ...view, mounts: await opts.mounts() } : view);
+  });
+
+  // The publish door (ADR-0069): the publish token ALONE. Publishing imports code into the process
+  // that holds the ServiceAccount, so the Instance token, which starts runs and answers Gates, is
+  // refused here by name. The checks after authentication are the door's (assembly.ts), each with
+  // its own status, and the old Assembly stays current through every refusal.
+  app.put("/assembly", async (c) => {
+    if (auth) {
+      const principal = auth(bearerOf(c));
+      if (!principal) return c.json({ error: "unauthorized" }, 401);
+      if (principal.kind !== "publisher") {
+        return c.json(
+          {
+            error:
+              "publishing an Assembly needs the publish token (JR2_PUBLISH_TOKEN in the instance Secret), which " +
+              "`jr2 up` reads through the kube API — the Instance token does not publish (ADR-0069)",
+          },
+          403,
+        );
+      }
+    }
+    if (!opts.assembly) return c.json({ error: "this instance has no publish door" }, 404);
+    const answer = await opts.assembly.publish({
+      kitVersion: c.req.header("jr2-kit-version"),
+      hash: c.req.header("jr2-assembly-hash"),
+      body: c.req.raw.body,
+    });
+    return c.json(answer.body, answer.status);
+  });
+
+  // ---- Named Charts (ADR-0070) ----------------------------------------------------------------
+  // Before every `/workflows/:name/*` route. A Chart is data and cannot reach the process, so a
+  // runtime Chart is kept behind the Instance token, not the publish token; the door that admits it
+  // is the one a run's Chart passes. A Chart file is the Assembly's own: no route changes one.
+
+  /** A `NamedChartError` as its answer. */
+  const chartError = (c: Context<JR2Env>, err: unknown): Response => {
+    if (err instanceof NamedChartError) return c.json(err.body, err.status);
+    throw err;
+  };
+  const noCharts = (c: Context<JR2Env>) => c.json({ error: "this instance keeps no named Charts" }, 404);
+
+  // The grammar narrowed to the wrapper's Open body: its lent names, guard, action and delay names,
+  // allowed models and Workspace accesses as `enum`s. Structure, like the Machine doc: open band.
+  app.get("/workflows/:wrapper/chart-schema", (c) => {
+    const wrapper = c.req.param("wrapper");
+    const schema = opts.charts?.schema(wrapper);
+    return schema ? c.json(schema) : c.json({ error: `no Chart wrapper "${wrapper}"` }, 404);
+  });
+
+  // The body is the Chart text alone. 201 when new, 200 when it replaced one: a live run of the
+  // old text holds its own.
+  app.put("/workflows/:wrapper/charts/:name", instanceOnly, async (c) => {
+    if (!opts.charts) return noCharts(c);
+    let text: unknown;
+    try {
+      text = JSON.parse(await c.req.text());
+    } catch (err) {
+      return c.json({ error: "refused", report: [`(chart): the body is not JSON: ${errMessage(err)}`] }, 422);
+    }
+    try {
+      const put = await opts.charts.put(c.req.param("wrapper"), c.req.param("name"), text);
+      // `unrunnable`: a publish bound a wrapper that refuses the text while it was written. It is
+      // kept on the volume and listed with that reason, as each publish's re-admission does.
+      return c.json(
+        {
+          name: put.name,
+          origin: "runtime",
+          hash: put.hash,
+          ...(put.unrunnable !== undefined ? { unrunnable: put.unrunnable } : {}),
+        },
+        put.created ? 201 : 200,
+      );
+    } catch (err) {
+      return chartError(c, err);
+    }
+  });
+
+  app.delete("/workflows/:wrapper/charts/:name", instanceOnly, async (c) => {
+    if (!opts.charts) return noCharts(c);
+    try {
+      await opts.charts.remove(c.req.param("wrapper"), c.req.param("name"));
+      return c.body(null, 204);
+    } catch (err) {
+      return chartError(c, err);
+    }
+  });
+
+  /**
+   * Start a run of a Workflow (ADR-0009). A named Chart that cannot run now is 409 with why
+   * (ADR-0070); a Chart in the input that the door refuses is 422 with the report; input failing
+   * the door schema is 400 (ADR-0033); an unknown Workflow is 404.
+   */
+  const startRun = async (c: Context<JR2Env>, name: string): Promise<Response> => {
+    const unrunnable = opts.charts?.get(name)?.unrunnable;
+    if (unrunnable !== undefined) return c.json({ error: `named Chart "${name}" cannot run: ${unrunnable}` }, 409);
+    const input = await readJson(c.req.text());
+    try {
+      const { runId, instanceId } = await host.start(name, input);
+      return c.json({ runId, instanceId }, 201);
+    } catch (err) {
+      if (err instanceof ChartRefusal) return c.json({ error: "refused", report: err.report }, 422);
+      if (err instanceof EventValidationError) return c.json({ error: errMessage(err) }, 400);
+      return c.json({ error: errMessage(err) }, 404);
+    }
+  };
+
+  // The documented start of a named Chart; `POST /workflows/<wrapper>%2F<name>/runs` is the same.
+  app.post("/workflows/:wrapper/charts/:name/runs", instanceOnly, (c) =>
+    startRun(c, chartName(c.req.param("wrapper"), c.req.param("name"))),
+  );
+
+  // The run's bound Machine (ADR-0070): the wrapper's doc with each bound body's Chart compiled in,
+  // so the Console draws the states a Chart run moves through. Structure: open band, and only
+  // through the Workflow the run is of, as the per-run feed is: an observer names the Workflow
+  // it already knows. A settled run's doc is served too, so the Console draws where it ended.
+  app.get("/workflows/:name/runs/:runId/machine", async (c) => {
+    const runId = c.req.param("runId");
+    const doc = await host.runMachine(runId);
+    return doc && doc.workflow === c.req.param("name")
+      ? c.json(doc)
+      : c.json({ error: `no run "${runId}" of workflow "${c.req.param("name")}"` }, 404);
+  });
+
+  /** A named Chart that is listed but cannot run now (ADR-0070): 409 with why, as its start is,
+   * never a 404 for a name `GET /workflows` lists. */
+  const unrunnableChart = (c: Context<JR2Env>, name: string): Response | undefined => {
+    const named = opts.charts?.get(name);
+    if (named?.unrunnable === undefined) return undefined;
+    return c.json(
+      {
+        error: `named Chart "${name}" cannot run: ${named.unrunnable}`,
+        unrunnable: named.unrunnable,
+        chart: { origin: named.origin, hash: named.hash },
+      },
+      409,
+    );
+  };
 
   // The two PAGE addresses (ADR-0032): `/` and `/workflows/:name` negotiate. A browser navigation
   // (`Accept` prefers text/html) gets the one Console shell — the path carries the selection, the
@@ -351,8 +645,19 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
     if (prefersHtml(c)) return consoleAsset("page.html", "text/html; charset=utf-8");
     const name = c.req.param("name");
     const doc = host.machine(name);
-    if (!doc) return c.json({ error: `no workflow "${name}"` }, 404);
-    return c.json({ name, machineId: doc.id, input: host.inputSchema(name) ?? null });
+    if (!doc) return unrunnableChart(c, name) ?? c.json({ error: `no workflow "${name}"` }, 404);
+    // A named Chart answers its door minus the Chart fields, and its origin and hash; a Chart
+    // wrapper answers its Open bodies' declarations, for a client that writes Charts (ADR-0070).
+    const split = splitChartName(name);
+    const named = split && (opts.charts?.get(name) ?? host.namedChart(split.wrapper, split.name));
+    const wrapper = split ? undefined : opts.charts?.wrapperOf(name);
+    return c.json({
+      name,
+      machineId: doc.id,
+      input: host.inputSchema(name) ?? null,
+      ...(named ? { chart: { origin: named.origin, hash: named.hash } } : {}),
+      ...(wrapper ? { openBodies: wrapper.bodies.map((b) => ({ slot: b.slot, ...b.declaration })) } : {}),
+    });
   });
 
   // Push work: start a run of a registered workflow. Unknown workflow → host.start throws → 404.
@@ -361,24 +666,16 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
   // schema (`POST /runs/:id/gates/:gate/events` below). Instance-band (ADR-0058): starting work is
   // run control, and a Sandbox token's scope is its own agent surface — the Custodian's route
   // allowlist not forwarding this route is a second layer, not the control (ADR-0059).
-  app.post("/workflows/:name/runs", instanceOnly, async (c) => {
-    const name = c.req.param("name");
-    const input = await readJson(c.req.text());
-    try {
-      const { runId, instanceId } = await host.start(name, input);
-      return c.json({ runId, instanceId }, 201);
-    } catch (err) {
-      if (err instanceof EventValidationError) return c.json({ error: errMessage(err) }, 400);
-      return c.json({ error: errMessage(err) }, 404);
-    }
-  });
+  // A Chart wrapper's one-shot door (ADR-0070): a Chart field in the body is admitted before the
+  // run starts, and a refusal is 422 with the report. A named Chart rides the path encoded.
+  app.post("/workflows/:name/runs", instanceOnly, (c) => startRun(c, c.req.param("name")));
 
   // The registered template Machine's structure — what the Console's diagram renders (structure
   // is provider-independent, so the un-`provide()`d template is exactly right).
   app.get("/workflows/:name/machine", (c) => {
     const name = c.req.param("name");
     const doc = host.machine(name);
-    return doc ? c.json(doc) : c.json({ error: `no workflow "${name}"` }, 404);
+    return doc ? c.json(doc) : (unrunnableChart(c, name) ?? c.json({ error: `no workflow "${name}"` }, 404));
   });
 
   // ---- Observation (`GET /workflows/:name/runs*`) ---------------------------------------------
@@ -395,6 +692,18 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
 
   app.get("/workflows/:name/runs", (c) => c.json(host.observations(c.req.param("name"))));
 
+  // The workflow feed, rendered once per event for every page attached to it (ADR-0022): the
+  // `observe()` projection and its JSON happen once, not once per open page.
+  const workflowFrame = renderOnce((ev: Exclude<WorkflowFeedEvent, { kind: "closed" }>) => {
+    if (ev.kind === "gone") return sseFrame("gone", { runId: ev.runId });
+    // The TYPE alone: an emit's payload is author data, the same class of thing as context.
+    if (ev.kind === "emit") return sseFrame("emit", { runId: ev.runId, type: ev.event.type });
+    // `{ child, attempt }` only — `reason` is mechanism/error text, which stays behind the
+    // Instance token like `fault` (ADR-0014/0016).
+    if (ev.kind === "retry") return sseFrame("retry", { runId: ev.runId, child: ev.child, attempt: ev.attempt });
+    return sseFrame("status", observe(ev.status));
+  });
+
   /**
    * SSE: a whole WORKFLOW's activity (ADR-0022) — every run of it appearing, moving, emitting and
    * leaving, on one connection that outlives all of them.
@@ -416,40 +725,46 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
     return streamSSE(c, async (stream) => {
       await new Promise<void>((resolve) => {
         const exit = closer(resolve);
+        const sub = subscriber(stream);
         // Subscribe and snapshot in one call, then write the snapshot in the same tick: nothing can
         // start, move or finish in between, so the client's first frame is a complete picture.
         const { runs, unsubscribe } = host.observeWorkflow(name, (ev) => {
           if (ev.kind === "closed") return exit.done();
-          if (ev.kind === "gone") {
-            void stream.writeSSE({ event: "gone", data: JSON.stringify({ runId: ev.runId }) });
-            return;
-          }
-          if (ev.kind === "emit") {
-            // The TYPE alone: an emit's payload is author data, the same class of thing as context.
-            void stream.writeSSE({ event: "emit", data: JSON.stringify({ runId: ev.runId, type: ev.event.type }) });
-            return;
-          }
-          if (ev.kind === "retry") {
-            // `{ child, attempt }` only — `reason` is mechanism/error text, which stays behind the
-            // Instance token like `fault` (ADR-0014/0016).
-            void stream.writeSSE({
-              event: "retry",
-              data: JSON.stringify({ runId: ev.runId, child: ev.child, attempt: ev.attempt }),
-            });
-            return;
-          }
-          void stream.writeSSE({ event: "status", data: JSON.stringify(observe(ev.status)) });
+          void sub.send(workflowFrame(ev));
         });
         exit.onExit(unsubscribe);
         // `retry` steers the browser's own EventSource backoff. This feed never ends on its own, so
-        // every close is a fault worth reconnecting from — the client does not decide that.
-        void stream.writeSSE({ event: "runs", data: JSON.stringify(runs.map(observe)), retry: 2000 });
-        exit.onExit(pinger(stream, exit.done, pingMs));
+        // every close is a fault worth reconnecting from — the client does not decide that. The
+        // snapshot is this subscriber's own, so it is the one frame rendered per client.
+        void sub.send(sseFrame("runs", runs.map(observe), 2000));
+        exit.onExit(pinger(stream, sub, exit.done, pingMs));
         stream.onAbort(exit.done);
       });
     });
   });
 
+  // The per-run feed in the OPEN band, rendered once per event for all its subscribers (ADR-0022).
+  // `undefined` = this band does not carry it.
+  const openRunFrame = renderOnce((ev: Exclude<RunFeedEvent, { kind: "closed" }>) => {
+    // The TYPE alone: an emit's payload is author data, the same class of thing as context.
+    if (ev.kind === "emit") return sseFrame("emit", { type: ev.event.type });
+    // `{ child, attempt }` only — `reason` is mechanism/error text, which stays behind the
+    // Instance token like `fault` (ADR-0014/0016).
+    if (ev.kind === "retry") return sseFrame("retry", { child: ev.child, attempt: ev.attempt });
+    // Turn markers (ADR-0023) carry an Agent's framing and pick payload — Instance-token class, so
+    // the OPEN band never sees them (not even their types).
+    if (ev.kind === "admission" || ev.kind === "pick") return undefined;
+    // A wait for capacity's lines carry the scheduler's words (ADR-0064): behind the token. The
+    // open band sees `placing` lit in the status's children, and nothing more.
+    if (ev.kind === "placing" || ev.kind === "placed") return undefined;
+    // A child's error below an Agent's state (ADR-0024) is mechanism text, like `fault`: behind the
+    // token.
+    if (ev.kind === "child-error") return undefined;
+    // An Effect's lines (ADR-0067) carry argv and output — working data: behind the token.
+    if (ev.kind === "effect") return undefined;
+    return sseFrame("status", observe(ev.status));
+  });
+
   app.get("/workflows/:name/runs/:runId/events", async (c) => {
     const name = c.req.param("name");
     const runId = c.req.param("runId");
@@ -460,28 +775,17 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
     return streamSSE(c, async (stream) => {
       await new Promise<void>((resolve) => {
         const exit = closer(resolve);
+        const sub = subscriber(stream);
         exit.onExit(
           host.subscribe(runId, (ev) => {
             // The host is shutting down under a feed that has no end of its own.
             if (ev.kind === "closed") return exit.done();
-            if (ev.kind === "emit") {
-              // The TYPE alone: an emit's payload is author data, the same class of thing as context.
-              void stream.writeSSE({ event: "emit", data: JSON.stringify({ type: ev.event.type }) });
-              return;
-            }
-            if (ev.kind === "retry") {
-              // `{ child, attempt }` only — `reason` is mechanism/error text, which stays behind
-              // the Instance token like `fault` (ADR-0014/0016).
-              void stream.writeSSE({ event: "retry", data: JSON.stringify({ child: ev.child, attempt: ev.attempt }) });
-              return;
-            }
-            // Turn markers (ADR-0023) carry an Agent's framing and pick payload — Instance-token
-            // class, so the OPEN band never sees them (not even their types).
-            if (ev.kind === "admission" || ev.kind === "pick") return;
+            const frame = openRunFrame(ev);
+            if (frame === undefined) return;
             // Terminal frame must flush before the handler returns and closes the stream (see the
             // guarded feed below for why the exit is chained off the write).
-            const terminal = ev.status.status !== "active";
-            void stream.writeSSE({ event: "status", data: JSON.stringify(observe(ev.status)) }).then(() => {
+            const terminal = ev.kind === "status" && ev.status.status !== "active";
+            void sub.send(frame).then(() => {
               if (terminal) exit.done();
             });
           }),
@@ -489,7 +793,7 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
         // Race guard: settled between the liveness check and the subscribe, which then attached to
         // nothing. No read-through on this route, so there is nothing to fall back to — just close.
         if (host.status(runId) === undefined) return exit.done();
-        exit.onExit(pinger(stream, exit.done, pingMs));
+        exit.onExit(pinger(stream, sub, exit.done, pingMs));
         stream.onAbort(exit.done);
       });
     });
@@ -587,10 +891,38 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
     } catch (err) {
       if (err instanceof UnknownAddressError) return c.json({ error: errMessage(err) }, 404);
       if (err instanceof EventValidationError) return c.json({ error: errMessage(err) }, 400);
+      // The run would not take it from this Gate now (ADR-0013): never `{ ok: true }`. The body
+      // names what the Gate takes now, as `GET /runs/:id` lists it.
+      if (err instanceof NotTakenError) {
+        const runId = c.req.param("runId");
+        const gate = c.req.param("gate");
+        const acceptsNow = host.gates(runId).find((g) => g.gate === gate)?.acceptsNow ?? [];
+        return c.json({ error: errMessage(err), acceptsNow }, 409);
+      }
       throw err;
     }
   });
 
+  // The per-run feed in the Instance band, rendered once per event for all its subscribers
+  // (ADR-0022).
+  const runFrame = renderOnce((ev: Exclude<RunFeedEvent, { kind: "closed" }>) => {
+    if (ev.kind === "emit") return sseFrame("emit", ev.event);
+    // The Instance's own feed: the full telemetry, reason included (same trust class as `fault`).
+    if (ev.kind === "retry") return sseFrame("retry", ev);
+    // Turn markers (ADR-0023) — Instance-token band, so the framing/payload ride whole. These stay
+    // OFF the open workflow feed entirely (run-host.ts feeds them per-run).
+    if (ev.kind === "admission" || ev.kind === "pick") return sseFrame(ev.kind, ev);
+    // A wait for capacity starting and ending (ADR-0064) — Instance-token band too: the message is
+    // the scheduler's or the quota's own words.
+    if (ev.kind === "placing" || ev.kind === "placed") return sseFrame(ev.kind, ev);
+    // A child's error below an Agent's state, which the Agent's answer does not carry (ADR-0024):
+    // the operator reads it here, with its message.
+    if (ev.kind === "child-error") return sseFrame(ev.kind, ev);
+    // An Effect's lines (ADR-0067): its start, its stream, its output, an abort — Instance-token band.
+    if (ev.kind === "effect") return sseFrame("effect", ev);
+    return sseFrame("status", ev.status);
+  });
+
   // SSE: a live run streams its status deltas + author `emit`s (current status replayed on attach,
   // then live until the terminal transition or client abort). A run that has already settled streams
   // its final status once and closes (so `jr2 logs -f` works on a finished run). Unknown run → 404.
@@ -601,35 +933,21 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
       if (!finalStatus) return c.json({ error: `no run "${runId}"` }, 404);
       return streamSSE(c, async (stream) => {
         await stream.writeSSE({ event: "status", data: JSON.stringify(finalStatus) });
+        await sayGoneIfParked(stream, finalStatus);
       });
     }
     return streamSSE(c, async (stream) => {
       await new Promise<void>((resolve) => {
         const exit = closer(resolve);
+        const sub = subscriber(stream);
         exit.onExit(
           host.subscribe(runId, (ev) => {
             if (ev.kind === "closed") return exit.done();
-            if (ev.kind === "emit") {
-              void stream.writeSSE({ event: "emit", data: JSON.stringify(ev.event) });
-              return;
-            }
-            if (ev.kind === "retry") {
-              // The Instance's own feed: the full telemetry, reason included (same trust class as
-              // `fault`).
-              void stream.writeSSE({ event: "retry", data: JSON.stringify(ev) });
-              return;
-            }
-            if (ev.kind === "admission" || ev.kind === "pick") {
-              // Turn markers (ADR-0023) — Instance-token band, so the framing/payload ride whole.
-              // These stay OFF the open workflow feed entirely (run-host.ts feeds them per-run).
-              void stream.writeSSE({ event: ev.kind, data: JSON.stringify(ev) });
-              return;
-            }
             // Exiting lets the handler return, which CLOSES the stream — so on the terminal frame we
             // must wait for the write to flush first, or a fire-and-forget write races the close and the
             // final status is dropped (the very frame `jr2 run` blocks on). Chain the exit off the write.
-            const terminal = ev.status.status !== "active";
-            void stream.writeSSE({ event: "status", data: JSON.stringify(ev.status) }).then(() => {
+            const terminal = ev.kind === "status" && ev.status.status !== "active";
+            void sub.send(runFrame(ev)).then(() => {
               if (terminal) exit.done();
             });
           }),
@@ -638,24 +956,27 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
         // which then attaches to nothing and never fires. Fall back to the terminal read-through.
         if (host.status(runId) === undefined) {
           exit.release();
-          void host.read(runId).then((s) => {
-            if (s) void stream.writeSSE({ event: "status", data: JSON.stringify(s) });
+          void host.read(runId).then(async (s) => {
+            if (s) {
+              await stream.writeSSE({ event: "status", data: JSON.stringify(s) });
+              await sayGoneIfParked(stream, s);
+            }
             exit.done();
           });
           return;
         }
-        exit.onExit(pinger(stream, exit.done, pingMs));
+        exit.onExit(pinger(stream, sub, exit.done, pingMs));
         stream.onAbort(exit.done);
       });
     });
   });
 
   // Run control (ADR-0002). CANCEL is the only event left on this seam: APPROVE and STEER answered
-  // held `deferred` calls and drained `poll` inboxes, and ADR-0013 reserves both semantics without
-  // building them. Workflow-defined events reach a run through its GATES, not through here.
+  // held calls and drained `poll` inboxes; a held call is now every pick answered at Rest (ADR-0024),
+  // and `poll` stays reserved (ADR-0013). Workflow-defined events reach a run through its GATES.
   //
-  // It ENDS the run (ADR-0025): the Agents' turns end with it and the run does not come back on
-  // the next restore. `RunHost.stop()` — park it, keep it restorable — is a different verb, and
+  // It ENDS the run (ADR-0025): the Agents' turns end with it, the run does not come back on the
+  // next restore, and its Workspaces are destroyed unless the body says `keep: true`. `RunHost.stop()` — park it, keep it restorable — is a different verb, and
   // deliberately not on the wire.
   app.post("/runs/:runId/events", instanceOnly, async (c) => {
     const runId = c.req.param("runId");
@@ -663,8 +984,12 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
     if (body.type !== "CANCEL") {
       return c.json({ error: `unknown event type "${body.type ?? ""}" (accepts: CANCEL)` }, 400);
     }
+    // Whether to keep is a fact of the moment of the cancel, never a config key (ADR-0025).
+    if (body.keep !== undefined && typeof body.keep !== "boolean") {
+      return c.json({ error: `"keep" must be a boolean` }, 400);
+    }
     try {
-      await host.cancel(runId);
+      await host.cancel(runId, { keep: body.keep === true });
       return c.json({ ok: true });
     } catch (err) {
       return c.json({ error: errMessage(err) }, 404);
@@ -685,7 +1010,23 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
   /** Guard: the surface must exist, and this principal must be allowed to speak for it. */
   const agentRegistration = (c: Context<JR2Env, "/agents/:instanceId/surface" | "/agents/:instanceId/events">) => {
     const instanceId = c.req.param("instanceId");
+    // Only a Turn that is over answers 404 (ADR-0026). While the host is stopping its runs are
+    // parked for restore, not over, so every `/agents/*` call answers 503: the Harness re-asks a
+    // surface read on ADR-0042's ladder and tells a pick to try again. BEFORE the lookup, because
+    // a stopping host may already have lost the registration the question is about.
+    if (host.stopping) {
+      return {
+        error: c.json({ error: `the Orchestrator is stopping — ask again for instance "${instanceId}"` }, 503),
+      };
+    }
     const surface = host.agentSurface(instanceId);
+    // A boot with no current Assembly listens before it restores (ADR-0069): until the first
+    // publish restores the waiting runs, a registration this host does not have may be one of
+    // theirs, and that Turn is not over either. AFTER the lookup, so a run that is hosted already
+    // (re-attached, or started under the published Assembly) is served.
+    if (!surface && host.restorePending) {
+      return { error: c.json({ error: restorePendingMessage(`instance "${instanceId}"`) }, 503) };
+    }
     // The one catch point (ADR-0011): no live registration (settled run, exited state, unknown
     // iid) → there is no surface to serve. 404 BEFORE the scope check — a caller with a valid
     // token learns nothing from it that it did not already know.
@@ -698,18 +1039,21 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
     return { surface };
   };
 
-  // This turn's menu: the events the invoking state accepts, their input schemas, their semantics.
-  // A transition swaps the registration, which swaps this — so the Harness gets a state-scoped
-  // toolset for free, and needs no `list_changed` to know it (flue re-lists on every submission).
+  // The Agent's Menu — every pick it can be offered in this Machine, with input schemas and
+  // semantics, fixed for the conversation — and this Turn's Allowed picks (ADR-0029). A transition
+  // swaps the registration, which swaps the Allowed picks, never the Menu; the Harness reads both
+  // per Submission and needs no `list_changed`.
   app.get("/agents/:instanceId/surface", authenticated, (c) => {
     const { surface, error } = agentRegistration(c);
-    return error ?? c.json(surface);
+    if (error) return error;
+    host.agentSurfaceServed(c.req.param("instanceId"));
+    return c.json(surface);
   });
 
   // The Agent's pick, delivered into the state that invoked it. The receipt describes itself
   // (ADR-0024): what was delivered, and whether that ended the turn — which the Harness renders as
-  // prose. Its `deliveryId` still makes an outcome addressable after the fact, the room a deferred
-  // result will need when it lands.
+  // prose. Its `deliveryId` makes an outstanding pick addressable after the fact: its answer is
+  // polled at `GET /agents/:iid/deliveries/:id`.
   app.post("/agents/:instanceId/events", authenticated, async (c) => {
     const { error } = agentRegistration(c);
     if (error) return error;
@@ -718,11 +1062,59 @@ export function createApp(host: RunHost, auth?: Authenticator, opts: CreateAppOp
       return c.json(host.sendToAgent(c.req.param("instanceId"), body));
     } catch (err) {
       if (err instanceof UnknownAddressError) return c.json({ error: errMessage(err) }, 404);
-      if (err instanceof EventValidationError) return c.json({ error: errMessage(err) }, 400);
+      // One pick at a time (ADR-0024): refused, nothing delivered, and nothing is allowed until the
+      // workflow is back at Rest. `outstanding` names the pick still running, when there is one; the
+      // Harness writes the words.
+      if (err instanceof OutstandingError) {
+        return c.json(
+          {
+            error: errMessage(err),
+            refused: "outstanding",
+            ...(err.outstanding !== undefined ? { outstanding: err.outstanding } : {}),
+            allowed: [],
+          },
+          409,
+        );
+      }
+      if (err instanceof EventValidationError) {
+        // The Allowed picks ride the refusal (ADR-0029), so "Allowed now: …" can follow it.
+        return c.json({ error: errMessage(err), ...(err.allowed ? { allowed: err.allowed } : {}) }, 400);
+      }
       throw err;
     }
   });
 
+  // An outstanding pick's answer (ADR-0013, ADR-0024), long-polled: the POST answered at once with a
+  // delivery id, and the Harness asks here inside the pick's one tool call, up to `deliveryWaitMs`
+  // per request, until the workflow is back at Rest in the Agent's state or has left it. A GET is
+  // safe to repeat, so a reset or a restarting Orchestrator costs a retry, never the pick: a stopping
+  // or parked host answers 503 — ask again — and the restored host answers from the saved hold.
+  // Scoped like the pick: the Sandbox the registration had when the pick was made, because the
+  // answer outlives a registration that a route out of the Agent's state ends.
+  app.get("/agents/:instanceId/deliveries/:deliveryId", authenticated, async (c) => {
+    const instanceId = c.req.param("instanceId");
+    const deliveryId = c.req.param("deliveryId");
+    if (host.stopping || host.parkedDelivery(instanceId, deliveryId)) {
+      return c.json({ error: `the Orchestrator is stopping — ask again for delivery "${deliveryId}"` }, 503);
+    }
+    const scope = host.deliveryScope(instanceId, deliveryId);
+    // Not restored yet is not over (ADR-0026, ADR-0069): the restored host answers from the hold.
+    if (!scope && host.restorePending) {
+      return c.json({ error: restorePendingMessage(`delivery "${deliveryId}"`) }, 503);
+    }
+    if (!scope) return c.json({ error: `no delivery "${deliveryId}" for instance "${instanceId}"` }, 404);
+    if (!mayDeliverToAgent(principalOf(c), scope.sandbox)) {
+      return c.json({ error: `this token cannot speak for instance "${instanceId}"` }, 403);
+    }
+    const receipt = await host.awaitDelivery(instanceId, deliveryId, opts.deliveryWaitMs, c.req.raw.signal);
+    if (host.stopping) {
+      return c.json({ error: `the Orchestrator is stopping — ask again for delivery "${deliveryId}"` }, 503);
+    }
+    return receipt
+      ? c.json(receipt)
+      : c.json({ error: `no delivery "${deliveryId}" for instance "${instanceId}"` }, 404);
+  });
+
   // The pod's one route out (ADR-0053). A `git fetch` inside a Sandbox runs a program on the
   // runtime volume, which asks the Custodian on localhost, which forwards to this. What lands here
   // is one identity; what goes back is the landing, or the cache as it stands with git's own words