@jr2/orchestrator 0.3.0 → 0.5.1

package/src/size.ts

@@ -0,0 +1,454 @@
+// A Workspace's SIZE (ADR-0060): the ceiling it states for its whole Sandbox, cpu and memory and
+// nothing else, which jr2 reserves in full. The shape is Agent Substrate's `ActorTemplate.resources`
+// — a Kubernetes `ResourceRequirements` of which only `limits.cpu` and `limits.memory` are
+// accepted — so a Workspace moves there unchanged.
+//
+// This module is the arithmetic and the refusals, with no Kubernetes client and no Machine in it:
+//
+//   - `assertSize` refuses anything but the two limits WHERE IT IS READ (`workspace()`,
+//     `customize()`, `jr2.config.ts`), naming the field — a request below the limit, a missing
+//     memory limit, or a key Substrate refuses would make the Machine depend on something the move
+//     drops.
+//   - `resolveSize` walks the chain, most specific first, one field at a time: the Machine's Size
+//     (a `customize()` layered over the `workspace()`), then the Instance's `sandbox.resources` — a
+//     default for a Workspace that states none, never an override of one it states — then the kit.
+//   - `splitSize` divides the Size inside the pod: the Custodian's fixed slice, the User
+//     Container's split if the Machine states one, and the Harness gets the rest as its own limit,
+//     so an OOM stays per container where it can. It refuses a split that leaves the Harness below
+//     its floor, naming where each number came from, because the fix is an edit of one of them.
+//
+// Both the provision (sandbox-kube.ts) and `jr2 up` call the same two functions, so a Size the
+// converge admits is a Size the provision composes.
+
+/** A Kubernetes quantity: `"2Gi"`, `"1500m"`, `"2"`, or a plain number (`2` cpus, bytes of memory). */
+export type Quantity = string | number;
+
+// A Size (ADR-0060, GLOSSARY.md "Size"): `limits.memory` and `limits.cpu`, nothing else. A field
+// left out resolves down the chain (`resolveSize`) — the Instance's `sandbox.resources`, then the
+// kit default (`KIT_SIZE`). The same shape is the User Container's split (`userSeatOf`,
+// `splitSize`), where a field left out means the seat has no limit of its own for it and shares
+// the pod's budget.
+/**
+ * The Size of a Workspace: the most memory and cpu that its whole Sandbox can use. jr2 reserves
+ * all of it on the node. Thus a Sandbox never gets less than its Size and never uses more.
+ *
+ * Write it as `{ limits: { memory, cpu } }`. You can write it in three places: the `resources`
+ * option of `workspace()`, `customize(machine, { resources })`, and `sandbox.resources` in
+ * `jr2.config.ts`. The Size is a fact of the Machine. The Instance's `sandbox.resources` applies
+ * only to a Workspace that states no Size. A field that neither states comes from the kit default:
+ * memory `2Gi`, cpu `1`.
+ *
+ * The same shape is the User Container's part of the Size, in `user: { image, resources }`. There,
+ * a field you leave out gives the User Container no limit of its own: it shares the pod's budget.
+ *
+ * It refuses any key but `limits.memory` and `limits.cpu`, where you write it. The message names
+ * the field: `<where>` is where you wrote the Size, for example `workspace(): resources`.
+ *
+ * - `<where>.requests is not accepted — jr2 reserves the whole Size, so every request equals its
+ *   limit — a Size is { limits: { memory?, cpu? } } and nothing else — Agent Substrate's template
+ *   size (ADR-0060)`
+ * - `<where>.<key> is not accepted — a Size is { limits: { memory?, cpu? } } and nothing else —
+ *   Agent Substrate's template size (ADR-0060)`
+ * - `<where>.limits.<key> is not accepted — a Size is { limits: { memory?, cpu? } } and nothing
+ *   else — Agent Substrate's template size (ADR-0060)`
+ * - `<where>.limits.memory: <value> is not a positive quantity (e.g. "2Gi", "512Mi")`, and for
+ *   cpu, `<where>.limits.cpu: <value> is not a positive quantity (e.g. "2", "1500m")`
+ *
+ * `jr2 up` also refuses a Size that leaves the Harness container less than its floor. The message
+ * starts with `the Size leaves the Harness container <memory> memory, below its 256Mi floor:` or
+ * with `the Size leaves the Harness container no cpu:`. The text that follows names each number
+ * and where it came from.
+ *
+ * @example
+ * ```ts
+ * import { customize, type Size } from "@jr2/orchestrator";
+ * import { task } from "@jr2/machines";
+ *
+ * const size: Size = { limits: { memory: "4Gi", cpu: "2" } };
+ *
+ * export const machine = customize(task, {
+ *   repos: { app: { url: "https://github.com/you/app.git" } },
+ *   agents: { coder: { model: "anthropic/claude-haiku-5-5" } },
+ *   resources: size,
+ * });
+ * ```
+ *
+ * @see docs/guides/size-a-workspace.md
+ * @see docs/concepts/workspaces.md
+ * @see docs/reference/decisions/0060-a-workspace-states-its-size-and-jr2-reserves-all-of-it.md
+ */
+export type Size = {
+  /** The two limits. Write `limits: {}` to state no field: each field then comes from the next
+   * source. */
+  limits: {
+    /** The memory limit, as a Kubernetes quantity: `"4Gi"`, `"512Mi"`, or a number of bytes.
+     * It must be more than zero. When you leave it out, it comes from the next source, as above. */
+    memory?: Quantity;
+    /** The cpu limit, as a Kubernetes quantity: `"2"`, `"1500m"`, or a number of cpus. It must be
+     * more than zero. jr2 enforces it. When you leave it out, it comes from the next source, as
+     * above. */
+    cpu?: Quantity;
+  };
+};
+
+/** The kit's default Size (ADR-0060). Node gives each process a V8 heap of about 55% of its
+ * container limit, and `tsc` on a mid-size repo aborted below about 1.1Gi, so 1Gi fails a common
+ * task; 2Gi covers `tsc` + vitest, `cargo build` and Playwright at 2 workers. */
+export const KIT_SIZE = { memory: "2Gi", cpu: "1" } as const;
+
+/** The Custodian's fixed slice of every Sandbox's Size (ADR-0060). */
+export const CUSTODIAN_SLICE = { memory: "64Mi", cpu: "50m" } as const;
+
+/** The Harness container's floor: a split that leaves it less memory than this is refused. */
+export const HARNESS_FLOOR = "256Mi";
+
+/** `/dev/shm` is 25% of the Harness container's share, never below this (ADR-0060). */
+export const SHM_FLOOR = "64Mi";
+
+/** The `/work` disk: an ephemeral-storage REQUEST on the Harness container, with no limit and no
+ * `sizeLimit` — passing either evicts the pod and loses `/work` (ADR-0060). Kit-fixed. */
+export const WORK_DISK_REQUEST = "10Gi";
+
+/** The preflight init step's resources (ADR-0037): small and fixed, and inside the pod's budget —
+ * an init container runs before the others start, so it borrows the pod's Size, it adds nothing. */
+export const PREFLIGHT_RESOURCES = {
+  requests: { cpu: "10m", memory: "32Mi" },
+  limits: { memory: "128Mi" },
+} as const;
+
+/** A container's or a pod's resources, in the Kubernetes (corev1.ResourceRequirements) shape. */
+export type ResourceRequirements = {
+  requests?: Record<string, string>;
+  limits?: Record<string, string>;
+};
+
+// --- quantities --------------------------------------------------------------------------------
+
+const QUANTITY = /^(\d+(?:\.\d+)?|\.\d+)(m|k|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$/;
+const SCALE: Record<string, number> = {
+  "": 1,
+  m: 1e-3,
+  k: 1e3,
+  M: 1e6,
+  G: 1e9,
+  T: 1e12,
+  P: 1e15,
+  E: 1e18,
+  Ki: 2 ** 10,
+  Mi: 2 ** 20,
+  Gi: 2 ** 30,
+  Ti: 2 ** 40,
+  Pi: 2 ** 50,
+  Ei: 2 ** 60,
+};
+
+/** The quantity as a plain number of base units, or undefined when it is not one Kubernetes reads.
+ * Zero and negatives are not a Size: a zero limit is a container that cannot start. */
+function baseUnits(q: unknown): number | undefined {
+  if (typeof q === "number") return Number.isFinite(q) && q > 0 ? q : undefined;
+  if (typeof q !== "string") return undefined;
+  const m = QUANTITY.exec(q.trim());
+  if (!m) return undefined;
+  const value = Number(m[1]) * SCALE[m[2] ?? ""]!;
+  return value > 0 ? value : undefined;
+}
+
+/** Memory in bytes, rounded up as Kubernetes rounds it. */
+export function memoryBytes(q: Quantity): number {
+  const v = baseUnits(q);
+  if (v === undefined) throw new Error(`${JSON.stringify(q)} is not a memory quantity (e.g. "2Gi", "512Mi")`);
+  return Math.ceil(v - 1e-9);
+}
+
+/** CPU in millicores, rounded up to 1m as Kubernetes rounds it. */
+export function cpuMillis(q: Quantity): number {
+  const v = baseUnits(q);
+  if (v === undefined) throw new Error(`${JSON.stringify(q)} is not a cpu quantity (e.g. "2", "1500m")`);
+  return Math.ceil(v * 1000 - 1e-9);
+}
+
+/** Bytes as the quantity a human would write: `Gi`, `Mi` or `Ki` when exact, else plain bytes. */
+export function formatMemory(bytes: number): string {
+  for (const [unit, scale] of [
+    ["Gi", 2 ** 30],
+    ["Mi", 2 ** 20],
+    ["Ki", 2 ** 10],
+  ] as const) {
+    if (bytes % scale === 0) return `${bytes / scale}${unit}`;
+  }
+  return String(bytes);
+}
+
+/** Millicores as the quantity a human would write: whole cpus when exact, else `m`. */
+export function formatCpu(millis: number): string {
+  return millis % 1000 === 0 ? String(millis / 1000) : `${millis}m`;
+}
+
+// --- the refusal where a Size is read ----------------------------------------------------------
+
+/**
+ * Refuse anything but a Size, BY FIELD (ADR-0060): `requests`, any key beside `limits`, any limit
+ * beside `memory` and `cpu`, or a value Kubernetes would not read. `where` names the reader and the
+ * field's own path (`workspace(): resources`, `jr2.config.ts at sandbox.resources`), so the message
+ * points at the text to edit. An empty `limits` is legal and states nothing: every field resolves
+ * down the chain.
+ */
+export function assertSize(where: string, value: unknown): asserts value is Size {
+  const hint = "a Size is { limits: { memory?, cpu? } } and nothing else — Agent Substrate's template size (ADR-0060)";
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new Error(`${where}: expected an object (got ${JSON.stringify(value)}) — ${hint}`);
+  }
+  for (const key of Object.keys(value)) {
+    if (key === "limits") continue;
+    const why =
+      key === "requests"
+        ? "is not accepted — jr2 reserves the whole Size, so every request equals its limit"
+        : "is not accepted";
+    throw new Error(`${where}.${key} ${why} — ${hint}`);
+  }
+  const limits = (value as { limits?: unknown }).limits;
+  if (typeof limits !== "object" || limits === null || Array.isArray(limits)) {
+    throw new Error(`${where}.limits: expected an object (got ${JSON.stringify(limits)}) — ${hint}`);
+  }
+  for (const [key, q] of Object.entries(limits)) {
+    if (key !== "memory" && key !== "cpu") {
+      throw new Error(`${where}.limits.${key} is not accepted — ${hint}`);
+    }
+    if (q === undefined) continue;
+    if (baseUnits(q) === undefined) {
+      const eg = key === "memory" ? '"2Gi", "512Mi"' : '"2", "1500m"';
+      throw new Error(`${where}.limits.${key}: ${JSON.stringify(q)} is not a positive quantity (e.g. ${eg})`);
+    }
+  }
+}
+
+/** Layer one Size over another, field by field — `customize(machine, { resources })` over the
+ * `workspace()`'s own, as an Agent override layers over its stock definition. */
+export function layerSize(base: Size | undefined, over: Size | undefined): Size | undefined {
+  if (over === undefined) return base;
+  if (base === undefined) return over;
+  return { limits: { ...base.limits, ...over.limits } };
+}
+
+// --- resolution --------------------------------------------------------------------------------
+
+/** Where a resolved number came from — what a refusal names, because the fix is an edit THERE. */
+export type SizeSource = "machine" | "instance" | "kit";
+
+/** A Size resolved to numbers, each with its source. */
+export type ResolvedSize = {
+  memory: { bytes: number; from: SizeSource };
+  cpu: { millis: number; from: SizeSource };
+};
+
+/**
+ * The chain (ADR-0060): the Machine's Size (its `workspace()`, with any `customize()` already
+ * layered over it), else the Instance's `sandbox.resources`, then the kit for a field neither
+ * states. The Instance default applies only to a Workspace that states NO Size — never field by
+ * field under a stated one — so a Machine's Size resolves the same on every Instance, and each
+ * fact has one override path, `customize()`.
+ */
+export function resolveSize(machine: Size | undefined, instance: Size | undefined): ResolvedSize {
+  const stated = (size: Size | undefined) =>
+    size !== undefined && (size.limits.memory !== undefined || size.limits.cpu !== undefined);
+  const [chosen, source]: [Size | undefined, SizeSource] = stated(machine)
+    ? [machine, "machine"]
+    : [instance, "instance"];
+  const pick = (field: "memory" | "cpu"): { q: Quantity; from: SizeSource } => {
+    const q = chosen?.limits[field];
+    return q !== undefined ? { q, from: source } : { q: KIT_SIZE[field], from: "kit" };
+  };
+  const memory = pick("memory");
+  const cpu = pick("cpu");
+  return {
+    memory: { bytes: memoryBytes(memory.q), from: memory.from },
+    cpu: { millis: cpuMillis(cpu.q), from: cpu.from },
+  };
+}
+
+// --- the split inside the pod ------------------------------------------------------------------
+
+/**
+ * One Sandbox pod's resources, as the CR carries them (ADR-0060). Every request equals its limit —
+ * the pod is Guaranteed, and the CPU limit is enforced — except the Harness container's
+ * `ephemeral-storage`, a request with no limit, and the preflight's small fixed step.
+ */
+export type SizeSplit = {
+  /** The whole Size, pod-level (KEP-2837): the ceiling the pod never passes. */
+  pod: ResourceRequirements;
+  /** The rest of the Size, after the Custodian and the User Container. */
+  harness: ResourceRequirements;
+  custodian: ResourceRequirements;
+  /** Only when the Machine states a split for the User Container; otherwise it has no limit of its
+   * own and shares the pod's budget. */
+  user?: ResourceRequirements;
+  preflight: ResourceRequirements;
+  /** The `/dev/shm` emptyDir's `sizeLimit`: 25% of the Harness share, minimum 64Mi. */
+  shmSizeLimit: string;
+  /** The Harness container's memory limit in bytes — what layer 4 of ADR-0061 guards. */
+  harnessMemoryBytes: number;
+};
+
+const SOURCE_TEXT: Record<SizeSource, (field: "memory" | "cpu") => string> = {
+  machine: (f) => `the Machine's Size — its workspace()'s resources.limits.${f}, or a customize() of it`,
+  instance: (f) =>
+    `sandbox.resources.limits.${f} in jr2.config.ts, the Instance's default for a Workspace that states none`,
+  kit: () => "the kit default — the Workspace states no Size",
+};
+
+/**
+ * Split a resolved Size inside the pod (ADR-0060), or refuse. `user` is the User Container's split
+ * as the Machine stated it. The refusal names each number and where it came from, because the fix
+ * is one of them: a larger Size, or a smaller split.
+ */
+export function splitSize(size: ResolvedSize, user?: Size): SizeSplit {
+  const custodian = { memory: memoryBytes(CUSTODIAN_SLICE.memory), cpu: cpuMillis(CUSTODIAN_SLICE.cpu) };
+  const userMemory = user?.limits.memory !== undefined ? memoryBytes(user.limits.memory) : undefined;
+  const userCpu = user?.limits.cpu !== undefined ? cpuMillis(user.limits.cpu) : undefined;
+  const harnessMemory = size.memory.bytes - custodian.memory - (userMemory ?? 0);
+  const harnessCpu = size.cpu.millis - custodian.cpu - (userCpu ?? 0);
+
+  const floor = memoryBytes(HARNESS_FLOOR);
+  if (harnessMemory < floor) {
+    throw new Error(
+      `the Size leaves the Harness container ${harnessMemory > 0 ? formatMemory(harnessMemory) : "no"} memory, below its ` +
+        `${HARNESS_FLOOR} floor: memory ${formatMemory(size.memory.bytes)} (${SOURCE_TEXT[size.memory.from]("memory")}) ` +
+        `− the Custodian's ${CUSTODIAN_SLICE.memory}` +
+        (userMemory !== undefined
+          ? ` − the User Container's ${formatMemory(userMemory)} (its user.resources.limits.memory)`
+          : "") +
+        " — raise the Size's limits.memory, or shrink the User Container's split (ADR-0060)",
+    );
+  }
+  if (harnessCpu <= 0) {
+    throw new Error(
+      `the Size leaves the Harness container no cpu: cpu ${formatCpu(size.cpu.millis)} ` +
+        `(${SOURCE_TEXT[size.cpu.from]("cpu")}) − the Custodian's ${CUSTODIAN_SLICE.cpu}` +
+        (userCpu !== undefined ? ` − the User Container's ${formatCpu(userCpu)} (its user.resources.limits.cpu)` : "") +
+        " — raise the Size's limits.cpu, or shrink the User Container's split (ADR-0060)",
+    );
+  }
+
+  const exact = (memory: number, cpu: number): ResourceRequirements => {
+    const limits = { cpu: formatCpu(cpu), memory: formatMemory(memory) };
+    return { requests: { ...limits }, limits };
+  };
+  const harness = exact(harnessMemory, harnessCpu);
+  harness.requests!["ephemeral-storage"] = WORK_DISK_REQUEST;
+
+  let userResources: ResourceRequirements | undefined;
+  if (userMemory !== undefined || userCpu !== undefined) {
+    const limits: Record<string, string> = {
+      ...(userCpu !== undefined ? { cpu: formatCpu(userCpu) } : {}),
+      ...(userMemory !== undefined ? { memory: formatMemory(userMemory) } : {}),
+    };
+    userResources = { requests: { ...limits }, limits };
+  }
+
+  return {
+    pod: exact(size.memory.bytes, size.cpu.millis),
+    harness,
+    custodian: exact(custodian.memory, custodian.cpu),
+    ...(userResources ? { user: userResources } : {}),
+    preflight: {
+      requests: { ...PREFLIGHT_RESOURCES.requests },
+      limits: { ...PREFLIGHT_RESOURCES.limits },
+    },
+    shmSizeLimit: formatMemory(Math.max(memoryBytes(SHM_FLOOR), Math.floor(harnessMemory / 4))),
+    harnessMemoryBytes: harnessMemory,
+  };
+}
+
+// --- the User Container seat -------------------------------------------------------------------
+
+// The User Container as a Machine states it (ADR-0005, ADR-0060): an image in ADR-0037's two
+// shapes, and optionally its split of the Size. The object form exists for the split alone.
+/**
+ * The User Container in its object form: its image, and its part of the Workspace's Size. The User
+ * Container is an optional container of your own in the Sandbox pod. It runs its own entrypoint
+ * with `/work` mounted, and jr2 never builds into it, probes it or sends it commands. Use it for a
+ * service that must run without an Agent, such as an sshd.
+ *
+ * Write it in the `user` option of `workspace()` or of `customize()`. That option also takes the
+ * image as a string; use this object form only to give the User Container a part of the Size. A
+ * `customize()` that names `user` replaces the whole User Container, its part of the Size
+ * included.
+ *
+ * It refuses, where you write it. `<where>` is `workspace()` or `customize()`:
+ *
+ * - an image that is not a non-empty string. The message starts with
+ *   `` <where>: `user` must be a non-empty string (an image), or { image, resources? } ``.
+ * - `<where>: user.<key> is not accepted — the User Container is { image, resources? } (ADR-0005,
+ *   ADR-0060)`
+ *
+ * `resources` is checked as a {@link Size}, with the same refusals.
+ *
+ * `jr2 up` refuses a part that leaves the Harness container less than its floor. The message starts
+ * with `the Size leaves the Harness container <memory> memory, below its 256Mi floor:` or with
+ * `the Size leaves the Harness container no cpu:`. The text that follows names each number and
+ * where it came from.
+ *
+ * @example
+ * ```ts
+ * import { customize, type UserContainer } from "@jr2/orchestrator";
+ * import { task } from "@jr2/machines";
+ *
+ * const sshd: UserContainer = {
+ *   image: "registry.example.com/sshd:1.0",
+ *   resources: { limits: { memory: "256Mi", cpu: "250m" } },
+ * };
+ *
+ * export const machine = customize(task, {
+ *   repos: { app: { url: "https://github.com/you/app.git" } },
+ *   agents: { coder: { model: "anthropic/claude-haiku-5-5" } },
+ *   user: sshd,
+ * });
+ * ```
+ *
+ * @see docs/concepts/workspaces.md
+ * @see docs/guides/size-a-workspace.md
+ * @see docs/reference/decisions/0005-sandbox-pod-composition.md
+ * @see docs/reference/decisions/0060-a-workspace-states-its-size-and-jr2-reserves-all-of-it.md
+ */
+export type UserContainer = {
+  /** The image: a `file:` URL to a docker context that your module ships
+   * (`import.meta.resolve("./sshd")`), which `jr2 up` builds, or a registry ref. It must not be
+   * empty. */
+  image: string;
+  /** This container's part of the Workspace's Size, as a Size. jr2 takes it from the Harness
+   * container's part. A field you leave out gives this container no limit of its own for it: it
+   * shares the pod's budget. Default: no part. */
+  resources?: Size;
+};
+
+/** The `user` option in either shape, read as its two halves. */
+export function userSeatOf(user: string | UserContainer | undefined): { image?: string; resources?: Size } {
+  if (user === undefined) return {};
+  if (typeof user === "string") return { image: user };
+  return { image: user.image, ...(user.resources !== undefined ? { resources: user.resources } : {}) };
+}
+
+/**
+ * Refuse a malformed `user` where it is written — a non-empty image string, or `{ image,
+ * resources? }` whose split is a Size. `where` names the caller (`workspace()`, `customize()`).
+ */
+export function assertUserSeat(where: string, value: unknown): asserts value is string | UserContainer {
+  const image = typeof value === "object" && value !== null ? (value as { image?: unknown }).image : value;
+  if (typeof image !== "string" || !image) {
+    throw new Error(
+      `${where}: \`user\` must be a non-empty string (an image), or { image, resources? } (got ${JSON.stringify(value)}) — ` +
+        'either a `file:` URL to a docker context this module ships (`import.meta.resolve("./image")`) or a ' +
+        "registry ref (ADR-0037, ADR-0060).",
+    );
+  }
+  if (typeof value !== "object" || value === null) return;
+  for (const key of Object.keys(value)) {
+    if (key !== "image" && key !== "resources") {
+      throw new Error(
+        `${where}: user.${key} is not accepted — the User Container is { image, resources? } (ADR-0005, ADR-0060)`,
+      );
+    }
+  }
+  const resources = (value as { resources?: unknown }).resources;
+  if (resources !== undefined) assertSize(`${where} user.resources`, resources);
+}

package/src/snapshot-store.ts

@@ -1,7 +1,11 @@
-// Persistence for durable Machine snapshots (ADR-0007). A `SnapshotStore` is keyed by `runId`;
-// `SqliteSnapshotStore` is the default backing for a deployed Orchestrator, with `:memory:` for
-// tests. `markLost` records a run whose live state could not be re-hydrated (e.g. its flue
-// handle is gone) without deleting its history.
+// Persistence for durable Machine snapshots (ADR-0007). A `SnapshotStore` is keyed by `runId`, one
+// blob per run: the snapshot is a tree xstate restores from its root, and one row keeps it atomic
+// (ADR-0065). `SqliteSnapshotStore` on the Instance's volume is the one implementation — no other
+// store is owed — with `:memory:` for tests. A persist survives a process crash, not a power loss
+// (WAL, `synchronous=NORMAL`), and the write stays synchronous so the row is on disk before the
+// Machine's next side effect runs. `markLost` records a run whose live state could not be
+// re-hydrated (e.g. its flue handle is gone), keeping any snapshot the row had; a non-live row
+// lives a week, then `sweep` deletes it.
 
 import { DatabaseSync } from "node:sqlite";
 import type { StoredSnapshot } from "./durability.ts";
@@ -9,8 +13,24 @@ import type { StoredSnapshot } from "./durability.ts";
 export interface SnapshotStore {
   init(): Promise<void>;
   load(runId: string): Promise<StoredSnapshot | null>;
-  /** Every persisted run, in insertion order. The host filters by `status` on restore. */
-  list(): Promise<StoredSnapshot[]>;
+  /** The `live` runs, in insertion order — what `restore()` reads (ADR-0065). Terminal rows are
+   * filtered by the store, not the host, so a boot never parses a week of finished runs. */
+  live(): Promise<StoredSnapshot[]>;
+  /**
+   * The `live` runs by the Assembly hash their blob names (ADR-0069), each list in insertion order:
+   * what holds an Assembly. A hold is the ROW, not the process: a run this process failed to
+   * restore is still live here, so it still holds its Assembly. Read without parsing a snapshot,
+   * because `GET /assembly` and every collection ask. A live row that names no Assembly is not
+   * listed.
+   */
+  liveAssemblies(): Promise<Map<string, string[]>>;
+  /**
+   * Delete every non-live row last written before `olderThan`, and answer how many (ADR-0065). A
+   * `live` row is never swept, whatever its age: a run parked on a Gate for a month is restored.
+   * A swept id is gone as if it never ran — `read()` answers undefined, and its abbreviations are
+   * free again (ADR-0009).
+   */
+  sweep(olderThan: Date): Promise<number>;
   /**
    * Persisted run ids sharing a prefix, sorted, capped at `limit` — the store half of abbreviated
    * run ids (ADR-0009). Includes `lost` rows: the ambiguity set is the ids that EXIST, not the ones
@@ -18,16 +38,31 @@ export interface SnapshotStore {
    * silently to the live one, which makes resolution unsound.
    */
   findIdsByPrefix(prefix: string, limit: number): Promise<string[]>;
+  /**
+   * The run whose blob carries the idempotency key `effect` (`dispatchedBy.effect`, ADR-0071), in
+   * any status, or undefined: what answers a re-sent dispatch with the run it already started. A
+   * row `markLost` wrote with no save before it has no snapshot, and so no key; a swept row is gone
+   * with its key.
+   */
+  findByDispatch(effect: string): Promise<string | undefined>;
   save(runId: string, snapshot: unknown, status?: string): Promise<void>;
+  /**
+   * Record a run whose live state could not be re-hydrated. An upsert: a row that was saved KEEPS
+   * its snapshot and its idempotency key, because a lost dispatched run must keep answering its
+   * key — a re-send from its caller would otherwise start a second run (ADR-0071). Only a run with
+   * no row before gets a row with no snapshot.
+   */
   markLost(runId: string, reason: string): Promise<void>;
   /**
-   * Record a run whose Machine changed shape underneath it (ADR-0030) — refused, not resumed.
+   * Record a run this Orchestrator can no longer read (ADR-0030, ADR-0069) — refused, not resumed:
+   * the Assembly its snapshot names is not on the volume, was published for another Kit version or
+   * does not load, or its Machine's shape does not match the snapshot's stamp.
    *
-   * Unlike {@link markLost} this KEEPS the snapshot. A lost run has nothing left to look at, so
-   * nulling it costs nothing; a drifted run is intact and merely unreadable by the Machine now
-   * loaded, and it is the one a human most needs to inspect. Nulling it would also hide the run
-   * entirely: `read()` returns undefined for a null blob, so `jr2 status <id>` would answer
-   * `no run` — a refusal indistinguishable from a run that never existed.
+   * An UPDATE of a row that is there, and it KEEPS the snapshot: a drifted run is intact and only
+   * unreadable by this Orchestrator, and it is the one a human most needs to inspect. Nulling it
+   * would also hide the run entirely: `read()` returns undefined for a null blob, so
+   * `jr2 status <id>` would answer `no run` — a refusal indistinguishable from a run that never
+   * existed. It keeps the row's idempotency key too (ADR-0071).
    */
   markDrifted(runId: string, reason: string): Promise<void>;
   close(): Promise<void>;
@@ -41,6 +76,20 @@ type Row = {
   updated_at: string;
 };
 
+/** The lookup `findByDispatch` runs, exported so a test can ask the planner about THIS statement,
+ * not a copy of it: the column is indexed, so the plan is a seek. */
+export const FIND_BY_DISPATCH_SQL = `SELECT run_id FROM machine_snapshots
+  WHERE dispatch_effect = ? ORDER BY rowid LIMIT 1`;
+
+/** The idempotency key a blob carries at `dispatchedBy.effect` (ADR-0071), or null. */
+function dispatchEffectOf(snapshot: unknown): string | null {
+  if (typeof snapshot !== "object" || snapshot === null) return null;
+  const by = (snapshot as { dispatchedBy?: unknown }).dispatchedBy;
+  if (typeof by !== "object" || by === null) return null;
+  const effect = (by as { effect?: unknown }).effect;
+  return typeof effect === "string" ? effect : null;
+}
+
 export class SqliteSnapshotStore implements SnapshotStore {
   private readonly db: DatabaseSync;
 
@@ -49,6 +98,10 @@ export class SqliteSnapshotStore implements SnapshotStore {
   }
 
   async init(): Promise<void> {
+    // A persist survives a process crash, not a power loss (ADR-0065): a write is durable once the
+    // OS holds it, and sqlite fsyncs only at a checkpoint. FULL cost 4.5 ms a persist, all fsync.
+    this.db.exec(`PRAGMA journal_mode = WAL`);
+    this.db.exec(`PRAGMA synchronous = NORMAL`);
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS machine_snapshots (
         run_id TEXT PRIMARY KEY,
@@ -58,6 +111,16 @@ export class SqliteSnapshotStore implements SnapshotStore {
         updated_at TEXT
       )
     `);
+    // The idempotency key of a dispatched run (ADR-0071) is copied out of its blob into a column of
+    // its own when `save` writes the row, and the column is indexed. Not an expression index over
+    // `json_extract(snapshot, …)`: that runs on every row and THROWS on a blob SQLite's JSON parser
+    // refuses (nested about a thousand deep), so one such row would fail its own save and every
+    // later `init()` over the volume. A column also costs no JSON parse a save.
+    const columns = this.db.prepare(`PRAGMA table_info(machine_snapshots)`).all() as Array<{ name: string }>;
+    if (!columns.some((c) => c.name === "dispatch_effect")) {
+      this.db.exec(`ALTER TABLE machine_snapshots ADD COLUMN dispatch_effect TEXT`);
+    }
+    this.db.exec(`CREATE INDEX IF NOT EXISTS machine_snapshots_dispatch_effect ON machine_snapshots(dispatch_effect)`);
   }
 
   async load(runId: string): Promise<StoredSnapshot | null> {
@@ -75,9 +138,12 @@ export class SqliteSnapshotStore implements SnapshotStore {
     return stored;
   }
 
-  async list(): Promise<StoredSnapshot[]> {
+  async live(): Promise<StoredSnapshot[]> {
     const rows = this.db
-      .prepare(`SELECT run_id, status, snapshot, reason, updated_at FROM machine_snapshots ORDER BY rowid`)
+      .prepare(
+        `SELECT run_id, status, snapshot, reason, updated_at FROM machine_snapshots
+         WHERE status = 'live' ORDER BY rowid`,
+      )
       .all() as Row[];
     return rows.map((row) => {
       const stored: StoredSnapshot = {
@@ -90,13 +156,27 @@ export class SqliteSnapshotStore implements SnapshotStore {
     });
   }
 
+  async liveAssemblies(): Promise<Map<string, string[]>> {
+    const rows = this.db
+      .prepare(
+        `SELECT run_id, json_extract(snapshot, '$.assembly') AS assembly FROM machine_snapshots
+         WHERE status = 'live' AND json_valid(snapshot) ORDER BY rowid`,
+      )
+      .all() as Array<{ run_id: string; assembly: unknown }>;
+    const held = new Map<string, string[]>();
+    for (const { run_id, assembly } of rows) {
+      if (typeof assembly !== "string") continue;
+      held.set(assembly, [...(held.get(assembly) ?? []), run_id]);
+    }
+    return held;
+  }
+
   /**
    * A range scan rather than `LIKE`/`GLOB`. `run_id` is the BINARY-collated PRIMARY KEY, but
    * SQLite's `LIKE` is ASCII-case-insensitive by default, so the planner declines the index range
    * and falls back to a full scan. `GLOB` would seek correctly but obliges every caller to escape
    * `*?[]` first — a store shouldn't have to trust that. The `CHAR(0x10FFFF)` sentinel as the upper
-   * bound also avoids incrementing the prefix's last character, which has edge cases. `>=`/`<`
-   * carries to Postgres unchanged, where `LIKE 'x%'` would need `text_pattern_ops` to use an index.
+   * bound also avoids incrementing the prefix's last character, which has edge cases.
    */
   async findIdsByPrefix(prefix: string, limit: number): Promise<string[]> {
     const rows = this.db
@@ -109,18 +189,24 @@ export class SqliteSnapshotStore implements SnapshotStore {
     return rows.map((row) => row.run_id);
   }
 
+  async findByDispatch(effect: string): Promise<string | undefined> {
+    const row = this.db.prepare(FIND_BY_DISPATCH_SQL).get(effect) as { run_id: string } | undefined;
+    return row?.run_id;
+  }
+
   async save(runId: string, snapshot: unknown, status = "live"): Promise<void> {
     this.db
       .prepare(
-        `INSERT INTO machine_snapshots (run_id, status, snapshot, reason, updated_at)
-         VALUES (?, ?, ?, NULL, ?)
+        `INSERT INTO machine_snapshots (run_id, status, snapshot, reason, updated_at, dispatch_effect)
+         VALUES (?, ?, ?, NULL, ?, ?)
          ON CONFLICT(run_id) DO UPDATE SET
            status = excluded.status,
            snapshot = excluded.snapshot,
            reason = NULL,
-           updated_at = excluded.updated_at`,
+           updated_at = excluded.updated_at,
+           dispatch_effect = excluded.dispatch_effect`,
       )
-      .run(runId, status, JSON.stringify(snapshot), new Date().toISOString());
+      .run(runId, status, JSON.stringify(snapshot), new Date().toISOString(), dispatchEffectOf(snapshot));
   }
 
   async markLost(runId: string, reason: string): Promise<void> {
@@ -144,6 +230,15 @@ export class SqliteSnapshotStore implements SnapshotStore {
       .run(reason, new Date().toISOString(), runId);
   }
 
+  /** `updated_at` is an ISO-8601 UTC string, so a string comparison is a time comparison. A
+   * terminal row's `updated_at` is its terminal write — the age the week counts from. */
+  async sweep(olderThan: Date): Promise<number> {
+    const result = this.db
+      .prepare(`DELETE FROM machine_snapshots WHERE status != 'live' AND updated_at < ?`)
+      .run(olderThan.toISOString());
+    return Number(result.changes);
+  }
+
   async close(): Promise<void> {
     this.db.close();
   }