@runuai/host 0.9.149 → 0.9.151

package/lib/project-env-task-replica.ts

@@ -0,0 +1,1752 @@
+/**
+ * ADR-154 task-keyed project-environment replica.
+ *
+ * Unlike the legacy sidecar replica, this authority is deliberately
+ * in-memory. Each task owns its reader, staging view, live values, delivery
+ * cursor and gate. Nothing in this module imports the host database or writes
+ * a secret to disk.
+ */
+
+import {
+  MAX_PROJECT_ENV_ACK_PAGE_BYTES,
+  isHostOpId,
+  isProjectEnvPolicyEpochs,
+  isProjectEnvReplicaV1ApplyFrame,
+  isProjectEnvReplicaV1BeginFrame,
+  isProjectEnvReplicaV1CommitFrame,
+  isProjectEnvReplicaV1PageFrame,
+  sameProjectEnvHead,
+  type ProjectEnvDeliveryCursor,
+  type ProjectEnvReplicaBindingState,
+  type ProjectEnvReplicaMaterialization,
+  type ProjectEnvReplicaResult,
+  type ProjectEnvReplicaV1AckFrame,
+  type ProjectEnvReplicaV1ApplyFrame,
+  type ProjectEnvReplicaV1BeginFrame,
+  type ProjectEnvReplicaV1CommitFrame,
+  type ProjectEnvReplicaV1PageFrame,
+} from "../src/protocol";
+import { PROJECT_ENV_POLICY_EPOCHS } from "../src/project-env-policy";
+import { RUNNER_BOOT_ID } from "./runner-boot";
+
+export type ProjectEnvTaskReplicaFrame =
+  | ProjectEnvReplicaV1BeginFrame
+  | ProjectEnvReplicaV1PageFrame
+  | ProjectEnvReplicaV1ApplyFrame
+  | ProjectEnvReplicaV1CommitFrame;
+
+/**
+ * The lease registry supplies one of these per task grant. Implementations
+ * must read through that task's project session and recheck their own lease
+ * fence after I/O. This replica independently rechecks its fence too.
+ */
+export interface ProjectEnvTaskValueReader {
+  /** Exact raw v2 parameter envelope, or null when the named value is absent. */
+  readStaged(
+    projectId: string,
+    key: string,
+    stagingId: string,
+  ): Promise<string | null>;
+  /** Exact raw v1 envelope and observed SSM version, or null when absent. */
+  readLegacy(
+    projectId: string,
+    key: string,
+    version: number,
+  ): Promise<{ value: string; version: number } | null>;
+}
+
+export interface ProjectEnvTaskAssignment {
+  taskId: string;
+  runEpoch: number;
+  orgId: string;
+  reader: ProjectEnvTaskValueReader;
+}
+
+export interface ProjectEnvTaskReplicaStatus {
+  taskId: string;
+  runEpoch: number;
+  runnerBootId: string;
+  gateOpen: boolean;
+  /** This run stays fail-closed until a new run or runner process starts. */
+  restartRequired: boolean;
+  streamId: string | null;
+  attemptId: string | null;
+  appliedThrough: ProjectEnvDeliveryCursor | null;
+  projectIds: string[];
+}
+
+export interface ProjectEnvTaskReplicaSnapshotEntry {
+  state: ProjectEnvReplicaBindingState;
+  materialization: ProjectEnvReplicaMaterialization;
+  value: string | null;
+}
+
+export interface ProjectEnvTaskReplicaSnapshotProject {
+  projectId: string;
+  recordVersion: number;
+  entries: ProjectEnvTaskReplicaSnapshotEntry[];
+}
+
+export interface ProjectEnvTaskReplicaSnapshot {
+  taskId: string;
+  runEpoch: number;
+  runnerBootId: string;
+  appliedThrough: ProjectEnvDeliveryCursor;
+  projects: ProjectEnvTaskReplicaSnapshotProject[];
+}
+
+export interface ProjectEnvTaskReplicaManagerOptions {
+  runnerBootId?: string;
+  policyEpochs?: readonly number[];
+  /** Optional lower limits for tests or unusually constrained runners. */
+  replayLedgerLimits?: {
+    maxTaskEntries?: number;
+    maxTaskBytes?: number;
+    maxTotalEntries?: number;
+    maxTotalBytes?: number;
+  };
+  onChange?: (taskId: string) => void;
+}
+
+interface ReplicaEntry {
+  state: ProjectEnvReplicaBindingState;
+  materialization: ProjectEnvReplicaMaterialization;
+  value: string | null;
+}
+
+interface ReplicaProject {
+  recordVersion: number;
+  records: Map<string, ReplicaEntry>;
+}
+
+interface PageReceipt {
+  fingerprint: string;
+  after: string | null;
+  recordVersion: number;
+  results: ProjectEnvReplicaResult[];
+  complete: boolean;
+  appliedThrough: ProjectEnvDeliveryCursor;
+}
+
+interface StagedProject extends ReplicaProject {
+  started: boolean;
+  complete: boolean;
+  lastKey: string | null;
+  pages: Map<string, PageReceipt>;
+}
+
+interface SnapshotAttempt {
+  attemptId: string;
+  beginFingerprint: string;
+  projectIds: string[];
+  projects: Map<string, StagedProject>;
+  /** The grant reader it began under (its `readerGeneration`): every value
+   *  it holds was read through it, or it is refused. */
+  readerGeneration: number;
+}
+
+type ApplyReceipt =
+  | {
+      fingerprint: string;
+      ok: true;
+      attemptId: string | null;
+      projectId: string;
+      recordVersion: number;
+      result: ProjectEnvReplicaResult;
+      appliedThrough: ProjectEnvDeliveryCursor;
+    }
+  | {
+      fingerprint: string;
+      ok: false;
+      attemptId: string | null;
+      code: string;
+      appliedThrough: ProjectEnvDeliveryCursor;
+    };
+
+interface CompletedAttempt {
+  fingerprint: string;
+  highWater: ProjectEnvDeliveryCursor;
+}
+
+interface ReadyWaiter {
+  resolve: () => void;
+  reject: (error: Error) => void;
+  timer: NodeJS.Timeout;
+}
+
+interface TaskReplica {
+  taskId: string;
+  runEpoch: number;
+  orgId: string;
+  reader: ProjectEnvTaskValueReader;
+  /** Advanced each time a grant's reader replaces the reader: a number, so
+   *  no earlier grant's reader (and its sessions) stays reachable here. */
+  readerGeneration: number;
+  /** The reader generation whose attempt produced the committed snapshot. */
+  snapshotReaderGeneration: number | null;
+  /** Advanced by synchronous lifecycle changes, fencing reads in flight. */
+  generation: number;
+  gateOpen: boolean;
+  streamId: string | null;
+  appliedSequence: number;
+  attempt: SnapshotAttempt | null;
+  projects: Map<string, ReplicaProject>;
+  retiredStreams: Set<string>;
+  retiredAttempts: Set<string>;
+  applyReceipts: Map<number, ApplyReceipt>;
+  applyReceiptBytes: Map<number, number>;
+  completedAttempts: Map<string, CompletedAttempt>;
+  replayLedgerEntries: number;
+  replayLedgerBytes: number;
+  restartRequired: boolean;
+  /** Materialized page/apply entries not yet installed into a task view. */
+  inFlightEntries: Set<ReplicaEntry>;
+  waiters: Set<ReadyWaiter>;
+}
+
+type FramePhase = ProjectEnvReplicaV1AckFrame["phase"];
+
+const NULL_PAGE_CURSOR = "\0";
+/**
+ * Replay receipts cannot be aged out while an assignment keeps serving. The
+ * wire contract has opaque stream/attempt ids and no acknowledgement watermark
+ * from the cloud, so an arbitrarily late retry can still be legitimate. These
+ * limits therefore fail the replica closed instead of evicting evidence. They
+ * count only the four cumulative replay ledgers below; live values and the one
+ * active snapshot are replica state, not replay metadata, and follow their own
+ * task/attempt lifetime.
+ */
+/** Maximum replay-evidence records retained by one live task assignment. */
+export const PROJECT_ENV_REPLAY_LEDGER_MAX_TASK_ENTRIES = 16_384;
+/** Maximum encoded replay-evidence bytes retained by one live assignment. */
+export const PROJECT_ENV_REPLAY_LEDGER_MAX_TASK_BYTES = 8 * 1_024 * 1_024;
+/** Aggregate replay-evidence record bound for this runner process. */
+export const PROJECT_ENV_REPLAY_LEDGER_MAX_TOTAL_ENTRIES = 65_536;
+/** Aggregate encoded replay-evidence byte bound for this runner process. */
+export const PROJECT_ENV_REPLAY_LEDGER_MAX_TOTAL_BYTES = 64 * 1_024 * 1_024;
+const REPLAY_LEDGER_ENTRY_OVERHEAD_BYTES = 64;
+const REPLAY_LEDGER_EXHAUSTED = "replica_restart_required";
+/** A grant for the run was accepted and its reader is not active yet: retry
+ *  the frame shortly. A snapshot read through the reader it replaces would
+ *  answer for nothing. */
+export const PROJECT_ENV_ACTIVATION_PENDING = "activation_pending";
+/** The run's grant reader changed under this attempt, this read or the live
+ *  snapshot this apply would change: begin a new attempt. */
+export const PROJECT_ENV_READER_CHANGED = "reader_changed";
+const CLOSED_READER: ProjectEnvTaskValueReader = {
+  readStaged: async () => {
+    throw new Error("task replica reader is closed");
+  },
+  readLegacy: async () => {
+    throw new Error("task replica reader is closed");
+  },
+};
+
+function canonicalJson(value: unknown): string {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+  const object = value as Record<string, unknown>;
+  return `{${Object.keys(object)
+    .sort()
+    .map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`)
+    .join(",")}}`;
+}
+
+function clone<T>(value: T): T {
+  return structuredClone(value);
+}
+
+function sameStrings(left: readonly string[], right: readonly string[]): boolean {
+  return (
+    left.length === right.length &&
+    left.every((value, index) => value === right[index])
+  );
+}
+
+function validRunEpoch(value: number): boolean {
+  return Number.isSafeInteger(value) && value >= 1;
+}
+
+function validCredentialId(value: string): boolean {
+  return (
+    value.length > 0 &&
+    value.length <= 128 &&
+    !/[\u0000-\u001f\u007f]/.test(value)
+  );
+}
+
+function validReplayLedgerLimit(value: number): boolean {
+  return Number.isSafeInteger(value) && value >= 1;
+}
+
+function replayLedgerEntryBytes(value: unknown): number {
+  return (
+    REPLAY_LEDGER_ENTRY_OVERHEAD_BYTES +
+    Buffer.byteLength(canonicalJson(value), "utf8")
+  );
+}
+
+function validAssignment(value: ProjectEnvTaskAssignment): boolean {
+  return (
+    isHostOpId(value.taskId) &&
+    validRunEpoch(value.runEpoch) &&
+    validCredentialId(value.orgId) &&
+    value.reader !== null &&
+    typeof value.reader === "object" &&
+    typeof value.reader.readStaged === "function" &&
+    typeof value.reader.readLegacy === "function"
+  );
+}
+
+function exactObject(
+  value: unknown,
+  keys: readonly string[],
+): value is Record<string, unknown> {
+  return (
+    value !== null &&
+    typeof value === "object" &&
+    !Array.isArray(value) &&
+    Object.keys(value).length === keys.length &&
+    keys.every((key) => Object.hasOwn(value, key))
+  );
+}
+
+function desiredMaterialization(
+  state: ProjectEnvReplicaBindingState,
+  policyEpochs: readonly number[],
+): ProjectEnvReplicaMaterialization {
+  if (state.head.tombstone) return { kind: "withheld", reason: "tombstone" };
+  if (state.policyEpoch === null || !policyEpochs.includes(state.policyEpoch)) {
+    return { kind: "withheld", reason: "policy_epoch_unknown" };
+  }
+  if (state.binding?.kind === "omit") {
+    return { kind: "withheld", reason: "binding_omits" };
+  }
+  if (state.binding?.kind === "withhold") {
+    return { kind: "withheld", reason: state.binding.reason };
+  }
+  return { kind: "unavailable", reason: "materialization_pending" };
+}
+
+function resultOf(entry: ReplicaEntry): ProjectEnvReplicaResult {
+  return {
+    state: clone(entry.state),
+    materialization: clone(entry.materialization),
+  } as ProjectEnvReplicaResult;
+}
+
+function scrubEntry(entry: ReplicaEntry): void {
+  entry.value = null;
+}
+
+function scrubProjects(projects: Map<string, ReplicaProject>): void {
+  for (const project of projects.values()) {
+    for (const entry of project.records.values()) scrubEntry(entry);
+    project.records.clear();
+  }
+  projects.clear();
+}
+
+function scrubAttempt(attempt: SnapshotAttempt | null): void {
+  if (attempt === null) return;
+  scrubProjects(attempt.projects);
+}
+
+function stateOrder(
+  incoming: ProjectEnvReplicaBindingState,
+  current: ProjectEnvReplicaBindingState,
+): "same" | "newer" | "older" | "conflict" {
+  const revisionDelta = incoming.head.revision - current.head.revision;
+  const generationDelta = incoming.head.generation - current.head.generation;
+  if (revisionDelta < 0 && generationDelta <= 0) return "older";
+  if (generationDelta < 0 && revisionDelta <= 0) return "older";
+  if (revisionDelta < 0 || generationDelta < 0) return "conflict";
+  if (revisionDelta === 0 && generationDelta === 0) {
+    if (!sameProjectEnvHead(incoming.head, current.head)) return "conflict";
+    if (
+      canonicalJson(incoming.binding) !== canonicalJson(current.binding) ||
+      incoming.policyEpoch !== current.policyEpoch
+    ) {
+      return "conflict";
+    }
+    if (incoming.listingGeneration < current.listingGeneration) return "older";
+    if (incoming.listingGeneration === current.listingGeneration) {
+      return canonicalJson(incoming) === canonicalJson(current)
+        ? "same"
+        : "conflict";
+    }
+    return "newer";
+  }
+  if (incoming.listingGeneration < current.listingGeneration) return "conflict";
+  if (
+    revisionDelta === 0 &&
+    (incoming.head.tombstone !== current.head.tombstone ||
+      canonicalJson(incoming.binding) !== canonicalJson(current.binding) ||
+      incoming.policyEpoch !== current.policyEpoch)
+  ) {
+    return "conflict";
+  }
+  if (
+    generationDelta === 0 &&
+    (incoming.head.tombstone !== current.head.tombstone ||
+      canonicalJson(incoming.head.valueRef) !==
+        canonicalJson(current.head.valueRef))
+  ) {
+    return "conflict";
+  }
+  return "newer";
+}
+
+function isReachableRecordAdvance(
+  incoming: ProjectEnvReplicaBindingState,
+  current: ProjectEnvReplicaBindingState | null,
+): boolean {
+  if (current === null) {
+    return (
+      (incoming.head.revision === 1 &&
+        incoming.head.generation === 1 &&
+        !incoming.head.tombstone) ||
+      (incoming.head.revision === 1 &&
+        incoming.head.generation === 0 &&
+        incoming.head.tombstone)
+    );
+  }
+  const revision = incoming.head.revision - current.head.revision;
+  const generation = incoming.head.generation - current.head.generation;
+  if (incoming.listingGeneration < current.listingGeneration) return false;
+  if (revision === 0 && generation === 0) {
+    return (
+      sameProjectEnvHead(incoming.head, current.head) &&
+      canonicalJson(incoming.binding) === canonicalJson(current.binding) &&
+      incoming.policyEpoch === current.policyEpoch &&
+      incoming.listingGeneration > current.listingGeneration
+    );
+  }
+  if (revision === 0 && generation === 1) {
+    return (
+      !current.head.tombstone &&
+      !incoming.head.tombstone &&
+      canonicalJson(incoming.binding) === canonicalJson(current.binding) &&
+      incoming.policyEpoch === current.policyEpoch
+    );
+  }
+  if (revision === 1 && generation === 0) {
+    return (
+      !current.head.tombstone &&
+      !incoming.head.tombstone &&
+      canonicalJson(incoming.head.valueRef) ===
+        canonicalJson(current.head.valueRef)
+    );
+  }
+  if (revision === 1 && generation === 1) {
+    if (current.head.generation === 0) {
+      return current.head.tombstone && !incoming.head.tombstone;
+    }
+    return (
+      current.head.tombstone !== incoming.head.tombstone &&
+      canonicalJson(incoming.binding) === canonicalJson(current.binding) &&
+      incoming.policyEpoch === current.policyEpoch
+    );
+  }
+  return false;
+}
+
+function parseStagedEnvelope(
+  raw: string,
+  orgId: string,
+  projectId: string,
+  key: string,
+  stagingId: string,
+): string | null {
+  const parsed: unknown = JSON.parse(raw);
+  if (
+    !exactObject(parsed, [
+      "v",
+      "kind",
+      "orgId",
+      "projectId",
+      "key",
+      "stagingId",
+      "value",
+    ]) ||
+    parsed.v !== 2 ||
+    parsed.kind !== "project-env" ||
+    parsed.orgId !== orgId ||
+    parsed.projectId !== projectId ||
+    parsed.key !== key ||
+    parsed.stagingId !== stagingId ||
+    typeof parsed.value !== "string"
+  ) {
+    return null;
+  }
+  return parsed.value;
+}
+
+function parseLegacyEnvelope(
+  raw: string,
+  orgId: string,
+  projectId: string,
+  key: string,
+): string | null {
+  const parsed: unknown = JSON.parse(raw);
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+    return null;
+  }
+  const object = parsed as Record<string, unknown>;
+  const keys = Object.keys(object);
+  const required = ["v", "kind", "projectId", "key", "value", "updatedAt"];
+  if (
+    required.some((field) => !Object.hasOwn(object, field)) ||
+    keys.some((field) => !required.includes(field) && field !== "orgId") ||
+    object.v !== 1 ||
+    object.kind !== "project-env" ||
+    object.projectId !== projectId ||
+    object.key !== key ||
+    typeof object.value !== "string" ||
+    typeof object.updatedAt !== "number" ||
+    !Number.isFinite(object.updatedAt) ||
+    !(object.orgId === undefined || object.orgId === orgId)
+  ) {
+    return null;
+  }
+  return object.value;
+}
+
+function framePhase(frame: ProjectEnvTaskReplicaFrame): FramePhase {
+  switch (frame.kind) {
+    case "env.replica.v1.begin":
+      return "begun";
+    case "env.replica.v1.page":
+      return "page";
+    case "env.replica.v1.apply":
+      return "applied";
+    case "env.replica.v1.commit":
+      return "complete";
+  }
+}
+
+/**
+ * One process-local registry. Constructing a replacement with a new boot id
+ * starts empty; a WebSocket reconnect keeps using the same instance.
+ */
+export class ProjectEnvTaskReplicaManager {
+  readonly runnerBootId: string;
+  readonly policyEpochs: readonly number[];
+
+  readonly #tasks = new Map<string, TaskReplica>();
+  /** Non-secret same-boot floor: cleanup cannot make an old grant current. */
+  readonly #runEpochFloors = new Map<string, number>();
+  /** A ledger-exhausted run cannot be recreated under the same boot fence. */
+  readonly #restartRequiredRuns = new Map<string, number>();
+  readonly #lanes = new Map<string, Promise<void>>();
+  /** Grants accepted and being installed whose reader is not active yet, by
+   *  task (each grant's token, and its run): the run's frames are refused,
+   *  retryably, while any is pending. */
+  readonly #pendingActivations = new Map<string, Map<number, number>>();
+  #activationSequence = 0;
+  readonly #onChange: (taskId: string) => void;
+  readonly #replayLedgerMaxTaskEntries: number;
+  readonly #replayLedgerMaxTaskBytes: number;
+  readonly #replayLedgerMaxTotalEntries: number;
+  readonly #replayLedgerMaxTotalBytes: number;
+  #replayLedgerEntries = 0;
+  #replayLedgerBytes = 0;
+
+  constructor(options: ProjectEnvTaskReplicaManagerOptions = {}) {
+    const runnerBootId = options.runnerBootId ?? RUNNER_BOOT_ID;
+    const policyEpochs = [...(options.policyEpochs ?? PROJECT_ENV_POLICY_EPOCHS)];
+    const replayLedgerMaxTaskEntries =
+      options.replayLedgerLimits?.maxTaskEntries ??
+      PROJECT_ENV_REPLAY_LEDGER_MAX_TASK_ENTRIES;
+    const replayLedgerMaxTaskBytes =
+      options.replayLedgerLimits?.maxTaskBytes ??
+      PROJECT_ENV_REPLAY_LEDGER_MAX_TASK_BYTES;
+    const replayLedgerMaxTotalEntries =
+      options.replayLedgerLimits?.maxTotalEntries ??
+      PROJECT_ENV_REPLAY_LEDGER_MAX_TOTAL_ENTRIES;
+    const replayLedgerMaxTotalBytes =
+      options.replayLedgerLimits?.maxTotalBytes ??
+      PROJECT_ENV_REPLAY_LEDGER_MAX_TOTAL_BYTES;
+    if (
+      !isHostOpId(runnerBootId) ||
+      !isProjectEnvPolicyEpochs(policyEpochs) ||
+      !validReplayLedgerLimit(replayLedgerMaxTaskEntries) ||
+      !validReplayLedgerLimit(replayLedgerMaxTaskBytes) ||
+      !validReplayLedgerLimit(replayLedgerMaxTotalEntries) ||
+      !validReplayLedgerLimit(replayLedgerMaxTotalBytes) ||
+      replayLedgerMaxTaskEntries > replayLedgerMaxTotalEntries ||
+      replayLedgerMaxTaskBytes > replayLedgerMaxTotalBytes
+    ) {
+      throw new Error("invalid task replica manager identity");
+    }
+    this.runnerBootId = runnerBootId;
+    this.policyEpochs = policyEpochs;
+    this.#replayLedgerMaxTaskEntries = replayLedgerMaxTaskEntries;
+    this.#replayLedgerMaxTaskBytes = replayLedgerMaxTaskBytes;
+    this.#replayLedgerMaxTotalEntries = replayLedgerMaxTotalEntries;
+    this.#replayLedgerMaxTotalBytes = replayLedgerMaxTotalBytes;
+    this.#onChange = options.onChange ?? (() => {});
+  }
+
+  /**
+   * Install or refresh the lease-owned reader for an assignment. Repeating
+   * the same assignment preserves its replica across a reconnect/renewal but
+   * fences any read still using the previous reader. A new run drops every
+   * plaintext reference before creating the closed replacement.
+   */
+  activateAssignment(
+    assignment: ProjectEnvTaskAssignment,
+  ): "created" | "retained" | "replaced" {
+    if (!validAssignment(assignment)) throw new Error("invalid task assignment");
+    const floor = this.#runEpochFloors.get(assignment.taskId) ?? 0;
+    if (assignment.runEpoch < floor) {
+      throw new Error("task assignment run epoch is stale");
+    }
+    const restartRequiredRun = this.#restartRequiredRuns.get(assignment.taskId);
+    if (restartRequiredRun === assignment.runEpoch) {
+      throw new Error("task replica requires a runner restart");
+    }
+    if (
+      restartRequiredRun !== undefined &&
+      assignment.runEpoch > restartRequiredRun
+    ) {
+      this.#restartRequiredRuns.delete(assignment.taskId);
+    }
+    const current = this.#tasks.get(assignment.taskId);
+    if (current?.runEpoch === assignment.runEpoch) {
+      if (current.orgId !== assignment.orgId) {
+        throw new Error("task assignment organization cannot change");
+      }
+      if (current.reader !== assignment.reader) {
+        current.reader = assignment.reader;
+        current.readerGeneration += 1;
+        current.generation += 1;
+      }
+      this.#activated(assignment);
+      return "retained";
+    }
+    if (current !== undefined) this.#destroyTask(current, "assignment replaced");
+    const task: TaskReplica = {
+      taskId: assignment.taskId,
+      runEpoch: assignment.runEpoch,
+      orgId: assignment.orgId,
+      reader: assignment.reader,
+      readerGeneration: 1,
+      snapshotReaderGeneration: null,
+      generation: 1,
+      gateOpen: false,
+      streamId: null,
+      appliedSequence: 0,
+      attempt: null,
+      projects: new Map(),
+      retiredStreams: new Set(),
+      retiredAttempts: new Set(),
+      applyReceipts: new Map(),
+      applyReceiptBytes: new Map(),
+      completedAttempts: new Map(),
+      replayLedgerEntries: 0,
+      replayLedgerBytes: 0,
+      restartRequired: false,
+      inFlightEntries: new Set(),
+      waiters: new Set(),
+    };
+    this.#runEpochFloors.set(task.taskId, task.runEpoch);
+    this.#tasks.set(task.taskId, task);
+    this.#activated(assignment);
+    this.#changed(task.taskId);
+    return current === undefined ? "created" : "replaced";
+  }
+
+  /** The run's grant reader is now active, and only now: its frames are
+   *  taken from here. An activation that throws leaves them refused. */
+  #activated(assignment: ProjectEnvTaskAssignment): void {
+    const pending = this.#pendingActivations.get(assignment.taskId);
+    if (!pending) return;
+    for (const [sequence, runEpoch] of pending) {
+      if (runEpoch === assignment.runEpoch) pending.delete(sequence);
+    }
+    if (pending.size === 0) this.#pendingActivations.delete(assignment.taskId);
+  }
+
+  /**
+   * A grant for the run was accepted and is being installed: until its
+   * reader activates (`activateAssignment` succeeding), the run's frames are
+   * refused with `activation_pending`, for the cloud to retry, so that no
+   * snapshot is read through the reader the grant replaces. Returns what
+   * `settleActivation` takes, once the grant's handling is over.
+   */
+  expectActivation(taskId: string, runEpoch: number): number {
+    const sequence = ++this.#activationSequence;
+    const pending = this.#pendingActivations.get(taskId) ?? new Map<number, number>();
+    pending.set(sequence, runEpoch);
+    this.#pendingActivations.set(taskId, pending);
+    return sequence;
+  }
+
+  /** That grant's handling is over, its reader active or not: its token
+   *  goes, and no other grant's. */
+  settleActivation(taskId: string, sequence: number): void {
+    const pending = this.#pendingActivations.get(taskId);
+    if (!pending) return;
+    pending.delete(sequence);
+    if (pending.size === 0) this.#pendingActivations.delete(taskId);
+  }
+
+  /** The run's grant reader generation now: what an install waits for a
+   *  snapshot of, and what serving is bound to. */
+  readerGeneration(taskId: string, runEpoch: number): number | null {
+    const task = this.#tasks.get(taskId);
+    return task !== undefined && task.runEpoch === runEpoch ? task.readerGeneration : null;
+  }
+
+  /** A stale Stop/deny for an older run may not erase the current run. */
+  dropAssignment(taskId: string, runEpoch: number): boolean {
+    const task = this.#tasks.get(taskId);
+    if (task === undefined || task.runEpoch !== runEpoch) return false;
+    this.#tasks.delete(taskId);
+    this.#destroyTask(task, "task credentials were dropped");
+    this.#changed(taskId);
+    return true;
+  }
+
+  /** Process shutdown defense in depth; a new process starts with a new id. */
+  dropAll(): void {
+    const tasks = [...this.#tasks.values()];
+    this.#tasks.clear();
+    this.#pendingActivations.clear();
+    for (const task of tasks) {
+      this.#destroyTask(task, "runner process ended");
+      this.#changed(task.taskId);
+    }
+  }
+
+  /**
+   * Serialized per task so page/apply order is deterministic even when a
+   * transport dispatcher calls this concurrently. Lifecycle drops remain
+   * synchronous and therefore can fence a read while it awaits AWS.
+   */
+  async handleFrame(value: unknown): Promise<ProjectEnvReplicaV1AckFrame> {
+    const frame = this.#parseFrame(value);
+    const previous = this.#lanes.get(frame.taskId) ?? Promise.resolve();
+    const operation = previous.catch(() => {}).then(() => this.#dispatch(frame));
+    const tail = operation.then(
+      () => {},
+      () => {},
+    );
+    this.#lanes.set(frame.taskId, tail);
+    void tail.finally(() => {
+      if (this.#lanes.get(frame.taskId) === tail) this.#lanes.delete(frame.taskId);
+    });
+    return operation;
+  }
+
+  readStatus(taskId: string): ProjectEnvTaskReplicaStatus | null {
+    const task = this.#tasks.get(taskId);
+    if (task === undefined) return null;
+    return {
+      taskId,
+      runEpoch: task.runEpoch,
+      runnerBootId: this.runnerBootId,
+      gateOpen: task.gateOpen,
+      restartRequired: task.restartRequired,
+      streamId: task.streamId,
+      attemptId: task.attempt?.attemptId ?? null,
+      appliedThrough:
+        task.streamId === null
+          ? null
+          : { streamId: task.streamId, sequence: task.appliedSequence },
+      projectIds: [...task.projects.keys()].sort(),
+    };
+  }
+
+  isReady(taskId: string, runEpoch: number): boolean {
+    const task = this.#tasks.get(taskId);
+    return Boolean(
+      task &&
+        task.runEpoch === runEpoch &&
+        task.gateOpen &&
+        task.attempt === null &&
+        task.streamId !== null,
+    );
+  }
+
+  awaitReady(taskId: string, runEpoch: number, timeoutMs: number): Promise<void> {
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) {
+      return Promise.reject(new Error("invalid task replica readiness timeout"));
+    }
+    const task = this.#tasks.get(taskId);
+    if (task === undefined || task.runEpoch !== runEpoch) {
+      return Promise.reject(new Error("task replica assignment is unavailable"));
+    }
+    if (task.restartRequired) {
+      return Promise.reject(new Error("task replica requires a runner restart"));
+    }
+    if (this.isReady(taskId, runEpoch)) return Promise.resolve();
+    return new Promise<void>((resolve, reject) => {
+      const waiter: ReadyWaiter = {
+        resolve: () => {
+          clearTimeout(waiter.timer);
+          task.waiters.delete(waiter);
+          resolve();
+        },
+        reject: (error) => {
+          clearTimeout(waiter.timer);
+          task.waiters.delete(waiter);
+          reject(error);
+        },
+        timer: setTimeout(() => {
+          waiter.reject(
+            new Error("task environment replica did not become ready in time"),
+          );
+        }, timeoutMs),
+      };
+      waiter.timer.unref?.();
+      task.waiters.add(waiter);
+      if (this.isReady(taskId, runEpoch)) waiter.resolve();
+    });
+  }
+
+  /**
+   * Fail closed unless the caller names the current assignment and exactly
+   * the committed project set. The caller's order is retained for env merge
+   * precedence, while set comparison remains canonical.
+   */
+  readView(
+    taskId: string,
+    runEpoch: number,
+    projectIds: readonly string[],
+  ): ProjectEnvTaskReplicaSnapshot | null {
+    const task = this.#tasks.get(taskId);
+    if (
+      task === undefined ||
+      task.runEpoch !== runEpoch ||
+      !task.gateOpen ||
+      task.attempt !== null ||
+      task.streamId === null
+    ) {
+      return null;
+    }
+    const requested = [...projectIds];
+    const canonical = [...new Set(requested)].sort();
+    const actual = [...task.projects.keys()].sort();
+    if (canonical.length !== requested.length || !sameStrings(canonical, actual)) {
+      return null;
+    }
+    return {
+      taskId,
+      runEpoch,
+      runnerBootId: this.runnerBootId,
+      appliedThrough: {
+        streamId: task.streamId,
+        sequence: task.appliedSequence,
+      },
+      projects: requested.map((projectId) => {
+        const project = task.projects.get(projectId)!;
+        return {
+          projectId,
+          recordVersion: project.recordVersion,
+          entries: [...project.records.values()]
+            .sort((left, right) => left.state.key.localeCompare(right.state.key))
+            .map((entry) => ({
+              state: clone(entry.state),
+              materialization: clone(entry.materialization),
+              value: entry.value,
+            })),
+        };
+      }),
+    };
+  }
+
+  /**
+   * `readView`, for a snapshot every value of which was read through the
+   * reader of `readerGeneration` (one grant's): a snapshot another grant's
+   * reader produced never answers for this grant, nor is served for it.
+   */
+  readViewAt(
+    taskId: string,
+    runEpoch: number,
+    projectIds: readonly string[],
+    readerGeneration: number,
+  ): ProjectEnvTaskReplicaSnapshot | null {
+    const task = this.#tasks.get(taskId);
+    if (task === undefined || task.snapshotReaderGeneration !== readerGeneration) return null;
+    return this.readView(taskId, runEpoch, projectIds);
+  }
+
+  #parseFrame(value: unknown): ProjectEnvTaskReplicaFrame {
+    if (isProjectEnvReplicaV1BeginFrame(value)) return clone(value);
+    if (isProjectEnvReplicaV1PageFrame(value)) return clone(value);
+    if (isProjectEnvReplicaV1ApplyFrame(value)) return clone(value);
+    if (isProjectEnvReplicaV1CommitFrame(value)) return clone(value);
+    throw new Error("invalid task project environment replica frame");
+  }
+
+  async #dispatch(
+    frame: ProjectEnvTaskReplicaFrame,
+  ): Promise<ProjectEnvReplicaV1AckFrame> {
+    const task = this.#tasks.get(frame.taskId);
+    const identityError = this.#identityError(task, frame);
+    if (identityError !== null) return this.#errorAck(frame, task, identityError);
+    if (task!.restartRequired) {
+      return this.#errorAck(frame, task, REPLAY_LEDGER_EXHAUSTED);
+    }
+    switch (frame.kind) {
+      case "env.replica.v1.begin":
+        return this.#begin(task!, frame);
+      case "env.replica.v1.page":
+        return this.#page(task!, frame);
+      case "env.replica.v1.apply":
+        return this.#apply(task!, frame);
+      case "env.replica.v1.commit":
+        return this.#commit(task!, frame);
+    }
+  }
+
+  #identityError(
+    task: TaskReplica | undefined,
+    frame: ProjectEnvTaskReplicaFrame,
+  ): string | null {
+    if (frame.runnerBootId !== this.runnerBootId) return "runner_boot_mismatch";
+    for (const runEpoch of this.#pendingActivations.get(frame.taskId)?.values() ?? []) {
+      if (runEpoch === frame.runEpoch) return PROJECT_ENV_ACTIVATION_PENDING;
+    }
+    if (task === undefined || task.runEpoch !== frame.runEpoch) {
+      return "task_assignment_mismatch";
+    }
+    return null;
+  }
+
+  #begin(
+    task: TaskReplica,
+    frame: ProjectEnvReplicaV1BeginFrame,
+  ): ProjectEnvReplicaV1AckFrame {
+    const fingerprint = canonicalJson({
+      streamId: frame.streamId,
+      attemptId: frame.attemptId,
+      projectIds: frame.projectIds,
+    });
+    if (task.retiredStreams.has(frame.streamId)) {
+      return this.#errorAck(frame, task, "stream_retired");
+    }
+    if (task.retiredAttempts.has(frame.attemptId)) {
+      return this.#errorAck(frame, task, "reconciliation_attempt_retired");
+    }
+    if (task.streamId === frame.streamId && task.attempt !== null) {
+      if (
+        task.attempt.attemptId === frame.attemptId &&
+        task.attempt.beginFingerprint === fingerprint
+      ) {
+        return this.#successBegin(frame, task);
+      }
+      if (task.attempt.attemptId === frame.attemptId) {
+        return this.#errorAck(frame, task, "reconciliation_attempt_conflict");
+      }
+    }
+    if (task.streamId !== null && task.streamId !== frame.streamId) {
+      // Receipts belong to the old ordered stream. Once a different stream's
+      // begin arrives, the retired stream id is the evidence that prevents a
+      // delayed old begin from becoming current again.
+      this.#clearApplyReceipts(task);
+      if (!this.#rememberRetiredStream(task, task.streamId)) {
+        return this.#exhaustReplayLedger(task, frame);
+      }
+      task.appliedSequence = 0;
+    }
+    if (task.attempt !== null) {
+      if (!this.#rememberRetiredAttempt(task, task.attempt.attemptId)) {
+        return this.#exhaustReplayLedger(task, frame);
+      }
+    }
+    scrubAttempt(task.attempt);
+    task.streamId = frame.streamId;
+    task.gateOpen = false;
+    task.generation += 1;
+    task.attempt = {
+      attemptId: frame.attemptId,
+      beginFingerprint: fingerprint,
+      readerGeneration: task.readerGeneration,
+      projectIds: [...frame.projectIds],
+      projects: new Map(
+        frame.projectIds.map((projectId) => [
+          projectId,
+          {
+            recordVersion: 0,
+            records: new Map(),
+            started: false,
+            complete: false,
+            lastKey: null,
+            pages: new Map(),
+          },
+        ]),
+      ),
+    };
+    this.#changed(task.taskId);
+    return this.#successBegin(frame, task);
+  }
+
+  async #page(
+    task: TaskReplica,
+    frame: ProjectEnvReplicaV1PageFrame,
+  ): Promise<ProjectEnvReplicaV1AckFrame> {
+    const attempt = task.attempt;
+    if (
+      task.streamId !== frame.streamId ||
+      attempt === null ||
+      attempt.attemptId !== frame.attemptId ||
+      task.gateOpen
+    ) {
+      return this.#errorAck(frame, task, "reconciliation_attempt_mismatch");
+    }
+    if (attempt.readerGeneration !== task.readerGeneration) {
+      return this.#errorAck(frame, task, PROJECT_ENV_READER_CHANGED);
+    }
+    const project = attempt.projects.get(frame.projectId);
+    if (project === undefined) {
+      return this.#errorAck(frame, task, "project_not_in_attempt");
+    }
+    const pageKey = frame.after ?? NULL_PAGE_CURSOR;
+    const fingerprint = canonicalJson({
+      projectId: frame.projectId,
+      after: frame.after,
+      recordVersion: frame.recordVersion,
+      records: frame.records,
+      complete: frame.complete,
+    });
+    const receipt = project.pages.get(pageKey);
+    if (receipt !== undefined) {
+      if (receipt.fingerprint !== fingerprint) {
+        return this.#errorAck(frame, task, "snapshot_page_conflict");
+      }
+      return this.#successPage(frame, receipt, task);
+    }
+    if (
+      (frame.after === null && project.started) ||
+      (frame.after !== null &&
+        (!project.started || project.complete || project.lastKey !== frame.after))
+    ) {
+      return this.#errorAck(frame, task, "snapshot_page_mismatch");
+    }
+    if (project.started && project.recordVersion !== frame.recordVersion) {
+      return this.#errorAck(frame, task, "snapshot_version_conflict");
+    }
+    const liveProject = task.projects.get(frame.projectId);
+    if (
+      !project.started &&
+      liveProject !== undefined &&
+      frame.recordVersion < liveProject.recordVersion
+    ) {
+      return this.#errorAck(frame, task, "record_version_stale");
+    }
+    if (
+      liveProject !== undefined &&
+      frame.recordVersion === liveProject.recordVersion
+    ) {
+      const conflicts = frame.records.some((state) => {
+        const live = liveProject.records.get(state.key);
+        return live === undefined || canonicalJson(live.state) !== canonicalJson(state);
+      });
+      const prospectiveKeys = new Set([
+        ...project.records.keys(),
+        ...frame.records.map((state) => state.key),
+      ]);
+      if (
+        conflicts ||
+        (frame.complete &&
+          (prospectiveKeys.size !== liveProject.records.size ||
+            [...prospectiveKeys].some((key) => !liveProject.records.has(key))))
+      ) {
+        return this.#errorAck(frame, task, "snapshot_version_conflict");
+      }
+    }
+
+    const generation = task.generation;
+    const reader = task.reader;
+    const entries: ReplicaEntry[] = [];
+    for (const state of frame.records) {
+      const live = liveProject?.records.get(state.key);
+      // Plaintext the live snapshot holds is kept only when the same reader
+      // read it: a newer grant reads every value again, through itself.
+      const mayKeepInstalled =
+        live !== undefined &&
+        task.snapshotReaderGeneration === attempt.readerGeneration &&
+        live.materialization.kind === "installed" &&
+        canonicalJson(live.state) === canonicalJson(state) &&
+        desiredMaterialization(state, this.policyEpochs).kind !== "withheld";
+      const entry = mayKeepInstalled
+        ? {
+            state: clone(live.state),
+            materialization: clone(live.materialization),
+            value: live.value,
+          }
+        : await this.#materialize(task, frame.projectId, state, reader);
+      task.inFlightEntries.add(entry);
+      if (!this.#readFenceCurrent(task, frame, generation, reader, attempt)) {
+        for (const candidate of entries) {
+          scrubEntry(candidate);
+          task.inFlightEntries.delete(candidate);
+        }
+        scrubEntry(entry);
+        task.inFlightEntries.delete(entry);
+        return this.#errorAck(frame, this.#tasks.get(frame.taskId), this.#fenceCode(task, attempt.readerGeneration));
+      }
+      entries.push(entry);
+    }
+    const results = entries.map(resultOf);
+    const appliedThrough = this.#cursor(task, frame.streamId);
+    const prospective = this.#successPage(
+      frame,
+      {
+        fingerprint,
+        after: frame.after,
+        recordVersion: frame.recordVersion,
+        results,
+        complete: frame.complete,
+        appliedThrough,
+      },
+      task,
+    );
+    if (
+      Buffer.byteLength(JSON.stringify(prospective), "utf8") >
+      MAX_PROJECT_ENV_ACK_PAGE_BYTES
+    ) {
+      for (const entry of entries) {
+        scrubEntry(entry);
+        task.inFlightEntries.delete(entry);
+      }
+      return this.#errorAck(frame, task, "ack_page_too_large");
+    }
+
+    for (const entry of entries) {
+      task.inFlightEntries.delete(entry);
+      project.records.set(entry.state.key, entry);
+    }
+    project.started = true;
+    project.recordVersion = frame.recordVersion;
+    project.lastKey = frame.records.at(-1)?.key ?? null;
+    project.complete = frame.complete;
+    project.pages.set(pageKey, {
+      fingerprint,
+      after: frame.after,
+      recordVersion: frame.recordVersion,
+      results: clone(results),
+      complete: frame.complete,
+      appliedThrough,
+    });
+    return prospective;
+  }
+
+  async #apply(
+    task: TaskReplica,
+    frame: ProjectEnvReplicaV1ApplyFrame,
+  ): Promise<ProjectEnvReplicaV1AckFrame> {
+    if (task.streamId !== frame.streamId) {
+      return this.#errorAck(frame, task, "stream_mismatch");
+    }
+    const fingerprint = canonicalJson({
+      attemptId: frame.attemptId,
+      projectId: frame.projectId,
+      recordVersion: frame.recordVersion,
+      state: frame.state,
+      delivery: frame.delivery,
+    });
+    if (frame.delivery.sequence <= task.appliedSequence) {
+      const receipt = task.applyReceipts.get(frame.delivery.sequence);
+      if (receipt === undefined) {
+        return this.#errorAck(frame, task, "receipt_missing");
+      }
+      if (receipt.fingerprint !== fingerprint) {
+        return this.#errorAck(frame, task, "delivery_conflict");
+      }
+      return this.#ackFromApplyReceipt(frame, receipt, task);
+    }
+    if (
+      task.gateOpen
+        ? frame.attemptId !== null || task.attempt !== null
+        : task.attempt === null || frame.attemptId !== task.attempt.attemptId
+    ) {
+      return this.#errorAck(frame, task, "reconciliation_attempt_mismatch");
+    }
+    if (frame.delivery.sequence !== task.appliedSequence + 1) {
+      return this.#errorAck(frame, task, "delivery_gap");
+    }
+    // Read through the run's current reader, into what that same reader
+    // read: an attempt begun under it, or a live snapshot it produced.
+    if (
+      task.attempt === null
+        ? task.snapshotReaderGeneration !== task.readerGeneration
+        : task.attempt.readerGeneration !== task.readerGeneration
+    ) {
+      return this.#errorAck(frame, task, PROJECT_ENV_READER_CHANGED);
+    }
+
+    const project = task.attempt === null
+      ? task.projects.get(frame.projectId)
+      : task.attempt.projects.get(frame.projectId);
+    if (project === undefined) {
+      return this.#consumeApplyError(
+        task,
+        frame,
+        fingerprint,
+        "project_not_in_replica",
+      );
+    }
+    if ("complete" in project && !project.complete) {
+      return this.#errorAck(frame, task, "snapshot_incomplete");
+    }
+    if (frame.recordVersion > project.recordVersion + 1) {
+      return this.#errorAck(frame, task, "record_version_gap");
+    }
+    const current = project.records.get(frame.state.key) ?? null;
+    const advances =
+      frame.recordVersion === project.recordVersion + 1 &&
+      (current === null || stateOrder(frame.state, current.state) === "newer") &&
+      isReachableRecordAdvance(frame.state, current?.state ?? null);
+    if (!advances && current === null) {
+      return this.#consumeApplyError(
+        task,
+        frame,
+        fingerprint,
+        "record_version_conflict",
+      );
+    }
+
+    const desiredState = advances ? frame.state : current!.state;
+    const generation = task.generation;
+    const reader = task.reader;
+    const readerGeneration = task.readerGeneration;
+    const attempt = task.attempt;
+    let entry: ReplicaEntry;
+    if (
+      !advances &&
+      current!.materialization.kind === "installed" &&
+      desiredMaterialization(current!.state, this.policyEpochs).kind !== "withheld"
+    ) {
+      entry = {
+        state: clone(current!.state),
+        materialization: clone(current!.materialization),
+        value: current!.value,
+      };
+    } else {
+      entry = await this.#materialize(task, frame.projectId, desiredState, reader);
+      task.inFlightEntries.add(entry);
+      if (!this.#readFenceCurrent(task, frame, generation, reader, attempt)) {
+        scrubEntry(entry);
+        task.inFlightEntries.delete(entry);
+        return this.#errorAck(frame, this.#tasks.get(frame.taskId), this.#fenceCode(task, readerGeneration));
+      }
+    }
+
+    const installedRecordVersion = advances
+      ? frame.recordVersion
+      : project.recordVersion;
+    const appliedThrough = clone(frame.delivery);
+    const receipt: ApplyReceipt = {
+      fingerprint,
+      ok: true,
+      attemptId: frame.attemptId,
+      projectId: frame.projectId,
+      recordVersion: installedRecordVersion,
+      result: resultOf(entry),
+      appliedThrough,
+    };
+    if (!this.#rememberApplyReceipt(task, frame.delivery.sequence, receipt)) {
+      scrubEntry(entry);
+      task.inFlightEntries.delete(entry);
+      return this.#exhaustReplayLedger(task, frame);
+    }
+    task.inFlightEntries.delete(entry);
+    if (current !== null && current !== entry) scrubEntry(current);
+    project.records.set(entry.state.key, entry);
+    project.recordVersion = installedRecordVersion;
+    task.appliedSequence = frame.delivery.sequence;
+    if (task.gateOpen) this.#changed(task.taskId);
+    return this.#ackFromApplyReceipt(frame, receipt, task);
+  }
+
+  #commit(
+    task: TaskReplica,
+    frame: ProjectEnvReplicaV1CommitFrame,
+  ): ProjectEnvReplicaV1AckFrame {
+    const fingerprint = canonicalJson({
+      streamId: frame.streamId,
+      attemptId: frame.attemptId,
+      highWater: frame.highWater,
+      versions: frame.versions,
+    });
+    const completed = task.completedAttempts.get(frame.attemptId);
+    if (completed !== undefined) {
+      if (completed.fingerprint !== fingerprint) {
+        return this.#errorAck(frame, task, "reconciliation_attempt_conflict");
+      }
+      return this.#successCommit(frame, completed.highWater, task);
+    }
+    const attempt = task.attempt;
+    if (
+      task.streamId !== frame.streamId ||
+      attempt === null ||
+      attempt.attemptId !== frame.attemptId ||
+      task.gateOpen
+    ) {
+      return this.#errorAck(frame, task, "reconciliation_attempt_mismatch");
+    }
+    if (attempt.readerGeneration !== task.readerGeneration) {
+      return this.#errorAck(frame, task, PROJECT_ENV_READER_CHANGED);
+    }
+    if (frame.highWater.sequence !== task.appliedSequence) {
+      return this.#errorAck(frame, task, "delivery_incomplete");
+    }
+    if (
+      frame.versions.length !== attempt.projectIds.length ||
+      frame.versions.some((version, index) => {
+        const projectId = attempt.projectIds[index];
+        const project = projectId === undefined
+          ? undefined
+          : attempt.projects.get(projectId);
+        return (
+          project === undefined ||
+          version.projectId !== projectId ||
+          !project.complete ||
+          project.recordVersion !== version.recordVersion
+        );
+      })
+    ) {
+      return this.#errorAck(frame, task, "record_version_changed");
+    }
+
+    const completedAttempt: CompletedAttempt = {
+      fingerprint,
+      highWater: clone(frame.highWater),
+    };
+    if (
+      !this.#rememberCompletedAttempt(
+        task,
+        frame.attemptId,
+        completedAttempt,
+      ) ||
+      !this.#rememberRetiredAttempt(task, frame.attemptId)
+    ) {
+      return this.#exhaustReplayLedger(task, frame);
+    }
+
+    const oldProjects = task.projects;
+    task.projects = new Map(
+      attempt.projectIds.map((projectId) => {
+        const staged = attempt.projects.get(projectId)!;
+        return [
+          projectId,
+          {
+            recordVersion: staged.recordVersion,
+            records: staged.records,
+          },
+        ];
+      }),
+    );
+    // Transfer ownership before clearing the attempt; then scrub the old live
+    // map so removed projects and superseded values lose their last reference.
+    for (const project of attempt.projects.values()) project.records = new Map();
+    task.attempt = null;
+    task.snapshotReaderGeneration = attempt.readerGeneration;
+    task.gateOpen = true;
+    task.generation += 1;
+    scrubProjects(oldProjects);
+    this.#resolveReady(task);
+    this.#changed(task.taskId);
+    return this.#successCommit(frame, frame.highWater, task);
+  }
+
+  async #materialize(
+    task: TaskReplica,
+    projectId: string,
+    state: ProjectEnvReplicaBindingState,
+    reader: ProjectEnvTaskValueReader,
+  ): Promise<ReplicaEntry> {
+    const materialization = desiredMaterialization(state, this.policyEpochs);
+    if (materialization.kind === "withheld" || state.head.tombstone) {
+      return { state: clone(state), materialization, value: null };
+    }
+    const ref = state.head.valueRef;
+    if (ref === null) {
+      return {
+        state: clone(state),
+        materialization: { kind: "unavailable", reason: "value_reference_invalid" },
+        value: null,
+      };
+    }
+    try {
+      if (ref.kind === "staged") {
+        const raw = await reader.readStaged(
+          projectId,
+          state.key,
+          ref.stagingId,
+        );
+        if (raw === null) return this.#unavailable(state, "value_missing");
+        const value = parseStagedEnvelope(
+          raw,
+          task.orgId,
+          projectId,
+          state.key,
+          ref.stagingId,
+        );
+        return value === null
+          ? this.#unavailable(state, "value_envelope_mismatch")
+          : {
+              state: clone(state),
+              materialization: { kind: "installed" },
+              value,
+            };
+      }
+      const found = await reader.readLegacy(
+        projectId,
+        state.key,
+        ref.ssmVersion,
+      );
+      if (found === null) return this.#unavailable(state, "value_missing");
+      if (found.version !== ref.ssmVersion) {
+        return this.#unavailable(state, "value_version_mismatch");
+      }
+      const value = parseLegacyEnvelope(
+        found.value,
+        task.orgId,
+        projectId,
+        state.key,
+      );
+      return value === null
+        ? this.#unavailable(state, "value_envelope_mismatch")
+        : {
+            state: clone(state),
+            materialization: { kind: "installed" },
+            value,
+          };
+    } catch {
+      return this.#unavailable(state, "value_unavailable");
+    }
+  }
+
+  #unavailable(
+    state: ProjectEnvReplicaBindingState,
+    reason: string,
+  ): ReplicaEntry {
+    return {
+      state: clone(state),
+      materialization: { kind: "unavailable", reason },
+      value: null,
+    };
+  }
+
+  /** Why a read crossed its fence: the run's reader changed (a newer
+   *  grant's), or anything else. */
+  #fenceCode(task: TaskReplica, readerGeneration: number): string {
+    return this.#tasks.get(task.taskId) === task && task.readerGeneration !== readerGeneration
+      ? PROJECT_ENV_READER_CHANGED
+      : "replica_state_changed";
+  }
+
+  #readFenceCurrent(
+    task: TaskReplica,
+    frame: ProjectEnvReplicaV1PageFrame | ProjectEnvReplicaV1ApplyFrame,
+    generation: number,
+    reader: ProjectEnvTaskValueReader,
+    attempt: SnapshotAttempt | null,
+  ): boolean {
+    return (
+      this.#tasks.get(task.taskId) === task &&
+      task.runEpoch === frame.runEpoch &&
+      frame.runnerBootId === this.runnerBootId &&
+      task.generation === generation &&
+      task.reader === reader &&
+      task.streamId === frame.streamId &&
+      task.attempt === attempt &&
+      (attempt === null
+        ? frame.kind === "env.replica.v1.apply" && frame.attemptId === null
+        : frame.attemptId === attempt.attemptId)
+    );
+  }
+
+  #consumeApplyError(
+    task: TaskReplica,
+    frame: ProjectEnvReplicaV1ApplyFrame,
+    fingerprint: string,
+    code: string,
+  ): ProjectEnvReplicaV1AckFrame {
+    const receipt: ApplyReceipt = {
+      fingerprint,
+      ok: false,
+      attemptId: frame.attemptId,
+      code,
+      appliedThrough: clone(frame.delivery),
+    };
+    if (!this.#rememberApplyReceipt(task, frame.delivery.sequence, receipt)) {
+      return this.#exhaustReplayLedger(task, frame);
+    }
+    task.appliedSequence = frame.delivery.sequence;
+    return this.#ackFromApplyReceipt(frame, receipt, task);
+  }
+
+  #ackFromApplyReceipt(
+    frame: ProjectEnvReplicaV1ApplyFrame,
+    receipt: ApplyReceipt,
+    task: TaskReplica,
+  ): ProjectEnvReplicaV1AckFrame {
+    const common = {
+      kind: "env.replica.v1.ack" as const,
+      opId: frame.opId,
+      taskId: frame.taskId,
+      runEpoch: frame.runEpoch,
+      runnerBootId: frame.runnerBootId,
+      streamId: frame.streamId,
+      attemptId: frame.attemptId,
+      phase: "applied" as const,
+      policyEpochs: [...this.policyEpochs],
+      appliedThrough: this.#cursor(task, frame.streamId),
+    };
+    if (!receipt.ok) {
+      return { ...common, ok: false, code: receipt.code, error: receipt.code };
+    }
+    return {
+      ...common,
+      ok: true,
+      projectId: receipt.projectId,
+      recordVersion: receipt.recordVersion,
+      result: clone(receipt.result),
+    };
+  }
+
+  #successBegin(
+    frame: ProjectEnvReplicaV1BeginFrame,
+    task: TaskReplica,
+  ): ProjectEnvReplicaV1AckFrame {
+    return {
+      ...this.#ackCommon(frame, task),
+      attemptId: frame.attemptId,
+      ok: true,
+      phase: "begun",
+    };
+  }
+
+  #successPage(
+    frame: ProjectEnvReplicaV1PageFrame,
+    receipt: PageReceipt,
+    task: TaskReplica,
+  ): ProjectEnvReplicaV1AckFrame {
+    return {
+      ...this.#ackCommon(frame, task, receipt.appliedThrough),
+      attemptId: frame.attemptId,
+      ok: true,
+      phase: "page",
+      projectId: frame.projectId,
+      after: receipt.after,
+      recordVersion: receipt.recordVersion,
+      results: clone(receipt.results),
+      complete: receipt.complete,
+    };
+  }
+
+  #successCommit(
+    frame: ProjectEnvReplicaV1CommitFrame,
+    highWater: ProjectEnvDeliveryCursor,
+    task: TaskReplica,
+  ): ProjectEnvReplicaV1AckFrame {
+    return {
+      ...this.#ackCommon(frame, task, highWater),
+      attemptId: frame.attemptId,
+      ok: true,
+      phase: "complete",
+      highWater: clone(highWater),
+    };
+  }
+
+  #errorAck(
+    frame: ProjectEnvTaskReplicaFrame,
+    task: TaskReplica | undefined,
+    code: string,
+  ): ProjectEnvReplicaV1AckFrame {
+    return {
+      ...this.#ackCommon(frame, task),
+      ok: false,
+      phase: framePhase(frame),
+      code,
+      error: code,
+    } as ProjectEnvReplicaV1AckFrame;
+  }
+
+  #ackCommon(
+    frame: ProjectEnvTaskReplicaFrame,
+    task: TaskReplica | undefined,
+    cursor?: ProjectEnvDeliveryCursor,
+  ) {
+    const matchedTask =
+      frame.runnerBootId === this.runnerBootId &&
+      task?.runEpoch === frame.runEpoch
+        ? task
+        : undefined;
+    return {
+      kind: "env.replica.v1.ack" as const,
+      opId: frame.opId,
+      taskId: frame.taskId,
+      runEpoch: frame.runEpoch,
+      runnerBootId: frame.runnerBootId,
+      streamId: frame.streamId,
+      attemptId: frame.attemptId,
+      policyEpochs: [...this.policyEpochs],
+      appliedThrough:
+        cursor ?? this.#cursor(matchedTask, frame.streamId),
+    };
+  }
+
+  #cursor(
+    task: TaskReplica | undefined,
+    streamId: string,
+  ): ProjectEnvDeliveryCursor {
+    return {
+      streamId,
+      sequence:
+        task?.streamId === streamId ? task.appliedSequence : 0,
+    };
+  }
+
+  #reserveReplayEvidence(task: TaskReplica, evidence: unknown): number | null {
+    const bytes = replayLedgerEntryBytes(evidence);
+    if (
+      task.replayLedgerEntries + 1 > this.#replayLedgerMaxTaskEntries ||
+      task.replayLedgerBytes + bytes > this.#replayLedgerMaxTaskBytes ||
+      this.#replayLedgerEntries + 1 > this.#replayLedgerMaxTotalEntries ||
+      this.#replayLedgerBytes + bytes > this.#replayLedgerMaxTotalBytes
+    ) {
+      return null;
+    }
+    task.replayLedgerEntries += 1;
+    task.replayLedgerBytes += bytes;
+    this.#replayLedgerEntries += 1;
+    this.#replayLedgerBytes += bytes;
+    return bytes;
+  }
+
+  #releaseReplayEvidence(task: TaskReplica, bytes: number): void {
+    task.replayLedgerEntries -= 1;
+    task.replayLedgerBytes -= bytes;
+    this.#replayLedgerEntries -= 1;
+    this.#replayLedgerBytes -= bytes;
+    if (
+      task.replayLedgerEntries < 0 ||
+      task.replayLedgerBytes < 0 ||
+      this.#replayLedgerEntries < 0 ||
+      this.#replayLedgerBytes < 0
+    ) {
+      throw new Error("task replica replay ledger accounting underflow");
+    }
+  }
+
+  #rememberRetiredStream(task: TaskReplica, streamId: string): boolean {
+    if (task.retiredStreams.has(streamId)) return true;
+    const evidence = ["retired_stream", streamId];
+    if (this.#reserveReplayEvidence(task, evidence) === null) return false;
+    task.retiredStreams.add(streamId);
+    return true;
+  }
+
+  #rememberRetiredAttempt(task: TaskReplica, attemptId: string): boolean {
+    if (task.retiredAttempts.has(attemptId)) return true;
+    const evidence = ["retired_attempt", attemptId];
+    if (this.#reserveReplayEvidence(task, evidence) === null) return false;
+    task.retiredAttempts.add(attemptId);
+    return true;
+  }
+
+  #rememberApplyReceipt(
+    task: TaskReplica,
+    sequence: number,
+    receipt: ApplyReceipt,
+  ): boolean {
+    if (task.applyReceipts.has(sequence)) return true;
+    const evidence = ["apply", sequence, receipt];
+    const bytes = this.#reserveReplayEvidence(task, evidence);
+    if (bytes === null) return false;
+    task.applyReceipts.set(sequence, receipt);
+    task.applyReceiptBytes.set(sequence, bytes);
+    return true;
+  }
+
+  #rememberCompletedAttempt(
+    task: TaskReplica,
+    attemptId: string,
+    completed: CompletedAttempt,
+  ): boolean {
+    if (task.completedAttempts.has(attemptId)) return true;
+    const evidence = ["completed_attempt", attemptId, completed];
+    if (this.#reserveReplayEvidence(task, evidence) === null) return false;
+    task.completedAttempts.set(attemptId, completed);
+    return true;
+  }
+
+  #clearApplyReceipts(task: TaskReplica): void {
+    for (const bytes of task.applyReceiptBytes.values()) {
+      this.#releaseReplayEvidence(task, bytes);
+    }
+    task.applyReceipts.clear();
+    task.applyReceiptBytes.clear();
+  }
+
+  #clearReplayLedger(task: TaskReplica): void {
+    this.#replayLedgerEntries -= task.replayLedgerEntries;
+    this.#replayLedgerBytes -= task.replayLedgerBytes;
+    if (this.#replayLedgerEntries < 0 || this.#replayLedgerBytes < 0) {
+      throw new Error("task replica replay ledger accounting underflow");
+    }
+    task.retiredStreams.clear();
+    task.retiredAttempts.clear();
+    task.applyReceipts.clear();
+    task.applyReceiptBytes.clear();
+    task.completedAttempts.clear();
+    task.replayLedgerEntries = 0;
+    task.replayLedgerBytes = 0;
+  }
+
+  #exhaustReplayLedger(
+    task: TaskReplica,
+    frame: ProjectEnvTaskReplicaFrame,
+  ): ProjectEnvReplicaV1AckFrame {
+    if (!task.restartRequired) {
+      task.restartRequired = true;
+      this.#restartRequiredRuns.set(task.taskId, task.runEpoch);
+      task.generation += 1;
+      task.gateOpen = false;
+      task.reader = CLOSED_READER;
+      scrubAttempt(task.attempt);
+      task.attempt = null;
+      scrubProjects(task.projects);
+      for (const entry of task.inFlightEntries) scrubEntry(entry);
+      task.inFlightEntries.clear();
+      this.#clearReplayLedger(task);
+      for (const waiter of [...task.waiters]) {
+        waiter.reject(new Error("task replica requires a runner restart"));
+      }
+      this.#changed(task.taskId);
+    }
+    return this.#errorAck(frame, task, REPLAY_LEDGER_EXHAUSTED);
+  }
+
+  #resolveReady(task: TaskReplica): void {
+    for (const waiter of [...task.waiters]) waiter.resolve();
+  }
+
+  #destroyTask(task: TaskReplica, reason: string): void {
+    task.generation += 1;
+    task.gateOpen = false;
+    task.reader = CLOSED_READER;
+    scrubAttempt(task.attempt);
+    task.attempt = null;
+    scrubProjects(task.projects);
+    this.#clearReplayLedger(task);
+    for (const entry of task.inFlightEntries) scrubEntry(entry);
+    task.inFlightEntries.clear();
+    for (const waiter of [...task.waiters]) waiter.reject(new Error(reason));
+  }
+
+  #changed(taskId: string): void {
+    try {
+      this.#onChange(taskId);
+    } catch {
+      // Replica authority cannot be rolled back by a cache notification.
+    }
+  }
+}