@tokenoftrust/storefront-runner 2.4.6 → 2.4.8

package/apps/storefront/migrations-apps/0014_retire_candidate_ledger.sql

@@ -0,0 +1,12 @@
+-- The candidate preparation ledger has no writer or reader: candidate progress lives in the KV
+-- ReviewEnvironment index, and the release state in the tenant pipeline. Its tables are renamed
+-- aside, not dropped, so the rows survive until a backup is taken; rename back to restore. The
+-- indexes are dropped so their names are free (they are re-creatable from the rows).
+DROP INDEX IF EXISTS `idx_candidate_attempts_tenant`;
+DROP INDEX IF EXISTS `idx_candidate_attempts_candidate`;
+DROP INDEX IF EXISTS `uq_candidate_attempts_candidate_number`;
+ALTER TABLE `candidate_attempts` RENAME TO `retired_candidate_attempts_20261010`;
+DROP INDEX IF EXISTS `idx_candidates_tenant_status`;
+DROP INDEX IF EXISTS `uq_candidates_tenant_change`;
+DROP INDEX IF EXISTS `uq_candidates_tenant_repo_pr`;
+ALTER TABLE `candidates` RENAME TO `retired_candidates_20261010`;

package/apps/storefront/package.json

@@ -14,7 +14,7 @@
     "preview": "wrangler dev",
     "typecheck": "astro check && tsc --noEmit",
     "astro": "astro",
-    "test": "NODE_OPTIONS=--experimental-sqlite vitest run && pnpm run test:pipeline"
+    "test": "NODE_OPTIONS=--experimental-sqlite vitest run && pnpm run test:pipeline && pnpm run test:pipeline-faults"
   },
   "dependencies": {
     "@astrojs/cloudflare": "^14.0.1",

package/apps/storefront/src/lib/d1/schema-apps.ts

@@ -564,94 +564,6 @@ export const newsletterSends = sqliteTable(
   (t) => ({ pk: primaryKey({ columns: [t.tenantId, t.collection, t.digestId] }) }),
 );
 
-// --- Canonical publish-candidate identity -----------------------------------
-// ONE canonical candidate per `(tenant_id, repository, pr_number)` — the real
-// forge PR. Retries/rebuilds are NOT new candidates; they are child `attempts`
-// (below). This is the durable identity spine other units resolve against; the
-// KV `ReviewEnvironment` (`cust:*`) and aggregate/release models (`rel:*`) are
-// read projections that reference this row's `change_id`/`pr_number`.
-//
-// `id` is a SYNTHETIC surrogate (`cand_<uuid>`), deliberately DISTINCT from the
-// canonical key so foreign keys (attempts) never depend on the mutable natural
-// key. `change_id` is the REAL forge change id — the stable branch/build
-// identity (`candidate/<change_id>`) — preserved, never derived from `id`.
-// `tenant_id` is the isolation boundary, written from the resolved session,
-// NEVER a client body. The UNIQUE `(tenant_id, repository, pr_number)` is what
-// makes "one candidate per PR" a schema invariant (upsert-by-key, not
-// insert-then-dedupe); the UNIQUE `(tenant_id, change_id)` keeps the branch/
-// build identity unambiguous for resolve-by-changeId callers.
-export const candidates = sqliteTable(
-  "candidates",
-  {
-    id: text("id").primaryKey(),
-    tenantId: text("tenant_id").notNull(),
-    // The `<tenant>-<tag>` forge repository the PR lives in.
-    repository: text("repository").notNull(),
-    // The forge PR number — the third leg of the canonical key.
-    prNumber: integer("pr_number").notNull(),
-    // The real forge change id — stable branch/build identity (`candidate/<change_id>`).
-    changeId: text("change_id").notNull(),
-    // Source branch, conventionally `candidate/<change_id>` (nullable convenience mirror).
-    branch: text("branch"),
-    title: text("title"),
-    author: text("author"),
-    // Canonical candidate lifecycle: open | preparing | ready | needs-attention | accepted | rejected | closed.
-    status: text("status").notNull().default("open"),
-    // The PR head/base shas as last reconciled (nullable until first read).
-    headSha: text("head_sha"),
-    baseSha: text("base_sha"),
-    createdAt: text("created_at").notNull(),
-    updatedAt: text("updated_at").notNull(),
-  },
-  (t) => ({
-    uqCandidatesCanonical: uniqueIndex("uq_candidates_tenant_repo_pr").on(
-      t.tenantId,
-      t.repository,
-      t.prNumber,
-    ),
-    uqCandidatesChange: uniqueIndex("uq_candidates_tenant_change").on(t.tenantId, t.changeId),
-    idxCandidatesTenantStatus: index("idx_candidates_tenant_status").on(t.tenantId, t.status),
-  }),
-);
-
-// One `attempt` = one build/check/rebase EXECUTION of a candidate. Attempts are
-// the child rows retries produce — they must NEVER queue as separate top-level
-// candidate rows. `candidate_id` references `candidates.id` in code only (no SQL
-// FK — repo convention, matches `webhook_deliveries.event_id` / `orders`).
-// `tenant_id` is denormalized for tenant-scoped reads/isolation without a join.
-// `attempt_number` is monotonic per candidate (1,2,3…) giving stable ordering;
-// UNIQUE `(candidate_id, attempt_number)` makes a double-assigned number a loud
-// failure rather than a silent duplicate. `kind`: build | check | rebase;
-// `status`: pending | running | passed | failed | cancelled (mirrors
-// `IntegrationRun`'s state vocabulary). `evidence` is a JSON EvidenceReport
-// snapshot; `detail` a human-legible terminal-state message.
-export const candidateAttempts = sqliteTable(
-  "candidate_attempts",
-  {
-    id: text("id").primaryKey(),
-    candidateId: text("candidate_id").notNull(),
-    tenantId: text("tenant_id").notNull(),
-    attemptNumber: integer("attempt_number").notNull(),
-    kind: text("kind").notNull(),
-    status: text("status").notNull().default("pending"),
-    // The candidate head sha this attempt executed against, or null if unknown.
-    headSha: text("head_sha"),
-    detail: text("detail"),
-    evidence: text("evidence"),
-    startedAt: text("started_at").notNull(),
-    // Null while pending/running; set on a terminal transition.
-    finishedAt: text("finished_at"),
-  },
-  (t) => ({
-    uqAttemptNumber: uniqueIndex("uq_candidate_attempts_candidate_number").on(
-      t.candidateId,
-      t.attemptNumber,
-    ),
-    idxAttemptsCandidate: index("idx_candidate_attempts_candidate").on(t.candidateId),
-    idxAttemptsTenant: index("idx_candidate_attempts_tenant").on(t.tenantId),
-  }),
-);
-
 // One active submit per tenant/change. This serializes identity authorization,
 // the Gitea mutation, and audit finalization so a verified callback cannot
 // become stale between its forge read and its KV writes. `expires_at` is

package/apps/storefront/test/pipeline/fake-preview-services.ts

@@ -17,17 +17,17 @@
  *                                 transient timeout instead of a refusal
  *   - `slow-build`                every file of a build takes a few milliseconds (a build that is
  *                                 still running seconds after its intent was answered)
- * The fake also records what each job reported to MCP (`fake_events` kind `report`) and which
- * shas the preview could not render (`refused`).
+ * The fake also records what each job reported to MCP (`fake_events` kind `report`), which
+ * shas the preview could not render (`refused`), and each undo candidate it opened (`undo`).
  * A fault fires `times` times (0 = every time); its firing is persisted before it acts.
  */
 import type { DurableObjectState } from "cloudflare:workers";
 import { ForgeToolError, ForgeTransportError } from "../../src/lib/forge/errors";
-import type {
-  MembershipEntry,
-  MergeCandidate,
-  PreviewJobServices,
-  ReadModelTip,
+import {
+  BUILD_CHECKPOINT_EVERY,
+  type MembershipEntry,
+  type MergeCandidate,
+  type PreviewJobServices,
 } from "../../src/lib/pipeline/preview-jobs";
 
 /** Files a fake build materializes. */
@@ -123,7 +123,14 @@ export function readFakeState(state: DurableObjectState): FakeState {
   };
 }
 
-export function fakePreviewServices(state: DurableObjectState): PreviewJobServices {
+/** The file at which a build reports `build:file` to the fault matrix: past its first checkpoint. */
+export const MATRIX_BUILD_FILE = BUILD_CHECKPOINT_EVERY + 5;
+
+/**
+ * `onEffect` hears the one effect only the fake can name: the build writing a file past its first
+ * checkpoint (`build:file`), where an eviction must resume from that checkpoint.
+ */
+export function fakePreviewServices(state: DurableObjectState, onEffect?: (effect: string) => Promise<void>): PreviewJobServices {
   fakeTables(state);
   const { sql } = state.storage;
 
@@ -176,11 +183,6 @@ export function fakePreviewServices(state: DurableObjectState): PreviewJobServic
       return row?.sha ?? null;
     },
 
-    async readModelTip(): Promise<ReadModelTip | null> {
-      const rm = sql.exec<{ sha: string; membership: string; green: number }>(`SELECT * FROM fake_read_model`).toArray()[0];
-      return rm ? { sha: rm.sha, membership: JSON.parse(rm.membership), green: rm.green === 1 } : null;
-    },
-
     async materialize({ sha, resume }) {
       if (await fires("evict-before-build")) evict("before build");
       const moveTo = sql
@@ -206,6 +208,7 @@ export function fakePreviewServices(state: DurableObjectState): PreviewJobServic
         if (await fires(`evict-at-file:${file}`)) evict(`at file ${file}`);
         sql.exec(`INSERT INTO fake_file_runs (sha, file) VALUES (?, ?)`, sha, file);
         await resume.record(path, { storedVersionId: `v-${sha}-${file}`, issues: [], skippedHere: false });
+        if (file === MATRIX_BUILD_FILE && onEffect) await onEffect("build:file");
       }
       return { ok: true, versionId: sha, digest: `digest-${sha}` };
     },
@@ -271,8 +274,15 @@ export function fakePreviewServices(state: DurableObjectState): PreviewJobServic
       return { outcome: "conflict" as const, notified: true };
     },
 
-    async prepareUndo(): Promise<MergeCandidate> {
-      throw new Error("prepareUndo is not exercised by the fault harness");
+    async prepareUndo({ target }): Promise<MergeCandidate> {
+      // The forge opens the undo of the change as a candidate of its own, once per change.
+      const of = "prNumber" in target ? `pr-${target.prNumber}` : target.changeId;
+      const changeId = `undo-${of}`;
+      const headSha = `head-${changeId}`;
+      const merged = sql.exec(`SELECT 1 FROM fake_merges WHERE change_id = ?`, changeId).toArray().length > 0;
+      if (!merged) sql.exec(`INSERT OR IGNORE INTO fake_open (change_id, head_sha) VALUES (?, ?)`, changeId, headSha);
+      sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('undo', ?)`, changeId);
+      return { changeId, prNumber: null, headSha, message: `Undo ${of}` };
     },
   };
 }

package/apps/storefront/test/pipeline/harness.ts

@@ -33,7 +33,8 @@ type TenantPipelineStub = {
     | "configureFake"
     | "fakeState"
     | "spans"
-    | "publishHarness"]: TenantPipeline[K];
+    | "publishHarness"
+    | "matrix"]: TenantPipeline[K];
 };
 
 export const harnessEnv = env as unknown as HarnessEnv;

package/apps/storefront/test/pipeline/matrix-faults.ts

@@ -0,0 +1,344 @@
+/**
+ * The systematic fault matrix's injection points inside the harness object (test-only; the suite is
+ * `test/pipeline-faults/`).
+ *
+ * Every job kind the object runs is wrapped so the object can be evicted (`ctx.abort()`) at any
+ * named point of any step:
+ *
+ *   before   the step's first execution, before its work
+ *   after    the step's work is done (every effect landed), before the engine records the outcome
+ *   resume   the step's second execution (a step that checkpointed, waited, or was retried)
+ *   effect   right after one named external effect of the step landed (mid-step), before the step
+ *            goes on — the forge merge, a CI dispatch, a pointer write, a notice, …
+ *
+ * and a step can be HELD (`hold`): its first execution parks inside the step — renewing its lease —
+ * until the test releases it, so "something arrives while this step runs" is a latch, not a race.
+ *
+ * Effects are reported by the stand-ins themselves (`landed(effect)`): the fake preview services,
+ * the publish world, the pointer writer the publish kinds write `live` through, the assets pointer.
+ * Each one is logged against the job and step that caused it (`matrix_effects`), so the suite
+ * derives a step's mid-step kill points from an unfaulted run, and counts duplicates.
+ *
+ * A fault is armed for one kind and step; its remaining count is persisted (and flushed) before it
+ * acts, so it fires once however often the step is re-run after the eviction it causes. At the kill
+ * instant the object records what the convergence guarantee needs (`matrix_kills`): the alarm that
+ * will resume the work, whether the job's compensation is armed, and what `live` names next to the
+ * engine's live record — a kill that leaves them apart with no compensation armed is a finding.
+ */
+import type { DurableObjectState } from "cloudflare:workers";
+import { JobKindRegistry, type JobKindDefinition, type StepContext, type StepDefinition } from "../../src/lib/pipeline/job-runner";
+import type { PreviewJobServices } from "../../src/lib/pipeline/preview-jobs";
+import { readLivePointer } from "../../src/lib/pipeline/publish/live-pointer";
+import { projectionStore, type TestKv } from "../../src/lib/pipeline/publish/publish-test-support";
+
+export type MatrixMode = "before" | "after" | "resume" | "effect" | "hold";
+
+export interface MatrixFault {
+  kind: string;
+  step: string;
+  mode: MatrixMode;
+  /** `effect` mode: the effect after which the object is evicted. */
+  effect?: string | null;
+}
+
+/** What the object looked like at the instant a matrix fault evicted it. */
+export interface MatrixKill {
+  jobId: string;
+  kind: string;
+  step: string;
+  mode: MatrixMode;
+  effect: string | null;
+  /** The alarm that resumes the work (null: nothing will). */
+  alarm: number | null;
+  /** Jobs not yet terminal, and compensations not yet settled. */
+  activeJobs: number;
+  pendingCompensations: number;
+  /** The job's compensation is recorded (armed or pending). */
+  armed: boolean;
+  /** Publish kinds only: what the `live` channel names, and the engine's live record for the job's provider. */
+  live: string | null;
+  record: string | null;
+  /** CloudFront jobs: what the public host serves (the `live` channel follows proven delivery there). */
+  served: string | null;
+}
+
+export interface MatrixEffect {
+  jobId: string;
+  kind: string;
+  step: string;
+  effect: string;
+}
+
+/** Report an external effect that just landed. Evicts the object when a fault is armed for it. */
+export type Landed = (effect: string) => Promise<void>;
+
+/**
+ * The preview services, reporting each external effect a preview job makes. A refresh that ran into
+ * a conflict emails the candidate's author (`email:conflict`); one that did not is `forge:rebase`.
+ */
+export function previewServicesWithEffects(services: PreviewJobServices, landed: Landed): PreviewJobServices {
+  const after =
+    <A extends unknown[], R>(effect: string, call: (...args: A) => Promise<R>) =>
+    async (...args: A): Promise<R> => {
+      const result = await call(...args);
+      await landed(effect);
+      return result;
+    };
+  return {
+    ...services,
+    integrateCandidate: after("forge:merge", services.integrateCandidate.bind(services)),
+    prepareUndo: after("forge:prepare-undo", services.prepareUndo.bind(services)),
+    materialize: after("build:materialized", services.materialize.bind(services)),
+    reportOutcome: after("mcp:report", services.reportOutcome.bind(services)),
+    recordPreviewRefused: after("preview:refused", services.recordPreviewRefused.bind(services)),
+    advancePreview: after("preview:advance", services.advancePreview.bind(services)),
+    stageAggregate: after("ci:stage-aggregate", services.stageAggregate.bind(services)),
+    rebaseCandidate: async (args) => {
+      const result = await services.rebaseCandidate(args);
+      await landed(result.outcome === "conflict" ? "email:conflict" : "forge:rebase");
+      return result;
+    },
+  };
+}
+
+interface Running {
+  jobId: string;
+  kind: string;
+  step: string;
+}
+
+function tables(state: DurableObjectState): void {
+  const { sql } = state.storage;
+  sql.exec(
+    `CREATE TABLE IF NOT EXISTS matrix_faults (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT, step TEXT, mode TEXT, effect TEXT, remaining INTEGER)`,
+  );
+  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_effects (seq INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT, kind TEXT, step TEXT, effect TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_execs (job_id TEXT, step TEXT, n INTEGER, PRIMARY KEY (job_id, step))`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_kills (seq INTEGER PRIMARY KEY AUTOINCREMENT, kill TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_holds (job_id TEXT, step TEXT, released INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (job_id, step))`);
+}
+
+/** The longest a held step waits for its release before it goes on by itself. */
+const HOLD_LIMIT_MS = 30_000;
+
+export class MatrixFaults {
+  private running: Running | null = null;
+
+  constructor(
+    private readonly state: DurableObjectState,
+    /** The KV the `live` channel is projected to (what the edge reads). */
+    private readonly projectionKv: () => TestKv,
+  ) {
+    tables(state);
+  }
+
+  /** Every kind of `registry`, its steps (and compensation) wrapped with the matrix's kill points. */
+  wrap(registry: JobKindRegistry): JobKindRegistry {
+    const out = new JobKindRegistry();
+    for (const name of registry.list()) {
+      const definition = registry.get(name)!;
+      const onTerminal = definition.onTerminal;
+      out.register({
+        ...definition,
+        ...(onTerminal
+          ? {
+              onTerminal: async (info: Parameters<NonNullable<JobKindDefinition<unknown>["onTerminal"]>>[0]) => {
+                const previous = this.running;
+                this.running = { jobId: info.jobId, kind: name, step: "on-terminal" };
+                try {
+                  await onTerminal(info);
+                } finally {
+                  this.running = previous;
+                }
+              },
+            }
+          : {}),
+        steps: definition.steps.map((step) => this.wrapStep(name, step)),
+      });
+    }
+    return out;
+  }
+
+  private wrapStep(kind: string, step: StepDefinition<unknown>): StepDefinition<unknown> {
+    return {
+      ...step,
+      run: async (ctx: StepContext<unknown>) => {
+        const { sql } = this.state.storage;
+        const n = sql
+          .exec<{ n: number }>(
+            `INSERT INTO matrix_execs (job_id, step, n) VALUES (?, ?, 1) ON CONFLICT(job_id, step) DO UPDATE SET n = n + 1 RETURNING n`,
+            ctx.jobId,
+            step.name,
+          )
+          .one().n;
+        this.running = { jobId: ctx.jobId, kind, step: step.name };
+        try {
+          if (n === 1) await this.maybeHold(kind, step.name, ctx);
+          if (n === 1) await this.maybeKill("before", null);
+          if (n === 2) await this.maybeKill("resume", null);
+          const outcome = await step.run(ctx);
+          await this.maybeKill("after", null);
+          return outcome;
+        } finally {
+          if (this.running?.jobId === ctx.jobId && this.running.step === step.name) this.running = null;
+        }
+      },
+    };
+  }
+
+  /** The effect hook handed to every stand-in. */
+  readonly landed: Landed = async (effect) => {
+    const running = this.running;
+    this.state.storage.sql.exec(
+      `INSERT INTO matrix_effects (job_id, kind, step, effect) VALUES (?, ?, ?, ?)`,
+      running?.jobId ?? "",
+      running?.kind ?? "",
+      running?.step ?? "",
+      effect,
+    );
+    if (running) await this.maybeKill("effect", effect);
+  };
+
+  arm(fault: MatrixFault): void {
+    this.state.storage.sql.exec(
+      `INSERT INTO matrix_faults (kind, step, mode, effect, remaining) VALUES (?, ?, ?, ?, 1)`,
+      fault.kind,
+      fault.step,
+      fault.mode,
+      fault.effect ?? null,
+    );
+  }
+
+  /** Steps parked by a `hold` fault and not yet released. */
+  held(): { jobId: string; step: string }[] {
+    return this.state.storage.sql
+      .exec<{ job_id: string; step: string }>(`SELECT job_id, step FROM matrix_holds WHERE released = 0`)
+      .toArray()
+      .map((r) => ({ jobId: r.job_id, step: r.step }));
+  }
+
+  release(): void {
+    this.state.storage.sql.exec(`UPDATE matrix_holds SET released = 1`);
+  }
+
+  private async maybeHold(kind: string, step: string, ctx: StepContext<unknown>): Promise<void> {
+    const { sql } = this.state.storage;
+    const fault = sql
+      .exec<{ id: number }>(`SELECT id FROM matrix_faults WHERE kind = ? AND step = ? AND mode = 'hold' AND remaining > 0 ORDER BY id LIMIT 1`, kind, step)
+      .toArray()[0];
+    if (!fault) return;
+    sql.exec(`UPDATE matrix_faults SET remaining = remaining - 1 WHERE id = ?`, fault.id);
+    sql.exec(`INSERT OR REPLACE INTO matrix_holds (job_id, step, released) VALUES (?, ?, 0)`, ctx.jobId, step);
+    await this.state.storage.sync();
+    const until = Date.now() + HOLD_LIMIT_MS;
+    for (;;) {
+      const row = sql.exec<{ released: number }>(`SELECT released FROM matrix_holds WHERE job_id = ? AND step = ?`, ctx.jobId, step).one();
+      if (row.released === 1 || Date.now() > until) return;
+      await new Promise((resolve) => setTimeout(resolve, 10));
+      await ctx.renewLease();
+    }
+  }
+
+  kills(): MatrixKill[] {
+    return this.state.storage.sql
+      .exec<{ kill: string }>(`SELECT kill FROM matrix_kills ORDER BY seq`)
+      .toArray()
+      .map((r) => JSON.parse(r.kill) as MatrixKill);
+  }
+
+  effects(jobId?: string): MatrixEffect[] {
+    const { sql } = this.state.storage;
+    const rows = jobId
+      ? sql.exec<{ job_id: string; kind: string; step: string; effect: string }>(
+          `SELECT job_id, kind, step, effect FROM matrix_effects WHERE job_id = ? ORDER BY seq`,
+          jobId,
+        )
+      : sql.exec<{ job_id: string; kind: string; step: string; effect: string }>(`SELECT job_id, kind, step, effect FROM matrix_effects ORDER BY seq`);
+    return rows.toArray().map((r) => ({ jobId: r.job_id, kind: r.kind, step: r.step, effect: r.effect }));
+  }
+
+  execs(jobId: string): { step: string; n: number }[] {
+    return this.state.storage.sql.exec<{ step: string; n: number }>(`SELECT step, n FROM matrix_execs WHERE job_id = ?`, jobId).toArray();
+  }
+
+  /** Non-terminal jobs, unsettled compensations and the alarm: an orphan is work no alarm will resume. */
+  async health(): Promise<{
+    alarm: number | null;
+    active: { jobId: string; state: string }[];
+    compensations: { jobId: string; exhausted: boolean; reported: boolean }[];
+  }> {
+    const { sql } = this.state.storage;
+    const active = sql
+      .exec<{ job_id: string; state: string }>(`SELECT job_id, state FROM jobs WHERE state NOT IN ('succeeded', 'needs-developer', 'stuck')`)
+      .toArray()
+      .map((r) => ({ jobId: r.job_id, state: r.state }));
+    const compensations = sql
+      .exec<{ job_id: string; exhausted_at: number | null; reported_at: number | null }>(
+        `SELECT job_id, exhausted_at, reported_at FROM terminal_hooks`,
+      )
+      .toArray()
+      .map((r) => ({ jobId: r.job_id, exhausted: r.exhausted_at !== null, reported: r.reported_at !== null }));
+    return { alarm: await this.state.storage.getAlarm(), active, compensations };
+  }
+
+  private async maybeKill(mode: MatrixMode, effect: string | null): Promise<void> {
+    const running = this.running;
+    if (!running) return;
+    const { sql } = this.state.storage;
+    const fault = sql
+      .exec<{ id: number }>(
+        `SELECT id FROM matrix_faults WHERE kind = ? AND step = ? AND mode = ? AND IFNULL(effect, '') = ? AND remaining > 0 ORDER BY id LIMIT 1`,
+        running.kind,
+        running.step,
+        mode,
+        effect ?? "",
+      )
+      .toArray()[0];
+    if (!fault) return;
+    sql.exec(`UPDATE matrix_faults SET remaining = remaining - 1 WHERE id = ?`, fault.id);
+    const kill = await this.snapshot(running, mode, effect);
+    sql.exec(`INSERT INTO matrix_kills (kill) VALUES (?)`, JSON.stringify(kill));
+    // Everything written so far — the effect itself included — is durable, exactly as a real crash
+    // after it would leave it.
+    await this.state.storage.sync();
+    this.state.abort(`synthetic eviction matrix ${mode} ${running.step}${effect ? ` ${effect}` : ""}`);
+    throw new Error("abort() returned");
+  }
+
+  private async snapshot(running: Running, mode: MatrixMode, effect: string | null): Promise<MatrixKill> {
+    const { sql } = this.state.storage;
+    const job = sql
+      .exec<{ tenant_id: string; input: string; lane: string }>(`SELECT tenant_id, input, lane FROM jobs WHERE job_id = ?`, running.jobId)
+      .toArray()[0];
+    const health = await this.health();
+    let live: string | null = null;
+    let record: string | null = null;
+    let served: string | null = null;
+    // Publish kinds (their input declares the serving provider) are the ones that move `live`.
+    const input = job ? (JSON.parse(job.input) as { provider?: string | null }) : null;
+    if (job?.lane === "live" && input && "provider" in input) {
+      const provider = input.provider ?? "worker-r2-pointer";
+      const kv = this.projectionKv();
+      live = (await readLivePointer(projectionStore(kv), job.tenant_id)).versionId;
+      record = sql.exec<{ sha: string }>(`SELECT sha FROM live WHERE provider = ?`, provider).toArray()[0]?.sha ?? null;
+      if (provider === "s3-overwrite") {
+        const raw = await kv.get("world:cloudfront");
+        served = raw ? ((JSON.parse(raw) as { sha?: string }).sha ?? null) : null;
+      }
+    }
+    return {
+      jobId: running.jobId,
+      kind: running.kind,
+      step: running.step,
+      mode,
+      effect,
+      alarm: health.alarm,
+      activeJobs: health.active.length,
+      pendingCompensations: health.compensations.length,
+      armed: health.compensations.some((c) => c.jobId === running.jobId),
+      live,
+      record,
+      served,
+    };
+  }
+}