@tokenoftrust/storefront-runner 2.4.7 → 2.4.9

package/apps/storefront/package.json

@@ -14,7 +14,7 @@
     "preview": "wrangler dev",
     "typecheck": "astro check && tsc --noEmit",
     "astro": "astro",
-    "test": "NODE_OPTIONS=--experimental-sqlite vitest run && pnpm run test:pipeline"
+    "test": "NODE_OPTIONS=--experimental-sqlite vitest run && pnpm run test:pipeline && pnpm run test:pipeline-faults"
   },
   "dependencies": {
     "@astrojs/cloudflare": "^14.0.1",

package/apps/storefront/test/pipeline/fake-preview-services.ts

@@ -17,16 +17,17 @@
  *                                 transient timeout instead of a refusal
  *   - `slow-build`                every file of a build takes a few milliseconds (a build that is
  *                                 still running seconds after its intent was answered)
- * The fake also records what each job reported to MCP (`fake_events` kind `report`) and which
- * shas the preview could not render (`refused`).
+ * The fake also records what each job reported to MCP (`fake_events` kind `report`), which
+ * shas the preview could not render (`refused`), and each undo candidate it opened (`undo`).
  * A fault fires `times` times (0 = every time); its firing is persisted before it acts.
  */
 import type { DurableObjectState } from "cloudflare:workers";
 import { ForgeToolError, ForgeTransportError } from "../../src/lib/forge/errors";
-import type {
-  MembershipEntry,
-  MergeCandidate,
-  PreviewJobServices,
+import {
+  BUILD_CHECKPOINT_EVERY,
+  type MembershipEntry,
+  type MergeCandidate,
+  type PreviewJobServices,
 } from "../../src/lib/pipeline/preview-jobs";
 
 /** Files a fake build materializes. */
@@ -122,7 +123,14 @@ export function readFakeState(state: DurableObjectState): FakeState {
   };
 }
 
-export function fakePreviewServices(state: DurableObjectState): PreviewJobServices {
+/** The file at which a build reports `build:file` to the fault matrix: past its first checkpoint. */
+export const MATRIX_BUILD_FILE = BUILD_CHECKPOINT_EVERY + 5;
+
+/**
+ * `onEffect` hears the one effect only the fake can name: the build writing a file past its first
+ * checkpoint (`build:file`), where an eviction must resume from that checkpoint.
+ */
+export function fakePreviewServices(state: DurableObjectState, onEffect?: (effect: string) => Promise<void>): PreviewJobServices {
   fakeTables(state);
   const { sql } = state.storage;
 
@@ -200,6 +208,7 @@ export function fakePreviewServices(state: DurableObjectState): PreviewJobServic
         if (await fires(`evict-at-file:${file}`)) evict(`at file ${file}`);
         sql.exec(`INSERT INTO fake_file_runs (sha, file) VALUES (?, ?)`, sha, file);
         await resume.record(path, { storedVersionId: `v-${sha}-${file}`, issues: [], skippedHere: false });
+        if (file === MATRIX_BUILD_FILE && onEffect) await onEffect("build:file");
       }
       return { ok: true, versionId: sha, digest: `digest-${sha}` };
     },
@@ -265,8 +274,15 @@ export function fakePreviewServices(state: DurableObjectState): PreviewJobServic
       return { outcome: "conflict" as const, notified: true };
     },
 
-    async prepareUndo(): Promise<MergeCandidate> {
-      throw new Error("prepareUndo is not exercised by the fault harness");
+    async prepareUndo({ target }): Promise<MergeCandidate> {
+      // The forge opens the undo of the change as a candidate of its own, once per change.
+      const of = "prNumber" in target ? `pr-${target.prNumber}` : target.changeId;
+      const changeId = `undo-${of}`;
+      const headSha = `head-${changeId}`;
+      const merged = sql.exec(`SELECT 1 FROM fake_merges WHERE change_id = ?`, changeId).toArray().length > 0;
+      if (!merged) sql.exec(`INSERT OR IGNORE INTO fake_open (change_id, head_sha) VALUES (?, ?)`, changeId, headSha);
+      sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('undo', ?)`, changeId);
+      return { changeId, prNumber: null, headSha, message: `Undo ${of}` };
     },
   };
 }

package/apps/storefront/test/pipeline/harness.ts

@@ -33,7 +33,8 @@ type TenantPipelineStub = {
     | "configureFake"
     | "fakeState"
     | "spans"
-    | "publishHarness"]: TenantPipeline[K];
+    | "publishHarness"
+    | "matrix"]: TenantPipeline[K];
 };
 
 export const harnessEnv = env as unknown as HarnessEnv;

package/apps/storefront/test/pipeline/matrix-faults.ts

@@ -0,0 +1,344 @@
+/**
+ * The systematic fault matrix's injection points inside the harness object (test-only; the suite is
+ * `test/pipeline-faults/`).
+ *
+ * Every job kind the object runs is wrapped so the object can be evicted (`ctx.abort()`) at any
+ * named point of any step:
+ *
+ *   before   the step's first execution, before its work
+ *   after    the step's work is done (every effect landed), before the engine records the outcome
+ *   resume   the step's second execution (a step that checkpointed, waited, or was retried)
+ *   effect   right after one named external effect of the step landed (mid-step), before the step
+ *            goes on — the forge merge, a CI dispatch, a pointer write, a notice, …
+ *
+ * and a step can be HELD (`hold`): its first execution parks inside the step — renewing its lease —
+ * until the test releases it, so "something arrives while this step runs" is a latch, not a race.
+ *
+ * Effects are reported by the stand-ins themselves (`landed(effect)`): the fake preview services,
+ * the publish world, the pointer writer the publish kinds write `live` through, the assets pointer.
+ * Each one is logged against the job and step that caused it (`matrix_effects`), so the suite
+ * derives a step's mid-step kill points from an unfaulted run, and counts duplicates.
+ *
+ * A fault is armed for one kind and step; its remaining count is persisted (and flushed) before it
+ * acts, so it fires once however often the step is re-run after the eviction it causes. At the kill
+ * instant the object records what the convergence guarantee needs (`matrix_kills`): the alarm that
+ * will resume the work, whether the job's compensation is armed, and what `live` names next to the
+ * engine's live record — a kill that leaves them apart with no compensation armed is a finding.
+ */
+import type { DurableObjectState } from "cloudflare:workers";
+import { JobKindRegistry, type JobKindDefinition, type StepContext, type StepDefinition } from "../../src/lib/pipeline/job-runner";
+import type { PreviewJobServices } from "../../src/lib/pipeline/preview-jobs";
+import { readLivePointer } from "../../src/lib/pipeline/publish/live-pointer";
+import { projectionStore, type TestKv } from "../../src/lib/pipeline/publish/publish-test-support";
+
+export type MatrixMode = "before" | "after" | "resume" | "effect" | "hold";
+
+export interface MatrixFault {
+  kind: string;
+  step: string;
+  mode: MatrixMode;
+  /** `effect` mode: the effect after which the object is evicted. */
+  effect?: string | null;
+}
+
+/** What the object looked like at the instant a matrix fault evicted it. */
+export interface MatrixKill {
+  jobId: string;
+  kind: string;
+  step: string;
+  mode: MatrixMode;
+  effect: string | null;
+  /** The alarm that resumes the work (null: nothing will). */
+  alarm: number | null;
+  /** Jobs not yet terminal, and compensations not yet settled. */
+  activeJobs: number;
+  pendingCompensations: number;
+  /** The job's compensation is recorded (armed or pending). */
+  armed: boolean;
+  /** Publish kinds only: what the `live` channel names, and the engine's live record for the job's provider. */
+  live: string | null;
+  record: string | null;
+  /** CloudFront jobs: what the public host serves (the `live` channel follows proven delivery there). */
+  served: string | null;
+}
+
+export interface MatrixEffect {
+  jobId: string;
+  kind: string;
+  step: string;
+  effect: string;
+}
+
+/** Report an external effect that just landed. Evicts the object when a fault is armed for it. */
+export type Landed = (effect: string) => Promise<void>;
+
+/**
+ * The preview services, reporting each external effect a preview job makes. A refresh that ran into
+ * a conflict emails the candidate's author (`email:conflict`); one that did not is `forge:rebase`.
+ */
+export function previewServicesWithEffects(services: PreviewJobServices, landed: Landed): PreviewJobServices {
+  const after =
+    <A extends unknown[], R>(effect: string, call: (...args: A) => Promise<R>) =>
+    async (...args: A): Promise<R> => {
+      const result = await call(...args);
+      await landed(effect);
+      return result;
+    };
+  return {
+    ...services,
+    integrateCandidate: after("forge:merge", services.integrateCandidate.bind(services)),
+    prepareUndo: after("forge:prepare-undo", services.prepareUndo.bind(services)),
+    materialize: after("build:materialized", services.materialize.bind(services)),
+    reportOutcome: after("mcp:report", services.reportOutcome.bind(services)),
+    recordPreviewRefused: after("preview:refused", services.recordPreviewRefused.bind(services)),
+    advancePreview: after("preview:advance", services.advancePreview.bind(services)),
+    stageAggregate: after("ci:stage-aggregate", services.stageAggregate.bind(services)),
+    rebaseCandidate: async (args) => {
+      const result = await services.rebaseCandidate(args);
+      await landed(result.outcome === "conflict" ? "email:conflict" : "forge:rebase");
+      return result;
+    },
+  };
+}
+
+interface Running {
+  jobId: string;
+  kind: string;
+  step: string;
+}
+
+function tables(state: DurableObjectState): void {
+  const { sql } = state.storage;
+  sql.exec(
+    `CREATE TABLE IF NOT EXISTS matrix_faults (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT, step TEXT, mode TEXT, effect TEXT, remaining INTEGER)`,
+  );
+  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_effects (seq INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT, kind TEXT, step TEXT, effect TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_execs (job_id TEXT, step TEXT, n INTEGER, PRIMARY KEY (job_id, step))`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_kills (seq INTEGER PRIMARY KEY AUTOINCREMENT, kill TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_holds (job_id TEXT, step TEXT, released INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (job_id, step))`);
+}
+
+/** The longest a held step waits for its release before it goes on by itself. */
+const HOLD_LIMIT_MS = 30_000;
+
+export class MatrixFaults {
+  private running: Running | null = null;
+
+  constructor(
+    private readonly state: DurableObjectState,
+    /** The KV the `live` channel is projected to (what the edge reads). */
+    private readonly projectionKv: () => TestKv,
+  ) {
+    tables(state);
+  }
+
+  /** Every kind of `registry`, its steps (and compensation) wrapped with the matrix's kill points. */
+  wrap(registry: JobKindRegistry): JobKindRegistry {
+    const out = new JobKindRegistry();
+    for (const name of registry.list()) {
+      const definition = registry.get(name)!;
+      const onTerminal = definition.onTerminal;
+      out.register({
+        ...definition,
+        ...(onTerminal
+          ? {
+              onTerminal: async (info: Parameters<NonNullable<JobKindDefinition<unknown>["onTerminal"]>>[0]) => {
+                const previous = this.running;
+                this.running = { jobId: info.jobId, kind: name, step: "on-terminal" };
+                try {
+                  await onTerminal(info);
+                } finally {
+                  this.running = previous;
+                }
+              },
+            }
+          : {}),
+        steps: definition.steps.map((step) => this.wrapStep(name, step)),
+      });
+    }
+    return out;
+  }
+
+  private wrapStep(kind: string, step: StepDefinition<unknown>): StepDefinition<unknown> {
+    return {
+      ...step,
+      run: async (ctx: StepContext<unknown>) => {
+        const { sql } = this.state.storage;
+        const n = sql
+          .exec<{ n: number }>(
+            `INSERT INTO matrix_execs (job_id, step, n) VALUES (?, ?, 1) ON CONFLICT(job_id, step) DO UPDATE SET n = n + 1 RETURNING n`,
+            ctx.jobId,
+            step.name,
+          )
+          .one().n;
+        this.running = { jobId: ctx.jobId, kind, step: step.name };
+        try {
+          if (n === 1) await this.maybeHold(kind, step.name, ctx);
+          if (n === 1) await this.maybeKill("before", null);
+          if (n === 2) await this.maybeKill("resume", null);
+          const outcome = await step.run(ctx);
+          await this.maybeKill("after", null);
+          return outcome;
+        } finally {
+          if (this.running?.jobId === ctx.jobId && this.running.step === step.name) this.running = null;
+        }
+      },
+    };
+  }
+
+  /** The effect hook handed to every stand-in. */
+  readonly landed: Landed = async (effect) => {
+    const running = this.running;
+    this.state.storage.sql.exec(
+      `INSERT INTO matrix_effects (job_id, kind, step, effect) VALUES (?, ?, ?, ?)`,
+      running?.jobId ?? "",
+      running?.kind ?? "",
+      running?.step ?? "",
+      effect,
+    );
+    if (running) await this.maybeKill("effect", effect);
+  };
+
+  arm(fault: MatrixFault): void {
+    this.state.storage.sql.exec(
+      `INSERT INTO matrix_faults (kind, step, mode, effect, remaining) VALUES (?, ?, ?, ?, 1)`,
+      fault.kind,
+      fault.step,
+      fault.mode,
+      fault.effect ?? null,
+    );
+  }
+
+  /** Steps parked by a `hold` fault and not yet released. */
+  held(): { jobId: string; step: string }[] {
+    return this.state.storage.sql
+      .exec<{ job_id: string; step: string }>(`SELECT job_id, step FROM matrix_holds WHERE released = 0`)
+      .toArray()
+      .map((r) => ({ jobId: r.job_id, step: r.step }));
+  }
+
+  release(): void {
+    this.state.storage.sql.exec(`UPDATE matrix_holds SET released = 1`);
+  }
+
+  private async maybeHold(kind: string, step: string, ctx: StepContext<unknown>): Promise<void> {
+    const { sql } = this.state.storage;
+    const fault = sql
+      .exec<{ id: number }>(`SELECT id FROM matrix_faults WHERE kind = ? AND step = ? AND mode = 'hold' AND remaining > 0 ORDER BY id LIMIT 1`, kind, step)
+      .toArray()[0];
+    if (!fault) return;
+    sql.exec(`UPDATE matrix_faults SET remaining = remaining - 1 WHERE id = ?`, fault.id);
+    sql.exec(`INSERT OR REPLACE INTO matrix_holds (job_id, step, released) VALUES (?, ?, 0)`, ctx.jobId, step);
+    await this.state.storage.sync();
+    const until = Date.now() + HOLD_LIMIT_MS;
+    for (;;) {
+      const row = sql.exec<{ released: number }>(`SELECT released FROM matrix_holds WHERE job_id = ? AND step = ?`, ctx.jobId, step).one();
+      if (row.released === 1 || Date.now() > until) return;
+      await new Promise((resolve) => setTimeout(resolve, 10));
+      await ctx.renewLease();
+    }
+  }
+
+  kills(): MatrixKill[] {
+    return this.state.storage.sql
+      .exec<{ kill: string }>(`SELECT kill FROM matrix_kills ORDER BY seq`)
+      .toArray()
+      .map((r) => JSON.parse(r.kill) as MatrixKill);
+  }
+
+  effects(jobId?: string): MatrixEffect[] {
+    const { sql } = this.state.storage;
+    const rows = jobId
+      ? sql.exec<{ job_id: string; kind: string; step: string; effect: string }>(
+          `SELECT job_id, kind, step, effect FROM matrix_effects WHERE job_id = ? ORDER BY seq`,
+          jobId,
+        )
+      : sql.exec<{ job_id: string; kind: string; step: string; effect: string }>(`SELECT job_id, kind, step, effect FROM matrix_effects ORDER BY seq`);
+    return rows.toArray().map((r) => ({ jobId: r.job_id, kind: r.kind, step: r.step, effect: r.effect }));
+  }
+
+  execs(jobId: string): { step: string; n: number }[] {
+    return this.state.storage.sql.exec<{ step: string; n: number }>(`SELECT step, n FROM matrix_execs WHERE job_id = ?`, jobId).toArray();
+  }
+
+  /** Non-terminal jobs, unsettled compensations and the alarm: an orphan is work no alarm will resume. */
+  async health(): Promise<{
+    alarm: number | null;
+    active: { jobId: string; state: string }[];
+    compensations: { jobId: string; exhausted: boolean; reported: boolean }[];
+  }> {
+    const { sql } = this.state.storage;
+    const active = sql
+      .exec<{ job_id: string; state: string }>(`SELECT job_id, state FROM jobs WHERE state NOT IN ('succeeded', 'needs-developer', 'stuck')`)
+      .toArray()
+      .map((r) => ({ jobId: r.job_id, state: r.state }));
+    const compensations = sql
+      .exec<{ job_id: string; exhausted_at: number | null; reported_at: number | null }>(
+        `SELECT job_id, exhausted_at, reported_at FROM terminal_hooks`,
+      )
+      .toArray()
+      .map((r) => ({ jobId: r.job_id, exhausted: r.exhausted_at !== null, reported: r.reported_at !== null }));
+    return { alarm: await this.state.storage.getAlarm(), active, compensations };
+  }
+
+  private async maybeKill(mode: MatrixMode, effect: string | null): Promise<void> {
+    const running = this.running;
+    if (!running) return;
+    const { sql } = this.state.storage;
+    const fault = sql
+      .exec<{ id: number }>(
+        `SELECT id FROM matrix_faults WHERE kind = ? AND step = ? AND mode = ? AND IFNULL(effect, '') = ? AND remaining > 0 ORDER BY id LIMIT 1`,
+        running.kind,
+        running.step,
+        mode,
+        effect ?? "",
+      )
+      .toArray()[0];
+    if (!fault) return;
+    sql.exec(`UPDATE matrix_faults SET remaining = remaining - 1 WHERE id = ?`, fault.id);
+    const kill = await this.snapshot(running, mode, effect);
+    sql.exec(`INSERT INTO matrix_kills (kill) VALUES (?)`, JSON.stringify(kill));
+    // Everything written so far — the effect itself included — is durable, exactly as a real crash
+    // after it would leave it.
+    await this.state.storage.sync();
+    this.state.abort(`synthetic eviction matrix ${mode} ${running.step}${effect ? ` ${effect}` : ""}`);
+    throw new Error("abort() returned");
+  }
+
+  private async snapshot(running: Running, mode: MatrixMode, effect: string | null): Promise<MatrixKill> {
+    const { sql } = this.state.storage;
+    const job = sql
+      .exec<{ tenant_id: string; input: string; lane: string }>(`SELECT tenant_id, input, lane FROM jobs WHERE job_id = ?`, running.jobId)
+      .toArray()[0];
+    const health = await this.health();
+    let live: string | null = null;
+    let record: string | null = null;
+    let served: string | null = null;
+    // Publish kinds (their input declares the serving provider) are the ones that move `live`.
+    const input = job ? (JSON.parse(job.input) as { provider?: string | null }) : null;
+    if (job?.lane === "live" && input && "provider" in input) {
+      const provider = input.provider ?? "worker-r2-pointer";
+      const kv = this.projectionKv();
+      live = (await readLivePointer(projectionStore(kv), job.tenant_id)).versionId;
+      record = sql.exec<{ sha: string }>(`SELECT sha FROM live WHERE provider = ?`, provider).toArray()[0]?.sha ?? null;
+      if (provider === "s3-overwrite") {
+        const raw = await kv.get("world:cloudfront");
+        served = raw ? ((JSON.parse(raw) as { sha?: string }).sha ?? null) : null;
+      }
+    }
+    return {
+      jobId: running.jobId,
+      kind: running.kind,
+      step: running.step,
+      mode,
+      effect,
+      alarm: health.alarm,
+      activeJobs: health.active.length,
+      pendingCompensations: health.compensations.length,
+      armed: health.compensations.some((c) => c.jobId === running.jobId),
+      live,
+      record,
+      served,
+    };
+  }
+}

package/apps/storefront/test/pipeline/pipeline-host.ts

@@ -4,21 +4,30 @@
  * restarts the object without touching its callers). It runs the production engine with the
  * synthetic job kinds, the production preview kinds (integrate / rebuild / revert / sync-preview) over
  * fake services (`./fake-preview-services.ts`) and the real publish kinds over the stand-in world
- * (`./publish-kinds.ts`), and answers intents through `submitIntent`.
+ * (`./publish-kinds.ts`), and answers intents through `submitIntent`. Every kind is wrapped with the
+ * fault matrix's kill points (`./matrix-faults.ts`), inert until a test arms one.
  */
 import type { DurableObjectState } from "cloudflare:workers";
 import { TenantPipeline as DeployedTenantPipeline } from "../../src/lib/pipeline/tenant-pipeline";
 import { submitIntent, tenantPipeline, type TenantPipelineNamespace } from "../../src/lib/pipeline/intake";
 import { submitPreviewSync } from "../../src/lib/pipeline/preview-sync-intake";
-import { StaleExecutionError, type JobKindRegistry, type PipelineIntent } from "../../src/lib/pipeline/job-runner";
+import { JobKindRegistry, StaleExecutionError, type PipelineIntent } from "../../src/lib/pipeline/job-runner";
 import { abandonedContexts, syntheticJobKinds, syntheticSpans } from "./synthetic-job";
 import { registerPreviewJobKinds, type PreviewJobTiming } from "../../src/lib/pipeline/preview-jobs";
 import { configureFake, fakePreviewServices, readFakeState, type FakeFault, type FakeState } from "./fake-preview-services";
+import { MatrixFaults, previewServicesWithEffects, type Landed, type MatrixFault } from "./matrix-faults";
+import { kvAbandonServices, legacyOperationKey, registerSeedJobKind, type SeedServices } from "../../src/lib/pipeline/seed-from-legacy";
+import { readAssetsPointer } from "@tot/public-runtime";
 import { readLivePointer } from "../../src/lib/pipeline/publish/live-pointer";
-import { listReleaseReceipts, readReleaseTruth } from "../../src/lib/change/store/aggregate-release-store";
+import {
+  listIntegrationRuns,
+  listReleaseReceipts,
+  readReleaseTruth,
+  writeIntegrationRun,
+  type ReleaseModelKv,
+} from "../../src/lib/change/store/aggregate-release-store";
 import { projectionStore, seedGreenAggregate, testVersionStore, type TestKv } from "../../src/lib/pipeline/publish/publish-test-support";
 import {
-  armedAtFaults,
   armPublishFault,
   firedPublishFaults,
   publishWorldOf,
@@ -27,6 +36,9 @@ import {
 } from "./publish-kinds";
 
 /** The preview kinds' timing in the harness: the production shape, scaled to milliseconds. */
+/** The seed kind's lease and retries in the harness. */
+const HARNESS_SEED_TIMING = { leaseMs: 1_000, deadlineMs: 30_000, retryBaseMs: 10 };
+
 export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
   leaseMs: 1_000,
   deadlineMs: 30_000,
@@ -35,13 +47,133 @@ export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
   defaultExpectedMs: 2_000,
 };
 
+/**
+ * The seed's services in the harness: what is served, read from this object's pointer authority (as
+ * the deployed object reads it), and the close-out of legacy records over the harness KV — each
+ * close-out reported to the fault matrix.
+ */
+function harnessSeedServices(
+  ctx: DurableObjectState,
+  pointers: { read(tenantId: string): Promise<{ channels?: { preview?: string; live?: string } }> },
+  landed: Landed,
+): SeedServices {
+  const abandon = kvAbandonServices(publishWorldOf(ctx).kv as unknown as ReleaseModelKv);
+  return {
+    async pointers(tenantId) {
+      const pointer = await pointers.read(tenantId);
+      return { preview: pointer.channels?.preview ?? null, live: pointer.channels?.live ?? null };
+    },
+    async abandonRun(tenantId, runId, at) {
+      const changed = await abandon.abandonRun(tenantId, runId, at);
+      await landed("kv:abandon-run");
+      return changed;
+    },
+    async abandonOp(tenantId, op, at) {
+      const changed = await abandon.abandonOp(tenantId, op, at);
+      await landed("kv:abandon-op");
+      return changed;
+    },
+  };
+}
+
+/** Per object: the fault matrix's injection points and the (wrapped) registry it runs. */
+const matrices = new WeakMap<DurableObjectState, { faults: MatrixFaults; registry: JobKindRegistry }>();
+
 export class TenantPipeline extends DeployedTenantPipeline {
   protected override jobKinds(ctx: DurableObjectState): JobKindRegistry {
-    return registerHarnessPublishKinds(
+    const faults = new MatrixFaults(ctx, () => publishWorldOf(ctx).kv);
+    const services = previewServicesWithEffects(fakePreviewServices(ctx, faults.landed), faults.landed);
+    const registry = registerHarnessPublishKinds(
       ctx,
-      registerPreviewJobKinds(syntheticJobKinds(ctx), fakePreviewServices(ctx), HARNESS_PREVIEW_TIMING),
+      registerPreviewJobKinds(syntheticJobKinds(ctx), services, HARNESS_PREVIEW_TIMING),
       this.pointers,
+      faults.landed,
     );
+    // The production seed kind, its lease and retries scaled to the harness (an evicted seed resumes
+    // when its lease runs out, as in production).
+    const seed = registerSeedJobKind(new JobKindRegistry(), harnessSeedServices(ctx, this.pointers, faults.landed));
+    for (const name of seed.list()) registry.register({ ...seed.get(name)!, ...HARNESS_SEED_TIMING });
+    const wrapped = faults.wrap(registry);
+    matrices.set(ctx, { faults, registry: wrapped });
+    return wrapped;
+  }
+
+  /** Harness-only: arm and read the fault matrix (`./matrix-faults.ts`, `test/pipeline-faults/`). */
+  async matrix(op: string, args: Record<string, unknown> = {}): Promise<unknown> {
+    const { faults, registry } = matrices.get(this.ctx)!;
+    switch (op) {
+      case "arm":
+        faults.arm(args as unknown as MatrixFault);
+        return null;
+      case "held":
+        return faults.held();
+      case "release":
+        faults.release();
+        return null;
+      case "kills":
+        return faults.kills();
+      case "effects":
+        return faults.effects(args.jobId as string | undefined);
+      case "execs":
+        return faults.execs(String(args.jobId));
+      case "health":
+        return faults.health();
+      case "kinds":
+        return registry.list().map((kind) => {
+          const definition = registry.get(kind)!;
+          return { kind, lane: definition.lane ?? "live", steps: definition.steps.map((s) => s.name) };
+        });
+      case "endState":
+        return this.endState(String(args.tenantId));
+      default:
+        throw new Error(`unknown matrix op ${op}`);
+    }
+  }
+
+  /** What a converged job leaves behind, on both lanes — the fault matrix compares it to an unfaulted run. */
+  private async endState(tenantId: string): Promise<Record<string, unknown>> {
+    const { kv, world } = publishWorldOf(this.ctx);
+    const truth = await readReleaseTruth(kv, tenantId, "preview");
+    const fake = readFakeState(this.ctx);
+    return {
+      live: await readLivePointer(projectionStore(kv), tenantId),
+      liveRecords: this.ctx.storage.sql
+        .exec<{ provider: string; sha: string; receipt_id: string }>(`SELECT provider, sha, receipt_id FROM live ORDER BY provider`)
+        .toArray()
+        .map((r) => ({ provider: r.provider, sha: r.sha, receiptId: r.receipt_id })),
+      truth: truth
+        ? {
+            liveReceiptId: truth.liveReceiptId ?? null,
+            pendingReceiptId: truth.pendingReceiptId ?? null,
+            deliveryState: truth.deliveryState ?? null,
+            unprovenLive: truth.unprovenLive ?? null,
+            liveDisagrees: truth.liveDisagrees ? { serving: truth.liveDisagrees.serving, recordedReceiptId: truth.liveDisagrees.recordedReceiptId } : null,
+          }
+        : null,
+      receipts: (await listReleaseReceipts(kv, tenantId, "preview")).map((r) => ({ receiptId: r.receiptId, sha: r.aggregateSha })),
+      promotion: await world.promotionStatus(),
+      cloudfront: await world.cloudFrontServes(),
+      assetsLive: (await readAssetsPointer({ get: (k: string) => kv.get(k) }, tenantId)).live ?? null,
+      legacy: {
+        runs: (await listIntegrationRuns(kv as unknown as ReleaseModelKv, tenantId, "preview")).map((r) => ({ runId: r.runId, state: r.state })),
+        op: await (async () => {
+          const raw = await kv.get(legacyOperationKey(tenantId, { operation: "publication", subjectId: "op-stale" }));
+          return raw ? ((JSON.parse(raw) as { problem?: { code?: string } | null }).problem?.code ?? "open") : null;
+        })(),
+      },
+      preview: {
+        pointer: fake.previewPointer,
+        readModel: fake.readModel,
+        merges: fake.merges.map((m) => ({ changeId: m.changeId, sha: m.sha })),
+        commits: this.ctx.storage.sql
+          .exec<{ sha: string; membership: string }>(`SELECT sha, membership FROM commits ORDER BY sha`)
+          .toArray()
+          .map((r) => ({ sha: r.sha, membership: JSON.parse(r.membership) as unknown })),
+        evidence: this.ctx.storage.sql
+          .exec<{ sha: string; digest: string }>(`SELECT sha, digest FROM evidence ORDER BY sha`)
+          .toArray(),
+      },
+    };
   }
 
   /**
@@ -123,13 +255,46 @@ export class TenantPipeline extends DeployedTenantPipeline {
           Number(args.times ?? 1),
         );
         return null;
+      case "seedPointers":
+        // What the shared preview and the live site are served from, as a legacy store left them.
+        await this.pointers.update(tenantId, (current) => ({
+          ...current,
+          channels: { ...current.channels, preview: String(args.preview), live: String(args.live) },
+          channelRevs: { ...current.channelRevs, preview: (current.channelRevs?.preview ?? 0) + 1, live: (current.channelRevs?.live ?? 0) + 1 },
+        }));
+        return null;
+      case "seedLegacy": {
+        // A request-scoped run and an operation that never finished.
+        const at = "2026-10-09T00:00:00.000Z";
+        await writeIntegrationRun(
+          kv as unknown as ReleaseModelKv,
+          {
+            tenantId,
+            aggregateId: "preview",
+            runId: String(args.runId),
+            aggregateSha: String(args.sha),
+            includedPrs: [],
+            state: "running",
+            previousRunId: null,
+            artifactDigest: null,
+            startedAt: at,
+            finishedAt: null,
+            statusMessage: null,
+          },
+          0,
+        );
+        const op = { operation: "publication" as const, subjectId: String(args.opSubject) };
+        await kv.put(
+          legacyOperationKey(tenantId, op),
+          JSON.stringify({ tenantId, ...op, revision: String(args.sha), stage: "publishing", problem: null, attemptsMade: 1, updatedAt: at, rev: 0 }),
+        );
+        return null;
+      }
       case "armFault":
         armPublishFault(this.ctx, args as unknown as PublishFault);
         return null;
       case "firedFaults":
         return firedPublishFaults(this.ctx);
-      case "armedAtFaults":
-        return armedAtFaults(this.ctx);
       default:
         throw new Error(`unknown publish harness op ${op}`);
     }