@tokenoftrust/storefront-runner 2.4.7 → 2.4.9

package/apps/storefront/test/pipeline-faults/matrix-lib.ts

@@ -0,0 +1,661 @@
+/**
+ * The fault matrix's scenarios, driver and convergence assertions (the cells are in
+ * `./fault-matrix.test.ts`; the kill points inside the object are `../pipeline/matrix-faults.ts`).
+ *
+ * A SCENARIO is one job kind on one provider, from a fresh tenant: what the world holds before the
+ * job (`setup`, which may run prerequisite jobs unfaulted) and the intent that starts it. The preview
+ * kinds take no provider — the shared preview is Worker-served for Cloudflare and CloudFront tenants
+ * alike, so they run once; every live-lane kind runs on both.
+ *
+ * A CELL runs a scenario with one fault armed and drives the job to its end the way the world would:
+ * a CI dispatch is completed and its callback delivered, a hotfix's static prebuild is staged and
+ * reported. The cell's outcome is then held against the scenario's unfaulted run (`baselineOf`).
+ */
+import type { JobView, TransitionView } from "../../src/lib/pipeline/job-runner";
+import type { MatrixEffect, MatrixFault, MatrixKill } from "../pipeline/matrix-faults";
+import type { CiDispatch } from "../../src/lib/pipeline/publish/publish-test-support";
+import { pipelineFor, uniqueTenant } from "../pipeline/harness";
+
+export type Provider = "worker-r2-pointer" | "s3-overwrite";
+
+export const PROVIDER_NAME: Record<Provider, string> = { "worker-r2-pointer": "cloudflare", "s3-overwrite": "cloudfront" };
+
+const TERMINAL = new Set(["succeeded", "needs-developer", "stuck"]);
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+type Stub = ReturnType<typeof pipelineFor>;
+
+/** One tenant's object, called the way a real caller would: a fresh stub per call, retried while it restarts. */
+export class Tenant {
+  /** The shas staged for the pointer plane so far (the world's staged table is replaced whole). */
+  readonly staged: string[] = [];
+
+  constructor(readonly id: string) {}
+
+  async call<T>(fn: (stub: Stub) => Promise<T>): Promise<T> {
+    for (let i = 0; ; i += 1) {
+      try {
+        return await fn(pipelineFor(this.id));
+      } catch (err) {
+        if (i > 400 || !/synthetic eviction|abort/i.test(String((err as Error).message))) throw err;
+        await sleep(25);
+      }
+    }
+  }
+
+  harness<T = unknown>(op: string, args: Record<string, unknown> = {}): Promise<T> {
+    return this.call((stub) => stub.publishHarness(op, { tenantId: this.id, ...args })) as Promise<T>;
+  }
+
+  matrix<T = unknown>(op: string, args: Record<string, unknown> = {}): Promise<T> {
+    return this.call((stub) => stub.matrix(op, args)) as Promise<T>;
+  }
+
+  async submit(kind: string, input: unknown): Promise<string> {
+    const result = await this.call((stub) =>
+      stub.submit(this.id, { kind, input, actor: { type: "test", id: "fault-matrix" }, admissionDecisionId: "decision-matrix" }),
+    );
+    if (!result.ok) throw new Error(`${kind} refused at intake: ${result.reason}`);
+    return result.jobId;
+  }
+
+  job(jobId: string): Promise<JobView | null> {
+    return this.call((stub) => stub.job(jobId));
+  }
+
+  transitions(jobId?: string): Promise<TransitionView[]> {
+    return this.call((stub) => stub.transitions(jobId, 2_000));
+  }
+
+  deliver(correlationId: string, payload: unknown) {
+    return this.call((stub) => stub.deliverExternal(correlationId, payload));
+  }
+}
+
+// ── Scenarios ─────────────────────────────────────────────────────────────────
+
+export interface Scenario {
+  /** `<kind> × <provider>` (or `<kind>` for a provider-neutral preview kind). */
+  id: string;
+  kind: string;
+  provider: Provider | null;
+  lane: "live" | "preview";
+  setup(t: Tenant): Promise<void>;
+  intent(t: Tenant): { kind: string; input: unknown };
+  /** How many times the intent is delivered (a duplicate delivery coalesces). Default 1. */
+  deliveries?: number;
+}
+
+export const REPO = "shop.example";
+
+const AGGREGATE = { kind: "aggregate", expectedAggregateSha: null, expectedArtifactDigest: null };
+
+export function publishInput(provider: Provider, target: unknown) {
+  return {
+    aggregateId: "preview",
+    repo: REPO,
+    provider,
+    staticPublishTarget: "www-test",
+    publicManifestUrl: "https://shop.test/manifest.json",
+    storefrontMode: "enabled",
+    workerHost: "storefront.test",
+    target,
+  };
+}
+
+const HOTFIX_TARGET = {
+  kind: "hotfix",
+  changeId: "fix-9",
+  prNumber: 9,
+  headSha: "head-fix-9",
+  expectedHeadSha: null,
+  expectedArtifactDigest: null,
+  message: null,
+};
+
+/** A green aggregate at `sha` (with an assets snapshot), staged for the pointer plane on Cloudflare. */
+async function seedRelease(t: Tenant, provider: Provider, sha: string, runId: string): Promise<void> {
+  await t.harness("seedGreen", { sha, runId, prs: [{ changeId: `c-${sha}`, prNumber: 1, headSha: `h-${sha}` }], assets: true });
+  if (provider === "worker-r2-pointer") {
+    // The world's staged table is replaced whole: every sha staged so far stays staged.
+    t.staged.push(sha);
+    await t.harness("configure", { patch: { staged: Object.fromEntries(t.staged.map((s) => [s, { digest: `digest-${s}`, validated: true }])) } });
+  }
+}
+
+/**
+ * A CloudFront tenant's public host already serves an earlier release (a re-dispatch proves it is not
+ * the new one), and GitHub lists a dispatched run within a second — well inside the CI window, as in
+ * production (seconds against 45 minutes).
+ */
+async function cloudFrontServesEarlier(t: Tenant, provider: Provider): Promise<void> {
+  if (provider !== "s3-overwrite") return;
+  await t.harness("serveOnCloudFront", { sha: "sha-0", treeDigest: "tree-0" });
+  await t.harness("configure", { patch: { runListingDelayMs: 1_000 } });
+}
+
+/** Run a prerequisite job unfaulted to `succeeded`. */
+async function prerequisite(t: Tenant, kind: string, input: unknown): Promise<void> {
+  const jobId = await t.submit(kind, input);
+  const { job } = await drive(t, jobId);
+  if (job.state !== "succeeded") throw new Error(`setup ${kind} ended ${job.state}: ${JSON.stringify(job.error)}`);
+}
+
+async function goLive(t: Tenant, provider: Provider, sha: string, runId: string): Promise<void> {
+  await seedRelease(t, provider, sha, runId);
+  await prerequisite(t, "publish", publishInput(provider, AGGREGATE));
+}
+
+const integrateInput = (changeId: string, pr: number) => ({ repo: REPO, changeId, prNumber: pr, headSha: `head-${changeId}` });
+
+async function integratedPr2(t: Tenant): Promise<void> {
+  await t.call((stub) => stub.configureFake({ open: [{ changeId: "pr-2", headSha: "head-pr-2" }] }));
+  await prerequisite(t, "integrate", integrateInput("pr-2", 2));
+}
+
+export const PREVIEW_SCENARIOS: Scenario[] = [
+  {
+    id: "integrate",
+    kind: "integrate",
+    provider: null,
+    lane: "preview",
+    // Two other open candidates: one rebases clean, one conflicts and emails its author.
+    setup: (t) =>
+      t.call((stub) =>
+        stub.configureFake({
+          open: [{ changeId: "pr-2", headSha: "head-pr-2" }],
+          rebaseCandidates: [
+            { changeId: "pr-5", pr: 5, conflicts: true },
+            { changeId: "pr-6", pr: 6 },
+          ],
+        } as never),
+      ),
+    intent: () => ({ kind: "integrate", input: integrateInput("pr-2", 2) }),
+  },
+  { id: "rebuild", kind: "rebuild", provider: null, lane: "preview", setup: integratedPr2, intent: () => ({ kind: "rebuild", input: { repo: REPO } }) },
+  {
+    id: "revert",
+    kind: "revert",
+    provider: null,
+    lane: "preview",
+    setup: integratedPr2,
+    intent: () => ({ kind: "revert", input: { repo: REPO, target: { prNumber: 2 } } }),
+  },
+  {
+    id: "sync-preview",
+    kind: "sync-preview",
+    provider: null,
+    lane: "preview",
+    setup: async () => {},
+    intent: () => ({ kind: "sync-preview", input: { repo: REPO, sha: "c-1", source: "push" } }),
+    // The same push forwarded twice (two forwarders, or a redelivery): one job.
+    deliveries: 2,
+  },
+];
+
+function liveScenarios(provider: Provider): Scenario[] {
+  const name = PROVIDER_NAME[provider];
+  return [
+    {
+      id: `publish × ${name}`,
+      kind: "publish",
+      provider,
+      lane: "live",
+      setup: async (t) => {
+        await cloudFrontServesEarlier(t, provider);
+        await seedRelease(t, provider, "sha-1", "run-1");
+      },
+      intent: () => ({ kind: "publish", input: publishInput(provider, AGGREGATE) }),
+    },
+    {
+      id: `rollback × ${name}`,
+      kind: "rollback",
+      provider,
+      lane: "live",
+      setup: async (t) => {
+        await cloudFrontServesEarlier(t, provider);
+        await goLive(t, provider, "sha-1", "run-1");
+        await goLive(t, provider, "sha-2", "run-2");
+      },
+      intent: (t) => ({ kind: "rollback", input: publishInput(provider, { kind: "release", receiptId: `rcpt:${t.id}:preview:run-1`, versionId: null }) }),
+    },
+    {
+      id: `hotfix × ${name}`,
+      kind: "hotfix",
+      provider,
+      lane: "live",
+      setup: async (t) => {
+        await cloudFrontServesEarlier(t, provider);
+        await t.harness("configure", { patch: { hotfixMainSha: "main-9", hotfixAssets: true } });
+      },
+      intent: () => ({ kind: "hotfix", input: publishInput(provider, HOTFIX_TARGET) }),
+    },
+  ];
+}
+
+export const LIVE_SCENARIOS: Scenario[] = [...liveScenarios("worker-r2-pointer"), ...liveScenarios("s3-overwrite")];
+
+const SEED_PREVIEW = "a".repeat(40);
+const SEED_LIVE = "b".repeat(40);
+
+/**
+ * `seed-from-legacy`: a store the legacy records left serving a preview tip and a live release,
+ * with a run and a publication that never finished. The seed records the pipeline's first facts
+ * for the provider the store serves on, and closes both records out.
+ */
+function seedScenario(provider: Provider): Scenario {
+  return {
+    id: `seed-from-legacy × ${PROVIDER_NAME[provider]}`,
+    kind: "seed-from-legacy",
+    provider,
+    lane: "live",
+    setup: async (t) => {
+      await t.harness("seedPointers", { preview: SEED_PREVIEW, live: SEED_LIVE });
+      await t.harness("seedLegacy", { runId: "run-stale", sha: SEED_PREVIEW, opSubject: "op-stale" });
+    },
+    intent: (t) => ({
+      kind: "seed-from-legacy",
+      input: {
+        tenantId: t.id,
+        preview: {
+          sha: SEED_PREVIEW,
+          membership: [{ pr: 2, changeId: "pr-2", headSha: "head-pr-2" }],
+          evidence: { digest: `digest-${SEED_PREVIEW}`, promotable: true },
+        },
+        live: { provider, sha: SEED_LIVE, receiptId: "rcpt-seeded", provedAt: null },
+        abandon: { runs: ["run-stale"], ops: [{ operation: "publication", subjectId: "op-stale" }] },
+        expect: { previewPointer: SEED_PREVIEW, livePointer: SEED_LIVE },
+      },
+    }),
+  };
+}
+
+export const SEED_SCENARIOS: Scenario[] = [seedScenario("worker-r2-pointer"), seedScenario("s3-overwrite")];
+
+export const SCENARIOS: Scenario[] = [...PREVIEW_SCENARIOS, ...LIVE_SCENARIOS, ...SEED_SCENARIOS];
+
+// ── The driver ────────────────────────────────────────────────────────────────
+
+/** How the CI answers a publish/rollback dispatch (a hotfix's static prebuild is always staged and reported). */
+export type CallbackMode = "deliver" | "never" | "twice" | "superseded" | "cancel-first";
+
+export interface DriveNotes {
+  /** Every answer to `deliverExternal`, in order, with the id it named. */
+  deliveries: { correlationId: string; ok: boolean; duplicate: boolean | null; jobId: string | null }[];
+  /** `superseded`: the job's state right after the stale callback was refused. */
+  stateAfterStale: string | null;
+  /** `cancel-first`: the cancelled dispatch, and the answer to its late callback. */
+  cancelled: string | null;
+  lateCallback: { ok: boolean } | null;
+}
+
+export const success = (sha: string, digest: string | null) => ({
+  status: "succeeded",
+  deliveredSourceSha: sha,
+  deliveredTreeDigest: digest,
+  runUrl: null,
+  error: null,
+  findings: [],
+});
+
+/**
+ * Drive `jobId` to a terminal state, answering every external wait as the world would. Returns the
+ * terminal job. `callbacks` shapes the answer to the FIRST publish/rollback CI dispatch only;
+ * `handled` names waits the caller already answered.
+ */
+export async function drive(
+  t: Tenant,
+  jobId: string,
+  opts: { callbacks?: CallbackMode; timeoutMs?: number; handled?: string[] } = {},
+): Promise<{ job: JobView; notes: DriveNotes }> {
+  const mode = opts.callbacks ?? "deliver";
+  const notes: DriveNotes = { deliveries: [], stateAfterStale: null, cancelled: null, lateCallback: null };
+  /** Waits already answered (by the caller, or earlier in this drive). */
+  const handled = new Set<string>(opts.handled ?? []);
+  let shaped = false;
+  const deadline = Date.now() + (opts.timeoutMs ?? 45_000);
+  const deliver = async (id: string, payload: unknown) => {
+    const r = await t.deliver(id, payload);
+    notes.deliveries.push({ correlationId: id, ok: r.ok, duplicate: r.ok ? r.duplicate : null, jobId: r.ok ? r.jobId : null });
+    return r;
+  };
+  for (;;) {
+    const job = await t.job(jobId);
+    if (!job) throw new Error(`job ${jobId} vanished`);
+    if (TERMINAL.has(job.state)) return { job, notes };
+    if (Date.now() > deadline) throw new Error(`job ${jobId} (${job.kind}) not terminal: ${JSON.stringify([job.step, job.state, job.attempt])}`);
+    const waitingOn = job.state === "awaiting-external" ? job.awaiting : null;
+    if (waitingOn && !handled.has(waitingOn)) {
+      const dispatch = (await t.harness<CiDispatch[]>("dispatches")).find((d) => d.dispatchId === waitingOn);
+      if (dispatch) {
+        handled.add(waitingOn);
+        if (dispatch.workflow === "www-candidate-prebuild") {
+          const digest = `digest-${dispatch.versionId}`;
+          await t.harness("completeStage", { dispatchId: waitingOn, conclusion: "success", digest });
+          await deliver(waitingOn, success(dispatch.versionId, digest));
+        } else {
+          const tree = `tree-${dispatch.versionId}`;
+          const shape = shaped ? "deliver" : mode;
+          shaped = true;
+          if (shape === "cancel-first") {
+            notes.cancelled = waitingOn;
+            await t.harness("completeRun", { dispatchId: waitingOn, conclusion: "cancelled" });
+          } else {
+            if (notes.cancelled) {
+              // The cancelled run's callback arrives late, after the job re-dispatched: refused.
+              const late = await t.deliver(notes.cancelled, success(dispatch.versionId, tree));
+              notes.lateCallback = { ok: late.ok };
+            }
+            await t.harness("completeRun", { dispatchId: waitingOn, conclusion: "success", served: { treeDigest: tree } });
+            if (shape === "superseded") {
+              await deliver(`dsp:superseded:${waitingOn}`, success(dispatch.versionId, tree));
+              notes.stateAfterStale = (await t.job(jobId))?.state ?? null;
+            }
+            if (shape !== "never") await deliver(waitingOn, success(dispatch.versionId, tree));
+            if (shape === "twice") await deliver(waitingOn, success(dispatch.versionId, tree));
+          }
+        }
+      }
+    }
+    await sleep(30);
+  }
+}
+
+// ── A cell ────────────────────────────────────────────────────────────────────
+
+export interface EndState {
+  live: { versionId: string | null; treeDigest: string | null };
+  liveRecords: { provider: string; sha: string; receiptId: string }[];
+  truth: {
+    liveReceiptId: string | null;
+    pendingReceiptId: string | null;
+    deliveryState: string | null;
+    unprovenLive: unknown;
+    liveDisagrees: unknown;
+  } | null;
+  receipts: { receiptId: string; sha: string }[];
+  promotion: { versionId: string; digest: string } | null;
+  cloudfront: { sha: string; treeDigest: string | null } | null;
+  assetsLive: string | null;
+  /** The request-scoped records a seed closes out. */
+  legacy: { runs: { runId: string; state: string }[]; op: string | null };
+  preview: Record<string, unknown>;
+}
+
+export interface Health {
+  alarm: number | null;
+  active: { jobId: string; state: string }[];
+  compensations: { jobId: string; exhausted: boolean; reported: boolean }[];
+}
+
+export interface CellRun {
+  tenant: Tenant;
+  jobId: string;
+  jobIds: string[];
+  job: JobView;
+  notes: DriveNotes;
+  log: TransitionView[];
+  all: TransitionView[];
+  kills: MatrixKill[];
+  effects: MatrixEffect[];
+  execs: { step: string; n: number }[];
+  dispatches: CiDispatch[];
+  end: EndState;
+  health: Health;
+}
+
+export interface CellSpec {
+  scenario: Scenario;
+  fault?: MatrixFault;
+  callbacks?: CallbackMode;
+  /** Extra arrangement after the scenario's setup, before the intent (e.g. a lost dispatch response). */
+  arrange?: (t: Tenant) => Promise<void>;
+}
+
+function slug(text: string): string {
+  return text.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase();
+}
+
+/** The tenant id never survives into a comparison: every cell has its own. */
+export function normalize<T>(value: T, tenantId: string): T {
+  return JSON.parse(JSON.stringify(value).split(tenantId).join("<tenant>")) as T;
+}
+
+/** Wait until nothing is active and every compensation has settled (or the budget runs out). */
+export async function settle(t: Tenant, timeoutMs = 20_000): Promise<Health> {
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    const health = await t.matrix<Health>("health");
+    const unsettled = health.compensations.filter((c) => !(c.exhausted && c.reported));
+    if ((health.active.length === 0 && unsettled.length === 0) || Date.now() > deadline) return health;
+    await sleep(50);
+  }
+}
+
+export async function collect(t: Tenant, jobId: string, jobIds: string[], job: JobView, notes: DriveNotes): Promise<CellRun> {
+  const health = await settle(t);
+  const [log, all, kills, effects, execs, dispatches, end, final] = await Promise.all([
+    t.transitions(jobId),
+    t.transitions(),
+    t.matrix<MatrixKill[]>("kills"),
+    t.matrix<MatrixEffect[]>("effects", { jobId }),
+    t.matrix<{ step: string; n: number }[]>("execs", { jobId }),
+    t.harness<CiDispatch[]>("dispatches"),
+    t.matrix<EndState>("endState", { tenantId: t.id }),
+    t.job(jobId),
+  ]);
+  return {
+    tenant: t,
+    jobId,
+    jobIds,
+    job: final ?? job,
+    notes,
+    log,
+    all,
+    kills,
+    effects,
+    execs,
+    dispatches,
+    end: normalize(end, t.id),
+    health,
+  };
+}
+
+export async function runCell(spec: CellSpec): Promise<CellRun> {
+  const t = new Tenant(uniqueTenant(slug(spec.scenario.id)));
+  await spec.scenario.setup(t);
+  await spec.arrange?.(t);
+  if (spec.fault) await t.matrix("arm", spec.fault as unknown as Record<string, unknown>);
+  const intent = spec.scenario.intent(t);
+  const jobIds: string[] = [];
+  for (let i = 0; i < (spec.scenario.deliveries ?? 1); i += 1) jobIds.push(await t.submit(intent.kind, intent.input));
+  const jobId = jobIds[0]!;
+  const { job, notes } = await drive(t, jobId, spec.callbacks ? { callbacks: spec.callbacks } : {});
+  return collect(t, jobId, jobIds, job, notes);
+}
+
+const baselines = new Map<string, Promise<CellRun>>();
+
+/** The scenario's unfaulted run, once per suite. */
+export function baselineOf(scenario: Scenario): Promise<CellRun> {
+  let run = baselines.get(scenario.id);
+  if (!run) {
+    run = runCell({ scenario });
+    baselines.set(scenario.id, run);
+  }
+  return run;
+}
+
+// ── Convergence ───────────────────────────────────────────────────────────────
+
+/**
+ * External effects that must happen at most as often as in the unfaulted run (never twice for one
+ * cause). The others are keyed or idempotent in production, so a resumed step repeating them is
+ * correct: `forge:prepare-undo` opens the undo candidate under a fixed change id (`undoChangeId`),
+ * `forge:forward-integrate` merges current main into preview ("already up to date" the second time),
+ * `notice:went-live` is deduplicated per post (`socialPostedKey`), and pointer, assets, report,
+ * designation and build writes put the same value again.
+ */
+export const ONCE_EFFECTS: ReadonlySet<string> = new Set([
+  "forge:merge",
+  "forge:rebase",
+  "email:conflict",
+  "ci:stage-aggregate",
+  "ci:www-publish",
+  "ci:www-rollback",
+  "ci:www-promote",
+  "ci:www-candidate-prebuild",
+  "forge:merge-to-main",
+]);
+
+const OUTCOME_NOTES = new Set(["step-done", "checkpoint", "lease-expired", "yielded"]);
+
+export function stepStarts(log: TransitionView[]): string[] {
+  return [...new Set(log.filter((t) => t.note?.startsWith("step-start")).map((t) => t.step))];
+}
+
+export function countBy<T>(items: T[], key: (item: T) => string): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const item of items) out[key(item)] = (out[key(item)] ?? 0) + 1;
+  return out;
+}
+
+/** The job's log is one complete chain: created, started, each step start resolved, one terminal state at the end. */
+export function chainViolations(log: TransitionView[]): string[] {
+  const v: string[] = [];
+  if (log.length < 3) return [`transition log has ${log.length} entries`];
+  if (log[0]!.from !== "none" || log[0]!.to !== "queued") v.push(`log starts ${log[0]!.from}→${log[0]!.to}`);
+  for (let i = 1; i < log.length; i += 1) {
+    if (log[i]!.from !== log[i - 1]!.to) v.push(`transition ${i} (${log[i]!.note}) does not continue the chain`);
+  }
+  const terminals = log.filter((t) => TERMINAL.has(t.to) && !TERMINAL.has(t.from));
+  if (terminals.length !== 1) v.push(`${terminals.length} terminal transitions`);
+  for (let i = 0; i < log.length; i += 1) {
+    const t = log[i]!;
+    if (!t.note?.startsWith("step-start")) continue;
+    const next = log[i + 1];
+    const resolved =
+      next &&
+      ((next.step === t.step && next.attempt === t.attempt && (OUTCOME_NOTES.has(next.note ?? "") || next.note?.startsWith("step-failed"))) ||
+        next.note === "step-start; previous execution did not finish" ||
+        next.to === "awaiting-external" ||
+        TERMINAL.has(next.to));
+    if (!resolved) v.push(`step-start ${t.step}#${t.attempt} has no recorded outcome`);
+  }
+  return v;
+}
+
+/** Across every job of the tenant: between a step's start and its outcome, no other step starts. */
+export function overlapViolations(all: TransitionView[]): string[] {
+  let open: TransitionView | null = null;
+  for (const t of all) {
+    if (t.note?.startsWith("step-start")) {
+      if (open && open.jobId !== t.jobId) return [`step ${t.step} of ${t.jobId} started while ${open.step} of ${open.jobId} was running`];
+      open = t;
+    } else if (
+      open &&
+      t.jobId === open.jobId &&
+      (OUTCOME_NOTES.has(t.note ?? "") || t.note?.startsWith("step-failed") || t.to === "awaiting-external" || TERMINAL.has(t.to))
+    ) {
+      open = null;
+    }
+  }
+  return [];
+}
+
+function diffKeys(a: Record<string, unknown>, b: Record<string, unknown>, keys: string[]): string[] {
+  return keys.filter((k) => JSON.stringify(a[k]) !== JSON.stringify(b[k]));
+}
+
+export const LIVE_END_KEYS = ["live", "liveRecords", "truth", "receipts", "promotion", "cloudfront", "assetsLive"] as const;
+/** Every end-state key a cell is compared on. */
+const END_KEYS = [...LIVE_END_KEYS, "legacy", "preview"] as const;
+
+export interface ConvergenceOptions {
+  /** The fault was armed and must have fired. */
+  expectKill: boolean;
+  /** End-state keys this cell legitimately changes (with why, in the caller). */
+  ignoreEnd?: string[];
+  /** Effect counts this cell legitimately adds (a cancelled run's re-dispatch). */
+  extraEffects?: Record<string, number>;
+}
+
+/**
+ * Everything the convergence guarantee says about one cell, as findings named
+ * `<assertion>: <detail>` (empty when it holds):
+ *   verdict       the job reaches the unfaulted run's terminal verdict
+ *   orphan        nothing is left non-terminal (or a compensation unsettled) without an alarm — at the
+ *                 kill instant and at the end
+ *   live==record  `live` and the engine's live record agree (or the ledger surfaces unprovenLive /
+ *                 liveDisagrees) at the end; at the kill instant they may differ only while the job's
+ *                 compensation is armed to put `live` back
+ *   duplicate     no once-only external effect (merge, dispatch, email, promote, notice) happens more
+ *                 often than unfaulted
+ *   compensation  a pending compensation settles
+ *   coalesced     a duplicate delivery of one intent is the same job
+ *   log           the job's transition log is one complete chain and finishes the unfaulted steps, in order
+ *   end-state     what the job leaves behind equals the unfaulted run's
+ */
+export function convergence(cell: CellRun, base: CellRun, opts: ConvergenceOptions): string[] {
+  const v: string[] = [];
+  const { job } = cell;
+  if (job.state !== base.job.state || job.error?.code !== base.job.error?.code) {
+    v.push(`verdict: ${job.state}${job.error ? ` (${job.error.code}: ${job.error.reason})` : ""}, unfaulted ${base.job.state}`);
+  }
+  if (opts.expectKill && cell.kills.length === 0) v.push("fired: the armed fault never fired");
+
+  for (const kill of cell.kills) {
+    const where = `${kill.step}/${kill.mode}${kill.effect ? `:${kill.effect}` : ""}`;
+    if (kill.alarm === null && (kill.activeJobs > 0 || kill.pendingCompensations > 0)) {
+      v.push(`orphan: killed at ${where} with ${kill.activeJobs} active job(s) and no alarm to resume them`);
+    }
+    // `live` may run ahead of the record only while something will reconcile them: the job's armed
+    // compensation (a Cloudflare flip before its proof), or — CloudFront — `live` naming what the
+    // public host is already proven to serve, which the resumed job records.
+    if (kill.live !== kill.record && !kill.armed && !(kill.served !== null && kill.live === kill.served)) {
+      v.push(`live==record: killed at ${where} with live=${kill.live} record=${kill.record} and no compensation armed`);
+    }
+  }
+  for (const a of cell.health.active) v.push(`orphan: job ${a.jobId === cell.jobId ? "under test" : a.jobId} still ${a.state}`);
+  for (const c of cell.health.compensations.filter((c) => !(c.exhausted && c.reported))) {
+    v.push(`compensation: ${c.jobId === cell.jobId ? "the job's" : c.jobId} compensation never settled${cell.health.alarm === null ? " (and no alarm)" : ""}`);
+  }
+
+  if (new Set(cell.jobIds).size !== 1) v.push(`coalesced: ${cell.jobIds.length} deliveries made ${new Set(cell.jobIds).size} jobs`);
+
+  const end = cell.end;
+  const input = job.input as { provider?: string | null };
+  if (job.lane === "live" && "provider" in input && job.state === "succeeded") {
+    const provider = input.provider ?? "worker-r2-pointer";
+    const record = end.liveRecords.find((r) => r.provider === provider)?.sha ?? null;
+    const receiptSha = end.receipts.find((r) => r.receiptId === end.truth?.liveReceiptId)?.sha ?? null;
+    const surfaced = end.truth?.unprovenLive != null || end.truth?.liveDisagrees != null;
+    if (!surfaced && (end.live.versionId !== record || end.live.versionId !== receiptSha)) {
+      v.push(`live==record: live=${end.live.versionId} record=${record} ledger=${receiptSha}, nothing surfaced`);
+    }
+    if (provider === "s3-overwrite" && !surfaced && end.cloudfront?.sha !== end.live.versionId) {
+      v.push(`live==record: CloudFront serves ${end.cloudfront?.sha ?? "nothing"} while live names ${end.live.versionId}`);
+    }
+  }
+
+  const cellOnce = countBy(cell.effects.filter((e) => ONCE_EFFECTS.has(e.effect)), (e) => e.effect);
+  const baseOnce = countBy(base.effects.filter((e) => ONCE_EFFECTS.has(e.effect)), (e) => e.effect);
+  for (const [effect, n] of Object.entries(cellOnce)) {
+    const allowed = (baseOnce[effect] ?? 0) + (opts.extraEffects?.[effect] ?? 0);
+    if (n > allowed) {
+      const steps = [...new Set(cell.effects.filter((e) => e.effect === effect).map((e) => e.step))].join(",");
+      v.push(`duplicate: ${effect} ${n}× (unfaulted ${baseOnce[effect] ?? 0}×) at ${steps}`);
+    }
+  }
+
+  for (const finding of chainViolations(cell.log)) v.push(`log: ${finding}`);
+  const done = cell.log.filter((t) => t.note === "step-done").map((t) => t.step);
+  const baseDone = base.log.filter((t) => t.note === "step-done").map((t) => t.step);
+  if (JSON.stringify(done) !== JSON.stringify(baseDone)) v.push(`log: finished [${done.join(",")}], unfaulted [${baseDone.join(",")}]`);
+
+  const keys = END_KEYS.filter((k) => !opts.ignoreEnd?.includes(k));
+  for (const key of diffKeys(end as unknown as Record<string, unknown>, base.end as unknown as Record<string, unknown>, keys)) {
+    v.push(`end-state: ${key} = ${JSON.stringify((end as unknown as Record<string, unknown>)[key])}, unfaulted ${JSON.stringify((base.end as unknown as Record<string, unknown>)[key])}`);
+  }
+  return v;
+}

package/apps/storefront/vitest.faults.config.cts

@@ -0,0 +1,13 @@
+// The systematic fault matrix (test/pipeline-faults/), inside workerd — see
+// test/pipeline/pool-config.cjs. Every cell runs against its own tenant's object, so cells run
+// concurrently; the suite's wall-time budget is two minutes (CI job `pipeline-faults`).
+process.env.VITE_CJS_IGNORE_WARNING = "1";
+const { pipelinePoolConfig } = require("./test/pipeline/pool-config.cjs");
+
+module.exports = pipelinePoolConfig({
+  root: __dirname,
+  name: "pipeline-faults",
+  include: ["test/pipeline-faults/**/*.test.ts"],
+  maxConcurrency: 24,
+  testTimeout: 90_000,
+});