@tokenoftrust/storefront-runner 2.4.6 → 2.4.8

package/apps/storefront/test/pipeline/pipeline-host.ts

@@ -4,21 +4,30 @@
  * restarts the object without touching its callers). It runs the production engine with the
  * synthetic job kinds, the production preview kinds (integrate / rebuild / revert / sync-preview) over
  * fake services (`./fake-preview-services.ts`) and the real publish kinds over the stand-in world
- * (`./publish-kinds.ts`), and answers intents through `submitIntent`.
+ * (`./publish-kinds.ts`), and answers intents through `submitIntent`. Every kind is wrapped with the
+ * fault matrix's kill points (`./matrix-faults.ts`), inert until a test arms one.
  */
 import type { DurableObjectState } from "cloudflare:workers";
 import { TenantPipeline as DeployedTenantPipeline } from "../../src/lib/pipeline/tenant-pipeline";
 import { submitIntent, tenantPipeline, type TenantPipelineNamespace } from "../../src/lib/pipeline/intake";
 import { submitPreviewSync } from "../../src/lib/pipeline/preview-sync-intake";
-import { StaleExecutionError, type JobKindRegistry, type PipelineIntent } from "../../src/lib/pipeline/job-runner";
+import { JobKindRegistry, StaleExecutionError, type PipelineIntent } from "../../src/lib/pipeline/job-runner";
 import { abandonedContexts, syntheticJobKinds, syntheticSpans } from "./synthetic-job";
 import { registerPreviewJobKinds, type PreviewJobTiming } from "../../src/lib/pipeline/preview-jobs";
 import { configureFake, fakePreviewServices, readFakeState, type FakeFault, type FakeState } from "./fake-preview-services";
+import { MatrixFaults, previewServicesWithEffects, type Landed, type MatrixFault } from "./matrix-faults";
+import { kvAbandonServices, legacyOperationKey, registerSeedJobKind, type SeedServices } from "../../src/lib/pipeline/seed-from-legacy";
+import { readAssetsPointer } from "@tot/public-runtime";
 import { readLivePointer } from "../../src/lib/pipeline/publish/live-pointer";
-import { listReleaseReceipts, readReleaseTruth } from "../../src/lib/change/store/aggregate-release-store";
+import {
+  listIntegrationRuns,
+  listReleaseReceipts,
+  readReleaseTruth,
+  writeIntegrationRun,
+  type ReleaseModelKv,
+} from "../../src/lib/change/store/aggregate-release-store";
 import { projectionStore, seedGreenAggregate, testVersionStore, type TestKv } from "../../src/lib/pipeline/publish/publish-test-support";
 import {
-  armedAtFaults,
   armPublishFault,
   firedPublishFaults,
   publishWorldOf,
@@ -27,6 +36,9 @@ import {
 } from "./publish-kinds";
 
 /** The preview kinds' timing in the harness: the production shape, scaled to milliseconds. */
+/** The seed kind's lease and retries in the harness. */
+const HARNESS_SEED_TIMING = { leaseMs: 1_000, deadlineMs: 30_000, retryBaseMs: 10 };
+
 export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
   leaseMs: 1_000,
   deadlineMs: 30_000,
@@ -35,13 +47,133 @@ export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
   defaultExpectedMs: 2_000,
 };
 
+/**
+ * The seed's services in the harness: what is served, read from this object's pointer authority (as
+ * the deployed object reads it), and the close-out of legacy records over the harness KV — each
+ * close-out reported to the fault matrix.
+ */
+function harnessSeedServices(
+  ctx: DurableObjectState,
+  pointers: { read(tenantId: string): Promise<{ channels?: { preview?: string; live?: string } }> },
+  landed: Landed,
+): SeedServices {
+  const abandon = kvAbandonServices(publishWorldOf(ctx).kv as unknown as ReleaseModelKv);
+  return {
+    async pointers(tenantId) {
+      const pointer = await pointers.read(tenantId);
+      return { preview: pointer.channels?.preview ?? null, live: pointer.channels?.live ?? null };
+    },
+    async abandonRun(tenantId, runId, at) {
+      const changed = await abandon.abandonRun(tenantId, runId, at);
+      await landed("kv:abandon-run");
+      return changed;
+    },
+    async abandonOp(tenantId, op, at) {
+      const changed = await abandon.abandonOp(tenantId, op, at);
+      await landed("kv:abandon-op");
+      return changed;
+    },
+  };
+}
+
+/** Per object: the fault matrix's injection points and the (wrapped) registry it runs. */
+const matrices = new WeakMap<DurableObjectState, { faults: MatrixFaults; registry: JobKindRegistry }>();
+
 export class TenantPipeline extends DeployedTenantPipeline {
   protected override jobKinds(ctx: DurableObjectState): JobKindRegistry {
-    return registerHarnessPublishKinds(
+    const faults = new MatrixFaults(ctx, () => publishWorldOf(ctx).kv);
+    const services = previewServicesWithEffects(fakePreviewServices(ctx, faults.landed), faults.landed);
+    const registry = registerHarnessPublishKinds(
       ctx,
-      registerPreviewJobKinds(syntheticJobKinds(ctx), fakePreviewServices(ctx), HARNESS_PREVIEW_TIMING),
+      registerPreviewJobKinds(syntheticJobKinds(ctx), services, HARNESS_PREVIEW_TIMING),
       this.pointers,
+      faults.landed,
     );
+    // The production seed kind, its lease and retries scaled to the harness (an evicted seed resumes
+    // when its lease runs out, as in production).
+    const seed = registerSeedJobKind(new JobKindRegistry(), harnessSeedServices(ctx, this.pointers, faults.landed));
+    for (const name of seed.list()) registry.register({ ...seed.get(name)!, ...HARNESS_SEED_TIMING });
+    const wrapped = faults.wrap(registry);
+    matrices.set(ctx, { faults, registry: wrapped });
+    return wrapped;
+  }
+
+  /** Harness-only: arm and read the fault matrix (`./matrix-faults.ts`, `test/pipeline-faults/`). */
+  async matrix(op: string, args: Record<string, unknown> = {}): Promise<unknown> {
+    const { faults, registry } = matrices.get(this.ctx)!;
+    switch (op) {
+      case "arm":
+        faults.arm(args as unknown as MatrixFault);
+        return null;
+      case "held":
+        return faults.held();
+      case "release":
+        faults.release();
+        return null;
+      case "kills":
+        return faults.kills();
+      case "effects":
+        return faults.effects(args.jobId as string | undefined);
+      case "execs":
+        return faults.execs(String(args.jobId));
+      case "health":
+        return faults.health();
+      case "kinds":
+        return registry.list().map((kind) => {
+          const definition = registry.get(kind)!;
+          return { kind, lane: definition.lane ?? "live", steps: definition.steps.map((s) => s.name) };
+        });
+      case "endState":
+        return this.endState(String(args.tenantId));
+      default:
+        throw new Error(`unknown matrix op ${op}`);
+    }
+  }
+
+  /** What a converged job leaves behind, on both lanes — the fault matrix compares it to an unfaulted run. */
+  private async endState(tenantId: string): Promise<Record<string, unknown>> {
+    const { kv, world } = publishWorldOf(this.ctx);
+    const truth = await readReleaseTruth(kv, tenantId, "preview");
+    const fake = readFakeState(this.ctx);
+    return {
+      live: await readLivePointer(projectionStore(kv), tenantId),
+      liveRecords: this.ctx.storage.sql
+        .exec<{ provider: string; sha: string; receipt_id: string }>(`SELECT provider, sha, receipt_id FROM live ORDER BY provider`)
+        .toArray()
+        .map((r) => ({ provider: r.provider, sha: r.sha, receiptId: r.receipt_id })),
+      truth: truth
+        ? {
+            liveReceiptId: truth.liveReceiptId ?? null,
+            pendingReceiptId: truth.pendingReceiptId ?? null,
+            deliveryState: truth.deliveryState ?? null,
+            unprovenLive: truth.unprovenLive ?? null,
+            liveDisagrees: truth.liveDisagrees ? { serving: truth.liveDisagrees.serving, recordedReceiptId: truth.liveDisagrees.recordedReceiptId } : null,
+          }
+        : null,
+      receipts: (await listReleaseReceipts(kv, tenantId, "preview")).map((r) => ({ receiptId: r.receiptId, sha: r.aggregateSha })),
+      promotion: await world.promotionStatus(),
+      cloudfront: await world.cloudFrontServes(),
+      assetsLive: (await readAssetsPointer({ get: (k: string) => kv.get(k) }, tenantId)).live ?? null,
+      legacy: {
+        runs: (await listIntegrationRuns(kv as unknown as ReleaseModelKv, tenantId, "preview")).map((r) => ({ runId: r.runId, state: r.state })),
+        op: await (async () => {
+          const raw = await kv.get(legacyOperationKey(tenantId, { operation: "publication", subjectId: "op-stale" }));
+          return raw ? ((JSON.parse(raw) as { problem?: { code?: string } | null }).problem?.code ?? "open") : null;
+        })(),
+      },
+      preview: {
+        pointer: fake.previewPointer,
+        readModel: fake.readModel,
+        merges: fake.merges.map((m) => ({ changeId: m.changeId, sha: m.sha })),
+        commits: this.ctx.storage.sql
+          .exec<{ sha: string; membership: string }>(`SELECT sha, membership FROM commits ORDER BY sha`)
+          .toArray()
+          .map((r) => ({ sha: r.sha, membership: JSON.parse(r.membership) as unknown })),
+        evidence: this.ctx.storage.sql
+          .exec<{ sha: string; digest: string }>(`SELECT sha, digest FROM evidence ORDER BY sha`)
+          .toArray(),
+      },
+    };
   }
 
   /**
@@ -123,13 +255,46 @@ export class TenantPipeline extends DeployedTenantPipeline {
           Number(args.times ?? 1),
         );
         return null;
+      case "seedPointers":
+        // What the shared preview and the live site are served from, as a legacy store left them.
+        await this.pointers.update(tenantId, (current) => ({
+          ...current,
+          channels: { ...current.channels, preview: String(args.preview), live: String(args.live) },
+          channelRevs: { ...current.channelRevs, preview: (current.channelRevs?.preview ?? 0) + 1, live: (current.channelRevs?.live ?? 0) + 1 },
+        }));
+        return null;
+      case "seedLegacy": {
+        // A request-scoped run and an operation that never finished.
+        const at = "2026-10-09T00:00:00.000Z";
+        await writeIntegrationRun(
+          kv as unknown as ReleaseModelKv,
+          {
+            tenantId,
+            aggregateId: "preview",
+            runId: String(args.runId),
+            aggregateSha: String(args.sha),
+            includedPrs: [],
+            state: "running",
+            previousRunId: null,
+            artifactDigest: null,
+            startedAt: at,
+            finishedAt: null,
+            statusMessage: null,
+          },
+          0,
+        );
+        const op = { operation: "publication" as const, subjectId: String(args.opSubject) };
+        await kv.put(
+          legacyOperationKey(tenantId, op),
+          JSON.stringify({ tenantId, ...op, revision: String(args.sha), stage: "publishing", problem: null, attemptsMade: 1, updatedAt: at, rev: 0 }),
+        );
+        return null;
+      }
       case "armFault":
         armPublishFault(this.ctx, args as unknown as PublishFault);
         return null;
       case "firedFaults":
         return firedPublishFaults(this.ctx);
-      case "armedAtFaults":
-        return armedAtFaults(this.ctx);
       default:
         throw new Error(`unknown publish harness op ${op}`);
     }

package/apps/storefront/test/pipeline/pool-config.cjs

@@ -0,0 +1,78 @@
+// The workerd pool both pipeline suites run in (@cloudflare/vitest-pool-workers): the TenantPipeline
+// Durable Object is exercised with real SQLite storage, real alarms and real eviction
+// (`ctx.abort()`). The Node unit suite (vitest.config.cts) never picks these tests up.
+//
+// The object lives in its own Worker, `pipeline-host`, not in the test runner's isolate: an
+// eviction must restart the object without touching its callers, as in production. Auxiliary
+// Workers take plain modules, so the host is bundled here, in memory, with esbuild.
+//
+//   vitest.workers.config.cts  test/pipeline/**         the hand-written harness tests, one at a time
+//   vitest.faults.config.cts   test/pipeline-faults/**  the systematic fault matrix, cells concurrent
+const { defineWorkersConfig } = require("@cloudflare/vitest-pool-workers/config");
+const esbuild = require("esbuild");
+const path = require("node:path");
+
+const compatibilityDate = "2024-11-01";
+const compatibilityFlags = ["nodejs_compat"];
+const HOST = "pipeline-host";
+
+/**
+ * @param {{ root: string; name: string; include: string[]; maxConcurrency?: number; testTimeout?: number }} options
+ */
+function pipelinePoolConfig({ root, name, include, maxConcurrency, testTimeout = 60_000 }) {
+  const hostModule = esbuild.buildSync({
+    entryPoints: [path.join(root, "test/pipeline/pipeline-host.ts")],
+    bundle: true,
+    format: "esm",
+    platform: "neutral",
+    target: "es2022",
+    external: ["cloudflare:*"],
+    write: false,
+  }).outputFiles[0].text;
+
+  const pipelineBinding = { TENANT_PIPELINE: { className: "TenantPipeline", scriptName: HOST } };
+
+  return defineWorkersConfig({
+    // The source tree's `@/` import alias (tsconfig `paths`), for modules the tests import directly.
+    resolve: { alias: { "@": path.join(root, "src") } },
+    test: {
+      name,
+      include,
+      cache: false,
+      testTimeout,
+      hookTimeout: 60_000,
+      ...(maxConcurrency ? { maxConcurrency } : {}),
+      poolOptions: {
+        workers: {
+          isolatedStorage: false,
+          singleWorker: true,
+          miniflare: {
+            compatibilityDate,
+            compatibilityFlags,
+            durableObjects: pipelineBinding,
+            serviceBindings: { PIPELINE_HOST: HOST, INTAKE_B: "pipeline-intake-b" },
+            workers: [
+              {
+                name: HOST,
+                modules: [{ type: "ESModule", path: "pipeline-host.mjs", contents: hostModule }],
+                compatibilityDate,
+                compatibilityFlags,
+                durableObjects: { TENANT_PIPELINE: { className: "TenantPipeline", useSQLite: true } },
+              },
+              {
+                name: "pipeline-intake-b",
+                modules: true,
+                scriptPath: path.join(root, "test/pipeline/intake-worker.mjs"),
+                compatibilityDate,
+                compatibilityFlags,
+                durableObjects: pipelineBinding,
+              },
+            ],
+          },
+        },
+      },
+    },
+  });
+}
+
+module.exports = { pipelinePoolConfig };

package/apps/storefront/test/pipeline/publish-kinds.ts

@@ -26,18 +26,26 @@ import {
   movePreview,
   standInPublishEnvironment,
   testVersionStore,
+  type CiDispatch,
   type TestKv,
 } from "../../src/lib/pipeline/publish/publish-test-support";
+import type { Landed } from "./matrix-faults";
 
 type SiteVersionStore = ReturnType<typeof testVersionStore>;
 type PointerWriter = NonNullable<Parameters<typeof testVersionStore>[1]>;
+type PointerRecord = Awaited<ReturnType<PointerWriter["read"]>>;
 
+/**
+ * Production's ratios, in seconds: a CI window (45 min) outlasts the lease an evicted object waits
+ * out before it resumes (10 min, here `leaseMs`), so a job resumed after an eviction is still inside
+ * its window — as in production.
+ */
 export const PUBLISH_HARNESS_TIMING: PublishTiming = {
-  awaitWindowMs: 1_500,
+  awaitWindowMs: 4_000,
   runGraceMs: 500,
   proofDelayMs: 30,
   proofProbes: { synchronous: 3, asynchronous: 5 },
-  stageWindowMs: 1_500,
+  stageWindowMs: 4_000,
   stageChecks: 3,
 };
 
@@ -64,25 +72,6 @@ function tables(state: DurableObjectState): void {
   sql.exec(`CREATE TABLE IF NOT EXISTS harness_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
   sql.exec(`CREATE TABLE IF NOT EXISTS harness_publish_faults (id INTEGER PRIMARY KEY AUTOINCREMENT, step TEXT, at TEXT, mode TEXT, remaining INTEGER, version_id TEXT)`);
   sql.exec(`CREATE TABLE IF NOT EXISTS harness_publish_fired (seq INTEGER PRIMARY KEY AUTOINCREMENT, step TEXT, at TEXT, mode TEXT)`);
-  sql.exec(`CREATE TABLE IF NOT EXISTS harness_publish_armed (seq INTEGER PRIMARY KEY AUTOINCREMENT, step TEXT, at TEXT, armed INTEGER)`);
-}
-
-/** At each fired fault: how many compensations the engine held armed for jobs not yet terminal. */
-export function armedAtFaults(state: DurableObjectState): { step: string; at: string; armed: number }[] {
-  tables(state);
-  return state.storage.sql.exec<{ step: string; at: string; armed: number }>(`SELECT step, at, armed FROM harness_publish_armed ORDER BY seq`).toArray();
-}
-
-function armedCompensations(state: DurableObjectState): number {
-  try {
-    return state.storage.sql
-      .exec<{ n: number }>(
-        `SELECT COUNT(*) AS n FROM terminal_hooks h JOIN jobs j ON j.job_id = h.job_id WHERE j.state NOT IN ('succeeded', 'needs-developer', 'stuck')`,
-      )
-      .one().n;
-  } catch {
-    return -1;
-  }
 }
 
 /** A KV over the object's SQLite: durable across evictions, visible to every later execution. */
@@ -133,7 +122,6 @@ async function maybeFault(state: DurableObjectState, step: string, at: "before"
   if (!fault) return;
   sql.exec(`UPDATE harness_publish_faults SET remaining = remaining - 1 WHERE id = ?`, fault.id);
   sql.exec(`INSERT INTO harness_publish_fired (step, at, mode) VALUES (?, ?, ?)`, step, at, fault.mode);
-  sql.exec(`INSERT INTO harness_publish_armed (step, at, armed) VALUES (?, ?, ?)`, step, at, armedCompensations(state));
   // Everything written so far — including a step's own effect, for an `after` fault — is made
   // durable first, exactly as a real crash after the write would leave it.
   await state.storage.sync();
@@ -182,11 +170,104 @@ export function publishWorldOf(state: DurableObjectState): { kv: TestKv; world:
   return { kv, world: new PublishWorld(kv) };
 }
 
-/** Register the publish kinds (fault-wrapped, over the stand-in world) into `registry`; `pointers` is the object's authority. */
-export function registerHarnessPublishKinds(state: DurableObjectState, registry: JobKindRegistry, pointers: PointerWriter): JobKindRegistry {
-  const { kv, world } = publishWorldOf(state);
-  const store = testVersionStore(kv, pointers);
-  const environment = standInPublishEnvironment({ kv, world, store, now: () => Date.now(), timing: PUBLISH_HARNESS_TIMING });
+/** The world's logs that record an external effect, by the effect name the fault matrix uses. */
+const NOTED_EFFECTS: Record<string, string> = {
+  "hotfix-merges": "forge:merge-to-main",
+  "forward-integrations": "forge:forward-integrate",
+  "staged-originals": "r2:stage-originals",
+  "went-live": "notice:went-live",
+};
+
+/** The publish world, reporting every external effect a job makes to the fault matrix. */
+class EffectWorld extends PublishWorld {
+  constructor(
+    kv: TestKv,
+    private readonly landed: Landed,
+  ) {
+    super(kv);
+  }
+
+  override async dispatch(dispatch: CiDispatch, at = 0): Promise<void> {
+    const before = (await this.dispatches()).length;
+    let lost: unknown = null;
+    try {
+      await super.dispatch(dispatch, at);
+    } catch (thrown) {
+      // A refused dispatch recorded nothing (no run exists); a lost response came after the run did.
+      if ((await this.dispatches()).length === before) throw thrown;
+      lost = thrown;
+    }
+    await this.landed(`ci:${dispatch.workflow}`);
+    if (lost) throw lost;
+  }
+
+  override async recordDesignation(entry: { target: string; versionId: string; digest: string }): Promise<void> {
+    await super.recordDesignation(entry);
+    await this.landed("mcp:designate");
+  }
+
+  override async note(key: string, value: unknown): Promise<void> {
+    await super.note(key, value);
+    const effect = NOTED_EFFECTS[key];
+    if (effect) await this.landed(effect);
+  }
+}
+
+/** The object's pointer writer, reporting each write that moved a channel (`pointer:<channel>`). */
+function effectPointers(inner: PointerWriter, landed: Landed): PointerWriter {
+  return {
+    read: (tenantId) => inner.read(tenantId),
+    ...(inner.peek ? { peek: (tenantId: string) => inner.peek!(tenantId) } : {}),
+    async update(tenantId, apply) {
+      let before: PointerRecord | null = null;
+      const after = await inner.update(tenantId, (current) => {
+        before = current;
+        return apply(current);
+      });
+      const prior = before as PointerRecord | null;
+      for (const channel of ["live", "preview"] as const) {
+        if (prior?.channels?.[channel] !== after.channels?.[channel] || prior?.channelRevs?.[channel] !== after.channelRevs?.[channel]) {
+          await landed(`pointer:${channel}`);
+        }
+      }
+      return after;
+    },
+  };
+}
+
+/** The job's KV, reporting each write of the assets pointer (`assets:live`). */
+function effectKv(kv: TestKv, landed: Landed): TestKv {
+  return {
+    get: (key) => kv.get(key),
+    put: async (key, value) => {
+      await kv.put(key, value);
+      if (key.startsWith("cust:assets-pointer:")) await landed("assets:live");
+    },
+    ...(kv.delete ? { delete: (key: string) => kv.delete!(key) } : {}),
+  };
+}
+
+const noEffects: Landed = async () => {};
+
+/**
+ * Register the publish kinds (fault-wrapped, over the stand-in world) into `registry`; `pointers` is
+ * the object's authority. `landed` hears every external effect the jobs make (the fault matrix).
+ */
+export function registerHarnessPublishKinds(
+  state: DurableObjectState,
+  registry: JobKindRegistry,
+  pointers: PointerWriter,
+  landed: Landed = noEffects,
+): JobKindRegistry {
+  const { kv } = publishWorldOf(state);
+  const store = testVersionStore(kv, effectPointers(pointers, landed));
+  const environment = standInPublishEnvironment({
+    kv: effectKv(kv, landed),
+    world: new EffectWorld(kv, landed),
+    store,
+    now: () => Date.now(),
+    timing: PUBLISH_HARNESS_TIMING,
+  });
   const kinds = registerPublishJobKinds(new JobKindRegistry(), async () => environment, PUBLISH_HARNESS_TUNING);
   for (const name of kinds.list()) registry.register(withFaults(state, kinds.get(name)!, store));
   return registry;