@tokenoftrust/storefront-runner 2.4.3 → 2.4.5

package/apps/storefront/src/lib/social/golive.ts

@@ -1,7 +1,7 @@
 /**
  * S1 — the social go-live hook + idempotency.
  *
- * WHERE IT FIRES: at the true go-live edge — after `AggregateShipOrchestrator`
+ * WHERE IT FIRES: at the true go-live edge — after the tenant pipeline's publish job
  * flips a tenant's `live` channel to a freshly-shipped version and records the
  * release ledger. NOT on `POST /api/admin/blog`'s `publish` (which only opens a
  * candidate PR a human still promotes) and NOT on accept-to-preview

package/apps/storefront/test/pipeline/harness.ts

@@ -21,7 +21,19 @@ interface HarnessEnv {
 }
 
 type TenantPipelineStub = {
-  [K in "submit" | "job" | "jobs" | "transitions" | "facts" | "harness" | "probeStaleWrites" | "configureFake" | "fakeState" | "spans"]: TenantPipeline[K];
+  [K in
+    | "submit"
+    | "deliverExternal"
+    | "job"
+    | "jobs"
+    | "transitions"
+    | "facts"
+    | "harness"
+    | "probeStaleWrites"
+    | "configureFake"
+    | "fakeState"
+    | "spans"
+    | "publishHarness"]: TenantPipeline[K];
 };
 
 export const harnessEnv = env as unknown as HarnessEnv;
@@ -89,7 +101,8 @@ const TERMINAL = new Set<JobState>(["succeeded", "needs-developer", "stuck"]);
 
 /**
  * Polls until every job is terminal. On every sample asserts the tenant never has more than one
- * running job. Create stubs for later reads after this returns (see the eviction note below).
+ * active (running or awaiting) job per lane. Create stubs for later reads after this returns (see
+ * the eviction note below).
  */
 export async function waitTerminal(tenant: string, jobIds: string[], timeoutMs = 20_000): Promise<JobView[]> {
   const deadline = Date.now() + timeoutMs;
@@ -105,9 +118,11 @@ export async function waitTerminal(tenant: string, jobIds: string[], timeoutMs =
       await new Promise((resolve) => setTimeout(resolve, 25));
       continue;
     }
-    const running = jobs.filter((j) => j.state === "running").length;
-    maxRunning = Math.max(maxRunning, running);
-    expect(running, "running jobs observed at once").toBeLessThanOrEqual(1);
+    const active = jobs.filter((j) => j.state === "running" || j.state === "awaiting-external");
+    maxRunning = Math.max(maxRunning, active.length);
+    for (const lane of ["live", "preview"]) {
+      expect(active.filter((j) => j.lane === lane).length, `active ${lane}-lane jobs observed at once`).toBeLessThanOrEqual(1);
+    }
     const wanted = jobIds.map((id) => jobs.find((j) => j.jobId === id));
     if (wanted.every((j) => j && TERMINAL.has(j.state))) return wanted as JobView[];
     if (Date.now() > deadline) {

package/apps/storefront/test/pipeline/pipeline-host.ts

@@ -2,8 +2,9 @@
  * The Worker that hosts the TenantPipeline Durable Object for the harness, in its own isolate
  * (as in production, where callers never share the object's isolate — and where an eviction
  * restarts the object without touching its callers). It runs the production engine with the
- * synthetic job kinds AND the production preview kinds (integrate / rebuild / revert / sync-preview) over fake
- * services (`./fake-preview-services.ts`), and answers intents through `submitIntent`.
+ * synthetic job kinds, the production preview kinds (integrate / rebuild / revert / sync-preview) over
+ * fake services (`./fake-preview-services.ts`) and the real publish kinds over the stand-in world
+ * (`./publish-kinds.ts`), and answers intents through `submitIntent`.
  */
 import type { DurableObjectState } from "cloudflare:workers";
 import { TenantPipeline as DeployedTenantPipeline } from "../../src/lib/pipeline/tenant-pipeline";
@@ -13,6 +14,17 @@ import { StaleExecutionError, type JobKindRegistry, type PipelineIntent } from "
 import { abandonedContexts, syntheticJobKinds, syntheticSpans } from "./synthetic-job";
 import { registerPreviewJobKinds, type PreviewJobTiming } from "../../src/lib/pipeline/preview-jobs";
 import { configureFake, fakePreviewServices, readFakeState, type FakeFault, type FakeState } from "./fake-preview-services";
+import { readLivePointer } from "../../src/lib/pipeline/publish/live-pointer";
+import { listReleaseReceipts, readReleaseTruth } from "../../src/lib/change/store/aggregate-release-store";
+import { seedGreenAggregate, testVersionStore } from "../../src/lib/pipeline/publish/publish-test-support";
+import {
+  armedAtFaults,
+  armPublishFault,
+  firedPublishFaults,
+  publishWorldOf,
+  registerHarnessPublishKinds,
+  type PublishFault,
+} from "./publish-kinds";
 
 /** The preview kinds' timing in the harness: the production shape, scaled to milliseconds. */
 export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
@@ -25,7 +37,10 @@ export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
 
 export class TenantPipeline extends DeployedTenantPipeline {
   protected override jobKinds(ctx: DurableObjectState): JobKindRegistry {
-    return registerPreviewJobKinds(syntheticJobKinds(ctx), fakePreviewServices(ctx), HARNESS_PREVIEW_TIMING);
+    return registerHarnessPublishKinds(
+      ctx,
+      registerPreviewJobKinds(syntheticJobKinds(ctx), fakePreviewServices(ctx), HARNESS_PREVIEW_TIMING),
+    );
   }
 
   /** Harness-only: open candidates and arm faults for the fake preview services. */
@@ -47,6 +62,45 @@ export class TenantPipeline extends DeployedTenantPipeline {
     return readFakeState(this.ctx);
   }
 
+  /** Harness-only: arrange and read the publish stand-in world inside this object's storage. */
+  async publishHarness(op: string, args: Record<string, unknown> = {}): Promise<unknown> {
+    const { kv, world } = publishWorldOf(this.ctx);
+    const store = testVersionStore(kv);
+    const tenantId = String(args.tenantId ?? "");
+    switch (op) {
+      case "configure":
+        return world.configure(args.patch as never);
+      case "seedGreen":
+        await seedGreenAggregate(kv, store, args as never);
+        return null;
+      case "completeRun":
+        return world.completeRun(String(args.dispatchId), args.conclusion as never, args.served as never);
+      case "serveOnCloudFront":
+        return world.serveOnCloudFront(String(args.sha), (args.treeDigest as string | null) ?? null);
+      case "completeStage":
+        return world.completeStage(String(args.dispatchId), args.conclusion as never, args.digest as never);
+      case "dispatches":
+        return world.dispatches();
+      case "promotionStatus":
+        return world.promotionStatus();
+      case "live":
+        return readLivePointer(store, tenantId);
+      case "receipts":
+        return (await listReleaseReceipts(kv, tenantId, "preview")).map((r) => r.receiptId);
+      case "truth":
+        return readReleaseTruth(kv, tenantId, "preview");
+      case "armFault":
+        armPublishFault(this.ctx, args as unknown as PublishFault);
+        return null;
+      case "firedFaults":
+        return firedPublishFaults(this.ctx);
+      case "armedAtFaults":
+        return armedAtFaults(this.ctx);
+      default:
+        throw new Error(`unknown publish harness op ${op}`);
+    }
+  }
+
   /** Harness-only reads of the synthetic step log. */
   async harness(jobId: string): Promise<{
     runs: { step: number; chunk: number; attempt: number }[];

package/apps/storefront/test/pipeline/publish-kinds.ts

@@ -0,0 +1,162 @@
+/**
+ * The REAL publish / rollback / hotfix job kinds for the TenantPipeline fault harness, over the
+ * stand-in world (`src/lib/pipeline/publish/publish-test-support.ts`: GitHub Actions, CloudFront,
+ * the Cloudflare edge, the MCP promotion ledger). Test-only.
+ *
+ * Every byte of state — the job's KV (release ledger, version store, `live` pointer) and the
+ * stand-ins' — lives in the Durable Object's own SQLite, so it survives the evictions the harness
+ * causes. Timing is shortened so the CloudFront callback window, GitHub run lookups and proof
+ * retries play out in seconds.
+ *
+ * Faults are armed per object (each test uses its own tenant) at a named step — or `on-terminal`, the
+ * kind's compensation after a terminal transition — `before` its work or `after` it (once the work
+ * is done, before the engine records the step): `evict` aborts the
+ * object, `throw` fails the attempt. A fault's remaining count is persisted (and flushed) before it
+ * acts, so it survives the eviction it causes.
+ */
+import type { DurableObjectState } from "cloudflare:workers";
+import { JobKindRegistry, type JobKindDefinition, type StepDefinition } from "../../src/lib/pipeline/job-runner";
+import { registerPublishJobKinds, type PublishKindTuning, type PublishTiming } from "../../src/lib/pipeline/publish/publish-job";
+import { PublishWorld, standInPublishEnvironment, type TestKv } from "../../src/lib/pipeline/publish/publish-test-support";
+
+export const PUBLISH_HARNESS_TIMING: PublishTiming = {
+  awaitWindowMs: 1_500,
+  runGraceMs: 500,
+  proofDelayMs: 30,
+  proofProbes: { synchronous: 3, asynchronous: 5 },
+  stageWindowMs: 1_500,
+  stageChecks: 3,
+};
+
+export const PUBLISH_HARNESS_TUNING: PublishKindTuning = {
+  leaseMs: 3_000,
+  retryBaseMs: 20,
+  deadlineMs: { synchronous: 30_000, asynchronous: 30_000 },
+  hotfixExtraMs: 0,
+  compensationRetryBaseMs: 300,
+};
+
+export interface PublishFault {
+  step: string;
+  at: "before" | "after";
+  mode: "evict" | "throw";
+  /** How many times it fires (default 1). */
+  times?: number;
+}
+
+function tables(state: DurableObjectState): void {
+  const { sql } = state.storage;
+  sql.exec(`CREATE TABLE IF NOT EXISTS harness_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS harness_publish_faults (id INTEGER PRIMARY KEY AUTOINCREMENT, step TEXT, at TEXT, mode TEXT, remaining INTEGER)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS harness_publish_fired (seq INTEGER PRIMARY KEY AUTOINCREMENT, step TEXT, at TEXT, mode TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS harness_publish_armed (seq INTEGER PRIMARY KEY AUTOINCREMENT, step TEXT, at TEXT, armed INTEGER)`);
+}
+
+/** At each fired fault: how many compensations the engine held armed for jobs not yet terminal. */
+export function armedAtFaults(state: DurableObjectState): { step: string; at: string; armed: number }[] {
+  tables(state);
+  return state.storage.sql.exec<{ step: string; at: string; armed: number }>(`SELECT step, at, armed FROM harness_publish_armed ORDER BY seq`).toArray();
+}
+
+function armedCompensations(state: DurableObjectState): number {
+  try {
+    return state.storage.sql
+      .exec<{ n: number }>(
+        `SELECT COUNT(*) AS n FROM terminal_hooks h JOIN jobs j ON j.job_id = h.job_id WHERE j.state NOT IN ('succeeded', 'needs-developer', 'stuck')`,
+      )
+      .one().n;
+  } catch {
+    return -1;
+  }
+}
+
+/** A KV over the object's SQLite: durable across evictions, visible to every later execution. */
+export function sqlKv(state: DurableObjectState): TestKv {
+  const { sql } = state.storage;
+  return {
+    get: async (key) => sql.exec<{ value: string }>(`SELECT value FROM harness_kv WHERE key = ?`, key).toArray()[0]?.value ?? null,
+    put: async (key, value) => {
+      sql.exec(`INSERT INTO harness_kv (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, key, value);
+    },
+    delete: async (key) => {
+      sql.exec(`DELETE FROM harness_kv WHERE key = ?`, key);
+    },
+  };
+}
+
+export function armPublishFault(state: DurableObjectState, fault: PublishFault): void {
+  tables(state);
+  state.storage.sql.exec(
+    `INSERT INTO harness_publish_faults (step, at, mode, remaining) VALUES (?, ?, ?, ?)`,
+    fault.step,
+    fault.at,
+    fault.mode,
+    fault.times ?? 1,
+  );
+}
+
+export function firedPublishFaults(state: DurableObjectState): { step: string; at: string; mode: string }[] {
+  tables(state);
+  return state.storage.sql.exec<{ step: string; at: string; mode: string }>(`SELECT step, at, mode FROM harness_publish_fired ORDER BY seq`).toArray();
+}
+
+async function maybeFault(state: DurableObjectState, step: string, at: "before" | "after"): Promise<void> {
+  const { sql } = state.storage;
+  const fault = sql
+    .exec<{ id: number; mode: string }>(`SELECT id, mode FROM harness_publish_faults WHERE step = ? AND at = ? AND remaining > 0 ORDER BY id LIMIT 1`, step, at)
+    .toArray()[0];
+  if (!fault) return;
+  sql.exec(`UPDATE harness_publish_faults SET remaining = remaining - 1 WHERE id = ?`, fault.id);
+  sql.exec(`INSERT INTO harness_publish_fired (step, at, mode) VALUES (?, ?, ?)`, step, at, fault.mode);
+  sql.exec(`INSERT INTO harness_publish_armed (step, at, armed) VALUES (?, ?, ?)`, step, at, armedCompensations(state));
+  // Everything written so far — including a step's own effect, for an `after` fault — is made
+  // durable first, exactly as a real crash after the write would leave it.
+  await state.storage.sync();
+  if (fault.mode === "evict") {
+    state.abort(`synthetic eviction ${at} ${step}`);
+    throw new Error("abort() returned");
+  }
+  throw new Error(`synthetic fault ${at} ${step}`);
+}
+
+function withFaults(state: DurableObjectState, definition: JobKindDefinition<unknown>): JobKindDefinition<unknown> {
+  const onTerminal = definition.onTerminal;
+  return {
+    ...definition,
+    // The compensation after a terminal transition can be faulted too (as the step "on-terminal").
+    ...(onTerminal
+      ? {
+          onTerminal: async (info: Parameters<NonNullable<JobKindDefinition<unknown>["onTerminal"]>>[0]) => {
+            await maybeFault(state, "on-terminal", "before");
+            await onTerminal(info);
+          },
+        }
+      : {}),
+    steps: definition.steps.map(
+      (step): StepDefinition<unknown> => ({
+        ...step,
+        async run(ctx) {
+          await maybeFault(state, step.name, "before");
+          const outcome = await step.run(ctx);
+          await maybeFault(state, step.name, "after");
+          return outcome;
+        },
+      }),
+    ),
+  };
+}
+
+export function publishWorldOf(state: DurableObjectState): { kv: TestKv; world: PublishWorld } {
+  tables(state);
+  const kv = sqlKv(state);
+  return { kv, world: new PublishWorld(kv) };
+}
+
+/** Register the publish kinds (fault-wrapped, over the stand-in world) into `registry`. */
+export function registerHarnessPublishKinds(state: DurableObjectState, registry: JobKindRegistry): JobKindRegistry {
+  const { kv, world } = publishWorldOf(state);
+  const environment = standInPublishEnvironment({ kv, world, now: () => Date.now(), timing: PUBLISH_HARNESS_TIMING });
+  const kinds = registerPublishJobKinds(new JobKindRegistry(), async () => environment, PUBLISH_HARNESS_TUNING);
+  for (const name of kinds.list()) registry.register(withFaults(state, kinds.get(name)!));
+  return registry;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tokenoftrust/storefront-runner",
-  "version": "2.4.3",
+  "version": "2.4.5",
   "license": "SEE LICENSE IN LICENSE",
   "description": "World-shareable storefront runner: multi-tenant renderer on Astro/Cloudflare. No control plane.",
   "packageManager": "pnpm@11.9.0",

package/packages/public-runtime/src/tenant-assets.ts

@@ -425,7 +425,7 @@ export class AssetsPointerConflictError extends Error {
 }
 
 /**
- * Point a channel at a snapshot versionId (CAS on `rev`). KV has no hardware CAS,
+ * Point a channel at a snapshot versionId — or unset it with `null` (CAS on `rev`). KV has no hardware CAS,
  * so the caller reads the pointer once, passes its `rev` as `expectedRev`, and
  * this rejects the write if the stored `rev` has since moved — two concurrent
  * asset-promotions for one tenant can no longer silently clobber each other
@@ -436,7 +436,7 @@ export async function setAssetsChannel(
   kv: AssetsKv,
   tenantId: string,
   channel: CustomizationEnv,
-  versionId: string,
+  versionId: string | null,
   expectedRev: number,
 ): Promise<AssetsPointer> {
   const current = await readAssetsPointer(kv, tenantId);