@tokenoftrust/storefront-runner 2.4.2 → 2.4.4

package/apps/storefront/src/lib/social/golive.ts

@@ -1,7 +1,7 @@
 /**
  * S1 — the social go-live hook + idempotency.
  *
- * WHERE IT FIRES: at the true go-live edge — after `AggregateShipOrchestrator`
+ * WHERE IT FIRES: at the true go-live edge — after the tenant pipeline's publish job
  * flips a tenant's `live` channel to a freshly-shipped version and records the
  * release ledger. NOT on `POST /api/admin/blog`'s `publish` (which only opens a
  * candidate PR a human still promotes) and NOT on accept-to-preview

package/apps/storefront/test/pipeline/fake-preview-services.ts

@@ -15,6 +15,10 @@
  *                                 email, then the object is evicted
  *   - `reaccept-times-out`        after the forge merged, the re-accept of a resumed merge answers a
  *                                 transient timeout instead of a refusal
+ *   - `slow-build`                every file of a build takes a few milliseconds (a build that is
+ *                                 still running seconds after its intent was answered)
+ * The fake also records what each job reported to MCP (`fake_events` kind `report`) and which
+ * shas the preview could not render (`refused`).
  * A fault fires `times` times (0 = every time); its firing is persisted before it acts.
  */
 import type { DurableObjectState } from "cloudflare:workers";
@@ -29,6 +33,9 @@ import type {
 /** Files a fake build materializes. */
 export const FAKE_FILE_COUNT = 250;
 
+/** How long one file of a `slow-build` takes: the whole build, about a second and a half. */
+export const SLOW_FILE_MS = 6;
+
 export interface FakeFault {
   point: string;
   times: number;
@@ -43,6 +50,9 @@ export interface FakeState {
   staged: string[];
   refreshes: string[];
   emails: string[];
+  /** `<sha>:<evidence|refused>`, in report order. */
+  reports: string[];
+  refused: string[];
 }
 
 export function fakeTables(state: DurableObjectState): void {
@@ -66,6 +76,8 @@ export function configureFake(
     faults?: FakeFault[];
     /** Other open candidates a base advance rebuilds; `conflicts` ones email their author. */
     rebaseCandidates?: { changeId: string; pr: number; conflicts?: boolean }[];
+    /** `[ancestor, descendant]` commit pairs the fake forge answers `isAncestor` true for. */
+    ancestry?: [string, string][];
   },
 ): void {
   fakeTables(state);
@@ -76,6 +88,9 @@ export function configureFake(
   for (const f of config.faults ?? []) {
     sql.exec(`INSERT OR REPLACE INTO fake_faults (point, times, fired) VALUES (?, ?, 0)`, f.point, f.times);
   }
+  for (const [ancestor, descendant] of config.ancestry ?? []) {
+    sql.exec(`INSERT OR IGNORE INTO fake_ancestry (ancestor, descendant) VALUES (?, ?)`, ancestor, descendant);
+  }
   for (const c of config.rebaseCandidates ?? []) {
     sql.exec(`INSERT OR REPLACE INTO fake_candidates (change_id, pr, conflicts) VALUES (?, ?, ?)`, c.changeId, c.pr, c.conflicts ? 1 : 0);
   }
@@ -103,6 +118,8 @@ export function readFakeState(state: DurableObjectState): FakeState {
     staged: events("stage"),
     refreshes: events("refresh"),
     emails: events("email"),
+    reports: events("report"),
+    refused: events("refused"),
   };
 }
 
@@ -181,9 +198,11 @@ export function fakePreviewServices(state: DurableObjectState): PreviewJobServic
           cause: new ForgeTransportError("timed out reading the body", { transient: true, errorKind: "timeout" }),
         };
       }
+      const slow = await fires("slow-build");
       for (let file = 0; file < FAKE_FILE_COUNT; file += 1) {
         const path = `content/file-${file}.json`;
         if (resume.done(path)) continue;
+        if (slow) await new Promise((resolve) => setTimeout(resolve, SLOW_FILE_MS));
         if (await fires(`evict-at-file:${file}`)) evict(`at file ${file}`);
         sql.exec(`INSERT INTO fake_file_runs (sha, file) VALUES (?, ?)`, sha, file);
         await resume.record(path, { storedVersionId: `v-${sha}-${file}`, issues: [], skippedHere: false });
@@ -191,8 +210,17 @@ export function fakePreviewServices(state: DurableObjectState): PreviewJobServic
       return { ok: true, versionId: sha, digest: `digest-${sha}` };
     },
 
-    async evidence() {
-      return { promotable: true };
+    async evidence({ sha }) {
+      return { promotable: true, report: { sha, promotable: true, checks: [] } };
+    },
+
+    async reportOutcome({ sha, outcome }) {
+      sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('report', ?)`, `${sha}:${outcome.kind}`);
+      return { reported: true };
+    },
+
+    async recordPreviewRefused({ sha }) {
+      sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('refused', ?)`, sha);
     },
 
     async previewPointer() {
@@ -204,8 +232,9 @@ export function fakePreviewServices(state: DurableObjectState): PreviewJobServic
       return sql.exec(`SELECT 1 FROM fake_ancestry WHERE ancestor = ? AND descendant = ?`, ancestor, descendant).toArray().length > 0;
     },
 
-    async advancePreview({ sha, membership, jobId, kind }) {
+    async advancePreview({ sha, membership, jobId, kind, releasable }) {
       setPointer(sha);
+      if (!releasable) return;
       sql.exec(
         `INSERT INTO fake_advances (sha, membership, job_id, kind) VALUES (?, ?, ?, ?)`,
         sha,

package/apps/storefront/test/pipeline/harness.ts

@@ -21,7 +21,19 @@ interface HarnessEnv {
 }
 
 type TenantPipelineStub = {
-  [K in "submit" | "job" | "jobs" | "transitions" | "facts" | "harness" | "probeStaleWrites" | "configureFake" | "fakeState" | "spans"]: TenantPipeline[K];
+  [K in
+    | "submit"
+    | "deliverExternal"
+    | "job"
+    | "jobs"
+    | "transitions"
+    | "facts"
+    | "harness"
+    | "probeStaleWrites"
+    | "configureFake"
+    | "fakeState"
+    | "spans"
+    | "publishHarness"]: TenantPipeline[K];
 };
 
 export const harnessEnv = env as unknown as HarnessEnv;
@@ -105,9 +117,9 @@ export async function waitTerminal(tenant: string, jobIds: string[], timeoutMs =
       await new Promise((resolve) => setTimeout(resolve, 25));
       continue;
     }
-    const running = jobs.filter((j) => j.state === "running").length;
+    const running = jobs.filter((j) => j.state === "running" || j.state === "awaiting-external").length;
     maxRunning = Math.max(maxRunning, running);
-    expect(running, "running jobs observed at once").toBeLessThanOrEqual(1);
+    expect(running, "running (or awaiting) jobs observed at once").toBeLessThanOrEqual(1);
     const wanted = jobIds.map((id) => jobs.find((j) => j.jobId === id));
     if (wanted.every((j) => j && TERMINAL.has(j.state))) return wanted as JobView[];
     if (Date.now() > deadline) {

package/apps/storefront/test/pipeline/pipeline-host.ts

@@ -2,16 +2,29 @@
  * The Worker that hosts the TenantPipeline Durable Object for the harness, in its own isolate
  * (as in production, where callers never share the object's isolate — and where an eviction
  * restarts the object without touching its callers). It runs the production engine with the
- * synthetic job kinds AND the production preview kinds (integrate / rebuild / revert) over fake
- * services (`./fake-preview-services.ts`), and answers intents through `submitIntent`.
+ * synthetic job kinds, the production preview kinds (integrate / rebuild / revert / sync-preview) over
+ * fake services (`./fake-preview-services.ts`) and the real publish kinds over the stand-in world
+ * (`./publish-kinds.ts`), and answers intents through `submitIntent`.
  */
 import type { DurableObjectState } from "cloudflare:workers";
 import { TenantPipeline as DeployedTenantPipeline } from "../../src/lib/pipeline/tenant-pipeline";
-import { submitIntent, type TenantPipelineNamespace } from "../../src/lib/pipeline/intake";
+import { submitIntent, tenantPipeline, type TenantPipelineNamespace } from "../../src/lib/pipeline/intake";
+import { submitPreviewSync } from "../../src/lib/pipeline/preview-sync-intake";
 import { StaleExecutionError, type JobKindRegistry, type PipelineIntent } from "../../src/lib/pipeline/job-runner";
 import { abandonedContexts, syntheticJobKinds, syntheticSpans } from "./synthetic-job";
 import { registerPreviewJobKinds, type PreviewJobTiming } from "../../src/lib/pipeline/preview-jobs";
 import { configureFake, fakePreviewServices, readFakeState, type FakeFault, type FakeState } from "./fake-preview-services";
+import { readLivePointer } from "../../src/lib/pipeline/publish/live-pointer";
+import { listReleaseReceipts, readReleaseTruth } from "../../src/lib/change/store/aggregate-release-store";
+import { seedGreenAggregate, testVersionStore } from "../../src/lib/pipeline/publish/publish-test-support";
+import {
+  armedAtFaults,
+  armPublishFault,
+  firedPublishFaults,
+  publishWorldOf,
+  registerHarnessPublishKinds,
+  type PublishFault,
+} from "./publish-kinds";
 
 /** The preview kinds' timing in the harness: the production shape, scaled to milliseconds. */
 export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
@@ -24,11 +37,18 @@ export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
 
 export class TenantPipeline extends DeployedTenantPipeline {
   protected override jobKinds(ctx: DurableObjectState): JobKindRegistry {
-    return registerPreviewJobKinds(syntheticJobKinds(ctx), fakePreviewServices(ctx), HARNESS_PREVIEW_TIMING);
+    return registerHarnessPublishKinds(
+      ctx,
+      registerPreviewJobKinds(syntheticJobKinds(ctx), fakePreviewServices(ctx), HARNESS_PREVIEW_TIMING),
+    );
   }
 
   /** Harness-only: open candidates and arm faults for the fake preview services. */
-  async configureFake(config: { open?: { changeId: string; headSha: string }[]; faults?: FakeFault[] }): Promise<void> {
+  async configureFake(config: {
+    open?: { changeId: string; headSha: string }[];
+    faults?: FakeFault[];
+    ancestry?: [string, string][];
+  }): Promise<void> {
     configureFake(this.ctx, config);
   }
 
@@ -42,6 +62,45 @@ export class TenantPipeline extends DeployedTenantPipeline {
     return readFakeState(this.ctx);
   }
 
+  /** Harness-only: arrange and read the publish stand-in world inside this object's storage. */
+  async publishHarness(op: string, args: Record<string, unknown> = {}): Promise<unknown> {
+    const { kv, world } = publishWorldOf(this.ctx);
+    const store = testVersionStore(kv);
+    const tenantId = String(args.tenantId ?? "");
+    switch (op) {
+      case "configure":
+        return world.configure(args.patch as never);
+      case "seedGreen":
+        await seedGreenAggregate(kv, store, args as never);
+        return null;
+      case "completeRun":
+        return world.completeRun(String(args.dispatchId), args.conclusion as never, args.served as never);
+      case "serveOnCloudFront":
+        return world.serveOnCloudFront(String(args.sha), (args.treeDigest as string | null) ?? null);
+      case "completeStage":
+        return world.completeStage(String(args.dispatchId), args.conclusion as never, args.digest as never);
+      case "dispatches":
+        return world.dispatches();
+      case "promotionStatus":
+        return world.promotionStatus();
+      case "live":
+        return readLivePointer(store, tenantId);
+      case "receipts":
+        return (await listReleaseReceipts(kv, tenantId, "preview")).map((r) => r.receiptId);
+      case "truth":
+        return readReleaseTruth(kv, tenantId, "preview");
+      case "armFault":
+        armPublishFault(this.ctx, args as unknown as PublishFault);
+        return null;
+      case "firedFaults":
+        return firedPublishFaults(this.ctx);
+      case "armedAtFaults":
+        return armedAtFaults(this.ctx);
+      default:
+        throw new Error(`unknown publish harness op ${op}`);
+    }
+  }
+
   /** Harness-only reads of the synthetic step log. */
   async harness(jobId: string): Promise<{
     runs: { step: number; chunk: number; attempt: number }[];
@@ -91,8 +150,28 @@ interface Env {
   TENANT_PIPELINE: TenantPipelineNamespace;
 }
 
+/**
+ * `/sync/<tenant>` answers a forwarded push the way `/api/preview/reconcile` does: submit the
+ * `sync-preview` intent and answer 202. `/sync-hold/<tenant>` submits, then holds the response open
+ * (a forward the caller kills mid-flight): the job must not depend on the request.
+ */
+async function syncRoute(request: Request, env: Env, tenant: string, hold: boolean): Promise<Response> {
+  const body = (await request.json()) as { repo: string; sha: string };
+  const result = await submitPreviewSync(
+    tenantPipeline(env.TENANT_PIPELINE, tenant),
+    tenant,
+    { repo: body.repo, sha: body.sha, source: "push" },
+    { actor: { type: "service", id: "harness-forward" }, admissionDecisionId: null },
+  );
+  if (!result.ok) return Response.json({ error: result.code }, { status: 400 });
+  if (hold) await new Promise((resolve) => setTimeout(resolve, 10_000));
+  return Response.json({ jobId: result.jobId, state: result.state, duplicate: result.duplicate }, { status: 202 });
+}
+
 export default {
   async fetch(request: Request, env: Env): Promise<Response> {
+    const sync = /^\/(sync|sync-hold)\/([^/]+)$/.exec(new URL(request.url).pathname);
+    if (request.method === "POST" && sync) return syncRoute(request, env, decodeURIComponent(sync[2]!), sync[1] === "sync-hold");
     const match = /^\/intents\/([^/]+)$/.exec(new URL(request.url).pathname);
     if (request.method !== "POST" || !match) return new Response("not found", { status: 404 });
     const response = await submitIntent(env.TENANT_PIPELINE, decodeURIComponent(match[1]!), (await request.json()) as PipelineIntent);

package/apps/storefront/test/pipeline/publish-kinds.ts

@@ -0,0 +1,162 @@
+/**
+ * The REAL publish / rollback / hotfix job kinds for the TenantPipeline fault harness, over the
+ * stand-in world (`src/lib/pipeline/publish/publish-test-support.ts`: GitHub Actions, CloudFront,
+ * the Cloudflare edge, the MCP promotion ledger). Test-only.
+ *
+ * Every byte of state — the job's KV (release ledger, version store, `live` pointer) and the
+ * stand-ins' — lives in the Durable Object's own SQLite, so it survives the evictions the harness
+ * causes. Timing is shortened so the CloudFront callback window, GitHub run lookups and proof
+ * retries play out in seconds.
+ *
+ * Faults are armed per object (each test uses its own tenant) at a named step — or `on-terminal`, the
+ * kind's compensation after a terminal transition — `before` its work or `after` it (once the work
+ * is done, before the engine records the step): `evict` aborts the
+ * object, `throw` fails the attempt. A fault's remaining count is persisted (and flushed) before it
+ * acts, so it survives the eviction it causes.
+ */
+import type { DurableObjectState } from "cloudflare:workers";
+import { JobKindRegistry, type JobKindDefinition, type StepDefinition } from "../../src/lib/pipeline/job-runner";
+import { registerPublishJobKinds, type PublishKindTuning, type PublishTiming } from "../../src/lib/pipeline/publish/publish-job";
+import { PublishWorld, standInPublishEnvironment, type TestKv } from "../../src/lib/pipeline/publish/publish-test-support";
+
+export const PUBLISH_HARNESS_TIMING: PublishTiming = {
+  awaitWindowMs: 1_500,
+  runGraceMs: 500,
+  proofDelayMs: 30,
+  proofProbes: { synchronous: 3, asynchronous: 5 },
+  stageWindowMs: 1_500,
+  stageChecks: 3,
+};
+
+export const PUBLISH_HARNESS_TUNING: PublishKindTuning = {
+  leaseMs: 3_000,
+  retryBaseMs: 20,
+  deadlineMs: { synchronous: 30_000, asynchronous: 30_000 },
+  hotfixExtraMs: 0,
+  compensationRetryBaseMs: 300,
+};
+
+export interface PublishFault {
+  step: string;
+  at: "before" | "after";
+  mode: "evict" | "throw";
+  /** How many times it fires (default 1). */
+  times?: number;
+}
+
+function tables(state: DurableObjectState): void {
+  const { sql } = state.storage;
+  sql.exec(`CREATE TABLE IF NOT EXISTS harness_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS harness_publish_faults (id INTEGER PRIMARY KEY AUTOINCREMENT, step TEXT, at TEXT, mode TEXT, remaining INTEGER)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS harness_publish_fired (seq INTEGER PRIMARY KEY AUTOINCREMENT, step TEXT, at TEXT, mode TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS harness_publish_armed (seq INTEGER PRIMARY KEY AUTOINCREMENT, step TEXT, at TEXT, armed INTEGER)`);
+}
+
+/** At each fired fault: how many compensations the engine held armed for jobs not yet terminal. */
+export function armedAtFaults(state: DurableObjectState): { step: string; at: string; armed: number }[] {
+  tables(state);
+  return state.storage.sql.exec<{ step: string; at: string; armed: number }>(`SELECT step, at, armed FROM harness_publish_armed ORDER BY seq`).toArray();
+}
+
+function armedCompensations(state: DurableObjectState): number {
+  try {
+    return state.storage.sql
+      .exec<{ n: number }>(
+        `SELECT COUNT(*) AS n FROM terminal_hooks h JOIN jobs j ON j.job_id = h.job_id WHERE j.state NOT IN ('succeeded', 'needs-developer', 'stuck')`,
+      )
+      .one().n;
+  } catch {
+    return -1;
+  }
+}
+
+/** A KV over the object's SQLite: durable across evictions, visible to every later execution. */
+export function sqlKv(state: DurableObjectState): TestKv {
+  const { sql } = state.storage;
+  return {
+    get: async (key) => sql.exec<{ value: string }>(`SELECT value FROM harness_kv WHERE key = ?`, key).toArray()[0]?.value ?? null,
+    put: async (key, value) => {
+      sql.exec(`INSERT INTO harness_kv (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, key, value);
+    },
+    delete: async (key) => {
+      sql.exec(`DELETE FROM harness_kv WHERE key = ?`, key);
+    },
+  };
+}
+
+export function armPublishFault(state: DurableObjectState, fault: PublishFault): void {
+  tables(state);
+  state.storage.sql.exec(
+    `INSERT INTO harness_publish_faults (step, at, mode, remaining) VALUES (?, ?, ?, ?)`,
+    fault.step,
+    fault.at,
+    fault.mode,
+    fault.times ?? 1,
+  );
+}
+
+export function firedPublishFaults(state: DurableObjectState): { step: string; at: string; mode: string }[] {
+  tables(state);
+  return state.storage.sql.exec<{ step: string; at: string; mode: string }>(`SELECT step, at, mode FROM harness_publish_fired ORDER BY seq`).toArray();
+}
+
+async function maybeFault(state: DurableObjectState, step: string, at: "before" | "after"): Promise<void> {
+  const { sql } = state.storage;
+  const fault = sql
+    .exec<{ id: number; mode: string }>(`SELECT id, mode FROM harness_publish_faults WHERE step = ? AND at = ? AND remaining > 0 ORDER BY id LIMIT 1`, step, at)
+    .toArray()[0];
+  if (!fault) return;
+  sql.exec(`UPDATE harness_publish_faults SET remaining = remaining - 1 WHERE id = ?`, fault.id);
+  sql.exec(`INSERT INTO harness_publish_fired (step, at, mode) VALUES (?, ?, ?)`, step, at, fault.mode);
+  sql.exec(`INSERT INTO harness_publish_armed (step, at, armed) VALUES (?, ?, ?)`, step, at, armedCompensations(state));
+  // Everything written so far — including a step's own effect, for an `after` fault — is made
+  // durable first, exactly as a real crash after the write would leave it.
+  await state.storage.sync();
+  if (fault.mode === "evict") {
+    state.abort(`synthetic eviction ${at} ${step}`);
+    throw new Error("abort() returned");
+  }
+  throw new Error(`synthetic fault ${at} ${step}`);
+}
+
+function withFaults(state: DurableObjectState, definition: JobKindDefinition<unknown>): JobKindDefinition<unknown> {
+  const onTerminal = definition.onTerminal;
+  return {
+    ...definition,
+    // The compensation after a terminal transition can be faulted too (as the step "on-terminal").
+    ...(onTerminal
+      ? {
+          onTerminal: async (info: Parameters<NonNullable<JobKindDefinition<unknown>["onTerminal"]>>[0]) => {
+            await maybeFault(state, "on-terminal", "before");
+            await onTerminal(info);
+          },
+        }
+      : {}),
+    steps: definition.steps.map(
+      (step): StepDefinition<unknown> => ({
+        ...step,
+        async run(ctx) {
+          await maybeFault(state, step.name, "before");
+          const outcome = await step.run(ctx);
+          await maybeFault(state, step.name, "after");
+          return outcome;
+        },
+      }),
+    ),
+  };
+}
+
+export function publishWorldOf(state: DurableObjectState): { kv: TestKv; world: PublishWorld } {
+  tables(state);
+  const kv = sqlKv(state);
+  return { kv, world: new PublishWorld(kv) };
+}
+
+/** Register the publish kinds (fault-wrapped, over the stand-in world) into `registry`. */
+export function registerHarnessPublishKinds(state: DurableObjectState, registry: JobKindRegistry): JobKindRegistry {
+  const { kv, world } = publishWorldOf(state);
+  const environment = standInPublishEnvironment({ kv, world, now: () => Date.now(), timing: PUBLISH_HARNESS_TIMING });
+  const kinds = registerPublishJobKinds(new JobKindRegistry(), async () => environment, PUBLISH_HARNESS_TUNING);
+  for (const name of kinds.list()) registry.register(withFaults(state, kinds.get(name)!));
+  return registry;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tokenoftrust/storefront-runner",
-  "version": "2.4.2",
+  "version": "2.4.4",
   "license": "SEE LICENSE IN LICENSE",
   "description": "World-shareable storefront runner: multi-tenant renderer on Astro/Cloudflare. No control plane.",
   "packageManager": "pnpm@11.9.0",

package/packages/public-runtime/src/tenant-assets.ts

@@ -425,7 +425,7 @@ export class AssetsPointerConflictError extends Error {
 }
 
 /**
- * Point a channel at a snapshot versionId (CAS on `rev`). KV has no hardware CAS,
+ * Point a channel at a snapshot versionId — or unset it with `null` (CAS on `rev`). KV has no hardware CAS,
  * so the caller reads the pointer once, passes its `rev` as `expectedRev`, and
  * this rejects the write if the stored `rev` has since moved — two concurrent
  * asset-promotions for one tenant can no longer silently clobber each other
@@ -436,7 +436,7 @@ export async function setAssetsChannel(
   kv: AssetsKv,
   tenantId: string,
   channel: CustomizationEnv,
-  versionId: string,
+  versionId: string | null,
   expectedRev: number,
 ): Promise<AssetsPointer> {
   const current = await readAssetsPointer(kv, tenantId);