@tokenoftrust/storefront-runner 2.4.1 → 2.4.3

package/apps/storefront/test/pipeline/fake-preview-services.ts

@@ -0,0 +1,278 @@
+/**
+ * Fake {@link PreviewJobServices} for the TenantPipeline fault harness: a forge, a version store
+ * and a release read model held in the Durable Object's own SQLite (`fake_*` tables), so every
+ * effect survives the evictions the tests inject and can be counted afterwards.
+ *
+ * Faults are configured per object (one object per tenant) through `fake_faults` rows:
+ *   - `evict-after-forge-merge`   the forge merges, then the object is evicted before the merge
+ *                                 step records anything (the placeholder-sha hole)
+ *   - `evict-before-build`        evicted when the build step starts (after the merge was recorded)
+ *   - `evict-at-file:<n>`         evicted while the build processes file n
+ *   - `slow-read`                 every build read fails with a typed transient timeout
+ *   - `pointer-moves-to:<sha>`    while the build runs, the preview webhook moves the shared preview
+ *                                 to <sha>, a descendant of the commit being built
+ *   - `evict-after-email:<id>`    the follow-up's rebase of candidate <id> sends its author a conflict
+ *                                 email, then the object is evicted
+ *   - `reaccept-times-out`        after the forge merged, the re-accept of a resumed merge answers a
+ *                                 transient timeout instead of a refusal
+ *   - `slow-build`                every file of a build takes a few milliseconds (a build that is
+ *                                 still running seconds after its intent was answered)
+ * The fake also records what each job reported to MCP (`fake_events` kind `report`) and which
+ * shas the preview could not render (`refused`).
+ * A fault fires `times` times (0 = every time); its firing is persisted before it acts.
+ */
+import type { DurableObjectState } from "cloudflare:workers";
+import { ForgeToolError, ForgeTransportError } from "../../src/lib/forge/errors";
+import type {
+  MembershipEntry,
+  MergeCandidate,
+  PreviewJobServices,
+  ReadModelTip,
+} from "../../src/lib/pipeline/preview-jobs";
+
+/** Files a fake build materializes. */
+export const FAKE_FILE_COUNT = 250;
+
+/** How long one file of a `slow-build` takes: the whole build, about a second and a half. */
+export const SLOW_FILE_MS = 6;
+
+export interface FakeFault {
+  point: string;
+  times: number;
+}
+
+export interface FakeState {
+  merges: { changeId: string; sha: string; message: string }[];
+  fileRuns: { sha: string; file: number }[];
+  advances: { sha: string; membership: MembershipEntry[]; jobId: string; kind: string }[];
+  readModel: { sha: string; membership: MembershipEntry[]; green: boolean } | null;
+  previewPointer: string | null;
+  staged: string[];
+  refreshes: string[];
+  emails: string[];
+  /** `<sha>:<evidence|refused>`, in report order. */
+  reports: string[];
+  refused: string[];
+}
+
+export function fakeTables(state: DurableObjectState): void {
+  const { sql } = state.storage;
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_open (change_id TEXT PRIMARY KEY, head_sha TEXT NOT NULL)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_merges (seq INTEGER PRIMARY KEY AUTOINCREMENT, change_id TEXT, sha TEXT, message TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_file_runs (seq INTEGER PRIMARY KEY AUTOINCREMENT, sha TEXT, file INTEGER)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_advances (seq INTEGER PRIMARY KEY AUTOINCREMENT, sha TEXT, membership TEXT, job_id TEXT, kind TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_read_model (k INTEGER PRIMARY KEY CHECK (k = 1), sha TEXT, membership TEXT, green INTEGER)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_faults (point TEXT PRIMARY KEY, times INTEGER NOT NULL, fired INTEGER NOT NULL DEFAULT 0)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_pointer (k INTEGER PRIMARY KEY CHECK (k = 1), sha TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_ancestry (ancestor TEXT, descendant TEXT, PRIMARY KEY (ancestor, descendant))`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_events (seq INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT, subject TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS fake_candidates (change_id TEXT PRIMARY KEY, pr INTEGER, conflicts INTEGER)`);
+}
+
+export function configureFake(
+  state: DurableObjectState,
+  config: {
+    open?: { changeId: string; headSha: string }[];
+    faults?: FakeFault[];
+    /** Other open candidates a base advance rebuilds; `conflicts` ones email their author. */
+    rebaseCandidates?: { changeId: string; pr: number; conflicts?: boolean }[];
+    /** `[ancestor, descendant]` commit pairs the fake forge answers `isAncestor` true for. */
+    ancestry?: [string, string][];
+  },
+): void {
+  fakeTables(state);
+  const { sql } = state.storage;
+  for (const c of config.open ?? []) {
+    sql.exec(`INSERT OR REPLACE INTO fake_open (change_id, head_sha) VALUES (?, ?)`, c.changeId, c.headSha);
+  }
+  for (const f of config.faults ?? []) {
+    sql.exec(`INSERT OR REPLACE INTO fake_faults (point, times, fired) VALUES (?, ?, 0)`, f.point, f.times);
+  }
+  for (const [ancestor, descendant] of config.ancestry ?? []) {
+    sql.exec(`INSERT OR IGNORE INTO fake_ancestry (ancestor, descendant) VALUES (?, ?)`, ancestor, descendant);
+  }
+  for (const c of config.rebaseCandidates ?? []) {
+    sql.exec(`INSERT OR REPLACE INTO fake_candidates (change_id, pr, conflicts) VALUES (?, ?, ?)`, c.changeId, c.pr, c.conflicts ? 1 : 0);
+  }
+}
+
+export function readFakeState(state: DurableObjectState): FakeState {
+  fakeTables(state);
+  const { sql } = state.storage;
+  const rm = sql.exec<{ sha: string; membership: string; green: number }>(`SELECT * FROM fake_read_model`).toArray()[0];
+  const pointer = sql.exec<{ sha: string }>(`SELECT sha FROM fake_pointer`).toArray()[0];
+  const events = (kind: string) =>
+    sql.exec<{ subject: string }>(`SELECT subject FROM fake_events WHERE kind = ? ORDER BY seq`, kind).toArray().map((r) => r.subject);
+  return {
+    merges: sql
+      .exec<{ change_id: string; sha: string; message: string }>(`SELECT * FROM fake_merges ORDER BY seq`)
+      .toArray()
+      .map((r) => ({ changeId: r.change_id, sha: r.sha, message: r.message })),
+    fileRuns: sql.exec<{ sha: string; file: number }>(`SELECT sha, file FROM fake_file_runs ORDER BY seq`).toArray(),
+    advances: sql
+      .exec<{ sha: string; membership: string; job_id: string; kind: string }>(`SELECT * FROM fake_advances ORDER BY seq`)
+      .toArray()
+      .map((r) => ({ sha: r.sha, membership: JSON.parse(r.membership), jobId: r.job_id, kind: r.kind })),
+    readModel: rm ? { sha: rm.sha, membership: JSON.parse(rm.membership), green: rm.green === 1 } : null,
+    previewPointer: pointer?.sha ?? null,
+    staged: events("stage"),
+    refreshes: events("refresh"),
+    emails: events("email"),
+    reports: events("report"),
+    refused: events("refused"),
+  };
+}
+
+export function fakePreviewServices(state: DurableObjectState): PreviewJobServices {
+  fakeTables(state);
+  const { sql } = state.storage;
+
+  /** True when `point` should fire now; records the firing durably first. */
+  async function fires(point: string): Promise<boolean> {
+    const row = sql.exec<{ times: number; fired: number }>(`SELECT times, fired FROM fake_faults WHERE point = ?`, point).toArray()[0];
+    if (!row) return false;
+    if (row.times !== 0 && row.fired >= row.times) return false;
+    sql.exec(`UPDATE fake_faults SET fired = fired + 1 WHERE point = ?`, point);
+    await state.storage.sync();
+    return true;
+  }
+
+  function evict(point: string): never {
+    state.abort(`synthetic eviction ${point}`);
+    throw new Error("abort() returned");
+  }
+
+  const setPointer = (sha: string) =>
+    sql.exec(`INSERT INTO fake_pointer (k, sha) VALUES (1, ?) ON CONFLICT(k) DO UPDATE SET sha = excluded.sha`, sha);
+
+  return {
+    async integrateCandidate(_repo, changeId, opts) {
+      const open = sql.exec<{ head_sha: string }>(`SELECT head_sha FROM fake_open WHERE change_id = ?`, changeId).toArray()[0];
+      if (!open) {
+        if (await fires("reaccept-times-out")) {
+          throw new ForgeTransportError("timed out reading the body", { transient: true, errorKind: "timeout" });
+        }
+        throw new ForgeToolError(`No open candidate "${changeId}".`, { data: { status: "not_found" } });
+      }
+      const n = sql.exec<{ n: number }>(`SELECT COUNT(*) AS n FROM fake_merges`).one().n;
+      const sha = `sha-${n + 1}-${changeId}`;
+      sql.exec(`DELETE FROM fake_open WHERE change_id = ?`, changeId);
+      sql.exec(`INSERT INTO fake_merges (change_id, sha, message) VALUES (?, ?, ?)`, changeId, sha, opts.message);
+      await state.storage.sync();
+      if (await fires("evict-after-forge-merge")) evict("after forge merge");
+      return { sha, integratedInto: "preview" };
+    },
+
+    async candidatePaths() {
+      return ["content/home.json"];
+    },
+
+    async findMergedPreviewCommit(_repo, paths, { tag }) {
+      if (paths.length === 0) return null;
+      const row = sql
+        .exec<{ sha: string; message: string }>(`SELECT sha, message FROM fake_merges ORDER BY seq DESC`)
+        .toArray()
+        .find((r) => r.message.split("\n", 1)[0]!.includes(tag));
+      return row?.sha ?? null;
+    },
+
+    async readModelTip(): Promise<ReadModelTip | null> {
+      const rm = sql.exec<{ sha: string; membership: string; green: number }>(`SELECT * FROM fake_read_model`).toArray()[0];
+      return rm ? { sha: rm.sha, membership: JSON.parse(rm.membership), green: rm.green === 1 } : null;
+    },
+
+    async materialize({ sha, resume }) {
+      if (await fires("evict-before-build")) evict("before build");
+      const moveTo = sql
+        .exec<{ point: string }>(`SELECT point FROM fake_faults WHERE point LIKE 'pointer-moves-to:%'`)
+        .toArray()[0]?.point;
+      if (moveTo && (await fires(moveTo))) {
+        const descendant = moveTo.slice("pointer-moves-to:".length);
+        sql.exec(`INSERT OR IGNORE INTO fake_ancestry (ancestor, descendant) VALUES (?, ?)`, sha, descendant);
+        setPointer(descendant);
+      }
+      if (await fires("slow-read")) {
+        return {
+          ok: false,
+          errors: ["fetch content/catalog: timed out"],
+          cause: new ForgeTransportError("timed out reading the body", { transient: true, errorKind: "timeout" }),
+        };
+      }
+      const slow = await fires("slow-build");
+      for (let file = 0; file < FAKE_FILE_COUNT; file += 1) {
+        const path = `content/file-${file}.json`;
+        if (resume.done(path)) continue;
+        if (slow) await new Promise((resolve) => setTimeout(resolve, SLOW_FILE_MS));
+        if (await fires(`evict-at-file:${file}`)) evict(`at file ${file}`);
+        sql.exec(`INSERT INTO fake_file_runs (sha, file) VALUES (?, ?)`, sha, file);
+        await resume.record(path, { storedVersionId: `v-${sha}-${file}`, issues: [], skippedHere: false });
+      }
+      return { ok: true, versionId: sha, digest: `digest-${sha}` };
+    },
+
+    async evidence({ sha }) {
+      return { promotable: true, report: { sha, promotable: true, checks: [] } };
+    },
+
+    async reportOutcome({ sha, outcome }) {
+      sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('report', ?)`, `${sha}:${outcome.kind}`);
+      return { reported: true };
+    },
+
+    async recordPreviewRefused({ sha }) {
+      sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('refused', ?)`, sha);
+    },
+
+    async previewPointer() {
+      return sql.exec<{ sha: string }>(`SELECT sha FROM fake_pointer`).toArray()[0]?.sha ?? null;
+    },
+
+    async isAncestor(_repo, ancestor, descendant) {
+      if (ancestor === descendant) return true;
+      return sql.exec(`SELECT 1 FROM fake_ancestry WHERE ancestor = ? AND descendant = ?`, ancestor, descendant).toArray().length > 0;
+    },
+
+    async advancePreview({ sha, membership, jobId, kind, releasable }) {
+      setPointer(sha);
+      if (!releasable) return;
+      sql.exec(
+        `INSERT INTO fake_advances (sha, membership, job_id, kind) VALUES (?, ?, ?, ?)`,
+        sha,
+        JSON.stringify(membership),
+        jobId,
+        kind,
+      );
+      sql.exec(
+        `INSERT INTO fake_read_model (k, sha, membership, green) VALUES (1, ?, ?, 1)
+         ON CONFLICT(k) DO UPDATE SET sha = excluded.sha, membership = excluded.membership, green = 1`,
+        sha,
+        JSON.stringify(membership),
+      );
+    },
+
+    async stageAggregate({ sha }) {
+      sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('stage', ?)`, sha);
+    },
+
+    async listRebaseCandidates({ acceptedChangeId }) {
+      return sql
+        .exec<{ change_id: string; pr: number }>(`SELECT change_id, pr FROM fake_candidates WHERE change_id != ? ORDER BY change_id`, acceptedChangeId)
+        .toArray()
+        .map((r) => ({ changeId: r.change_id, prNumber: r.pr }));
+    },
+
+    async rebaseCandidate({ candidate }) {
+      sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('refresh', ?)`, candidate.changeId);
+      const conflicts = sql.exec<{ conflicts: number }>(`SELECT conflicts FROM fake_candidates WHERE change_id = ?`, candidate.changeId).toArray()[0];
+      if (!conflicts?.conflicts) return { outcome: "refreshed" as const };
+      sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('email', ?)`, candidate.changeId);
+      await state.storage.sync();
+      if (await fires(`evict-after-email:${candidate.changeId}`)) evict(`after emailing ${candidate.changeId}`);
+      return { outcome: "conflict" as const, notified: true };
+    },
+
+    async prepareUndo(): Promise<MergeCandidate> {
+      throw new Error("prepareUndo is not exercised by the fault harness");
+    },
+  };
+}

package/apps/storefront/test/pipeline/harness.ts

@@ -21,7 +21,7 @@ interface HarnessEnv {
 }
 
 type TenantPipelineStub = {
-  [K in "submit" | "job" | "jobs" | "transitions" | "facts" | "harness" | "probeStaleWrites"]: TenantPipeline[K];
+  [K in "submit" | "job" | "jobs" | "transitions" | "facts" | "harness" | "probeStaleWrites" | "configureFake" | "fakeState" | "spans"]: TenantPipeline[K];
 };
 
 export const harnessEnv = env as unknown as HarnessEnv;
@@ -45,6 +45,23 @@ export interface Submitted {
   state: string;
 }
 
+/** Submit an intent of any kind through a Worker's intake (its own isolate), timing the answer. */
+export async function submitIntentVia(
+  tenant: string,
+  intent: { kind: string; input: unknown },
+  via: Fetcher = harnessEnv.PIPELINE_HOST,
+): Promise<Submitted> {
+  const started = performance.now();
+  const res = await via.fetch(`https://pipeline.test/intents/${encodeURIComponent(tenant)}`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ ...intent, actor: { type: "test", id: "harness" }, admissionDecisionId: null }),
+  });
+  const ms = performance.now() - started;
+  const body = (await res.json()) as { jobId: string; state: string };
+  return { status: res.status, jobId: body.jobId, ms, intake: res.headers.get("x-intake"), state: body.state };
+}
+
 export async function submit(
   tenant: string,
   faults: Fault[] = [],
@@ -68,7 +85,7 @@ export async function submit(
   return { status: res.status, jobId: body.jobId, ms, intake: res.headers.get("x-intake"), state: body.state };
 }
 
-const TERMINAL = new Set<JobState>(["succeeded", "failed", "stuck"]);
+const TERMINAL = new Set<JobState>(["succeeded", "needs-developer", "stuck"]);
 
 /**
  * Polls until every job is terminal. On every sample asserts the tenant never has more than one
@@ -100,14 +117,19 @@ export async function waitTerminal(tenant: string, jobIds: string[], timeoutMs =
   }
 }
 
-const OUTCOME_NOTES = ["step-done", "checkpoint", "lease-expired"];
+const OUTCOME_NOTES = ["step-done", "checkpoint", "lease-expired", "yielded"];
 
 /**
  * The job's transition log is a complete chain: created queued, started once, every step start
  * has exactly one recorded outcome (or is superseded by a resume after an eviction), each step
  * finished exactly once and in order, and it ends at `terminal` with nothing after it.
  */
-export function assertCompleteLog(log: TransitionView[], terminal: JobState, stepNames = ["s1", "s2", "s3", "s4", "s5"]): void {
+export function assertCompleteLog(
+  log: TransitionView[],
+  terminal: JobState,
+  stepNames = ["s1", "s2", "s3", "s4", "s5"],
+  opts: { checkpointsOf?: { step: string; count: number } | null } = {},
+): void {
   expect(log.length).toBeGreaterThan(2);
   expect([log[0]!.from, log[0]!.to]).toEqual(["none", "queued"]);
   expect([log[1]!.from, log[1]!.to]).toEqual(["queued", "running"]);
@@ -136,7 +158,8 @@ export function assertCompleteLog(log: TransitionView[], terminal: JobState, ste
   if (terminal === "succeeded") {
     const done = log.filter((t) => t.note === "step-done").map((t) => t.step);
     expect(done).toEqual(stepNames);
-    expect(log.filter((t) => t.note === "checkpoint" && t.step === "s3")).toHaveLength(S3_CHUNKS - 1);
+    const checkpoints = opts.checkpointsOf === undefined ? { step: "s3", count: S3_CHUNKS - 1 } : opts.checkpointsOf;
+    if (checkpoints) expect(log.filter((t) => t.note === "checkpoint" && t.step === checkpoints.step)).toHaveLength(checkpoints.count);
   }
 }
 

package/apps/storefront/test/pipeline/pipeline-host.ts

@@ -2,17 +2,49 @@
  * The Worker that hosts the TenantPipeline Durable Object for the harness, in its own isolate
  * (as in production, where callers never share the object's isolate — and where an eviction
  * restarts the object without touching its callers). It runs the production engine with the
- * synthetic job kinds registered and answers intents through `submitIntent`.
+ * synthetic job kinds AND the production preview kinds (integrate / rebuild / revert / sync-preview) over fake
+ * services (`./fake-preview-services.ts`), and answers intents through `submitIntent`.
  */
 import type { DurableObjectState } from "cloudflare:workers";
 import { TenantPipeline as DeployedTenantPipeline } from "../../src/lib/pipeline/tenant-pipeline";
-import { submitIntent, type TenantPipelineNamespace } from "../../src/lib/pipeline/intake";
+import { submitIntent, tenantPipeline, type TenantPipelineNamespace } from "../../src/lib/pipeline/intake";
+import { submitPreviewSync } from "../../src/lib/pipeline/preview-sync-intake";
 import { StaleExecutionError, type JobKindRegistry, type PipelineIntent } from "../../src/lib/pipeline/job-runner";
-import { abandonedContexts, syntheticJobKinds } from "./synthetic-job";
+import { abandonedContexts, syntheticJobKinds, syntheticSpans } from "./synthetic-job";
+import { registerPreviewJobKinds, type PreviewJobTiming } from "../../src/lib/pipeline/preview-jobs";
+import { configureFake, fakePreviewServices, readFakeState, type FakeFault, type FakeState } from "./fake-preview-services";
+
+/** The preview kinds' timing in the harness: the production shape, scaled to milliseconds. */
+export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
+  leaseMs: 1_000,
+  deadlineMs: 30_000,
+  maxAttemptsPerStep: 4,
+  retryBaseMs: 10,
+  defaultExpectedMs: 2_000,
+};
 
 export class TenantPipeline extends DeployedTenantPipeline {
   protected override jobKinds(ctx: DurableObjectState): JobKindRegistry {
-    return syntheticJobKinds(ctx);
+    return registerPreviewJobKinds(syntheticJobKinds(ctx), fakePreviewServices(ctx), HARNESS_PREVIEW_TIMING);
+  }
+
+  /** Harness-only: open candidates and arm faults for the fake preview services. */
+  async configureFake(config: {
+    open?: { changeId: string; headSha: string }[];
+    faults?: FakeFault[];
+    ancestry?: [string, string][];
+  }): Promise<void> {
+    configureFake(this.ctx, config);
+  }
+
+  /** Harness-only: execution spans of the cooperative-hang kind. */
+  async spans(jobId: string): Promise<{ started: number; ended: number | null }[]> {
+    return syntheticSpans(this.ctx, jobId);
+  }
+
+  /** Harness-only: what the fake forge, build and read model recorded. */
+  async fakeState(): Promise<FakeState> {
+    return readFakeState(this.ctx);
   }
 
   /** Harness-only reads of the synthetic step log. */
@@ -64,8 +96,28 @@ interface Env {
   TENANT_PIPELINE: TenantPipelineNamespace;
 }
 
+/**
+ * `/sync/<tenant>` answers a forwarded push the way `/api/preview/reconcile` does: submit the
+ * `sync-preview` intent and answer 202. `/sync-hold/<tenant>` submits, then holds the response open
+ * (a forward the caller kills mid-flight): the job must not depend on the request.
+ */
+async function syncRoute(request: Request, env: Env, tenant: string, hold: boolean): Promise<Response> {
+  const body = (await request.json()) as { repo: string; sha: string };
+  const result = await submitPreviewSync(
+    tenantPipeline(env.TENANT_PIPELINE, tenant),
+    tenant,
+    { repo: body.repo, sha: body.sha, source: "push" },
+    { actor: { type: "service", id: "harness-forward" }, admissionDecisionId: null },
+  );
+  if (!result.ok) return Response.json({ error: result.code }, { status: 400 });
+  if (hold) await new Promise((resolve) => setTimeout(resolve, 10_000));
+  return Response.json({ jobId: result.jobId, state: result.state, duplicate: result.duplicate }, { status: 202 });
+}
+
 export default {
   async fetch(request: Request, env: Env): Promise<Response> {
+    const sync = /^\/(sync|sync-hold)\/([^/]+)$/.exec(new URL(request.url).pathname);
+    if (request.method === "POST" && sync) return syncRoute(request, env, decodeURIComponent(sync[2]!), sync[1] === "sync-hold");
     const match = /^\/intents\/([^/]+)$/.exec(new URL(request.url).pathname);
     if (request.method !== "POST" || !match) return new Response("not found", { status: 404 });
     const response = await submitIntent(env.TENANT_PIPELINE, decodeURIComponent(match[1]!), (await request.json()) as PipelineIntent);

package/apps/storefront/test/pipeline/synthetic-job.ts

@@ -7,12 +7,14 @@
  * execution is logged to `synthetic_runs` (to prove resume-not-restart) and every effect to
  * `synthetic_effects` with INSERT OR IGNORE (the step is idempotent, as the engine requires).
  *
- * The intent carries a fault plan: throw / hang past the lease / evict the object, at a step
+ * The intent carries a fault plan: throw a transient fault / hang past the lease / evict the
+ * object / refuse (a developer-class refusal) / crash (a platform-class error), at a step
  * boundary (`before` the step's work, `after` it but before the engine records it) or `mid` s3
  * (after its first chunk's checkpoint). A fault fires once unless `persistent`; its firing is
  * persisted before it acts so it survives the eviction it causes.
  */
 import type { DurableObjectState } from "cloudflare:workers";
+import { PipelineRefusal } from "../../src/lib/pipeline/classify";
 import {
   InvalidJobInput,
   JobKindRegistry,
@@ -23,7 +25,16 @@ import {
   type StepOutcome,
 } from "../../src/lib/pipeline/job-runner";
 
-export type FaultMode = "throw" | "hang" | "evict";
+export type FaultMode = "throw" | "hang" | "evict" | "refuse" | "crash";
+
+/**
+ * The transient fault a `throw` raises: a typed connection reset, which `classify()` names
+ * `transient` — so the engine retries it, exactly as it retries a real network blip.
+ */
+export class SyntheticTransientFault extends Error {
+  override name = "SyntheticTransientFault";
+  readonly code = "ECONNRESET";
+}
 export interface Fault {
   at: "before" | "after" | "mid";
   step: number;
@@ -49,7 +60,7 @@ function parseInput(raw: unknown): SyntheticInput {
   if (!Array.isArray(faults)) throw new InvalidJobInput("faults must be an array");
   for (const f of faults as Fault[]) {
     if (!["before", "after", "mid"].includes(f.at)) throw new InvalidJobInput("fault.at is invalid");
-    if (!["throw", "hang", "evict"].includes(f.mode)) throw new InvalidJobInput("fault.mode is invalid");
+    if (!["throw", "hang", "evict", "refuse", "crash"].includes(f.mode)) throw new InvalidJobInput("fault.mode is invalid");
     if (!Number.isInteger(f.step) || f.step < 1 || f.step > 5) throw new InvalidJobInput("fault.step is invalid");
   }
   return { faults: faults as Fault[] };
@@ -81,7 +92,9 @@ function syntheticKind(state: DurableObjectState, kind: string, deadlineMs: numb
       index,
     );
     await state.storage.sync();
-    if (fault.mode === "throw") throw new Error(`synthetic fault ${at} s${step}`);
+    if (fault.mode === "throw") throw new SyntheticTransientFault(`synthetic fault ${at} s${step}`);
+    if (fault.mode === "refuse") throw new PipelineRefusal("build_refused", [`synthetic refusal ${at} s${step}`]);
+    if (fault.mode === "crash") throw new Error(`synthetic crash ${at} s${step}`);
     if (fault.mode === "evict") {
       state.abort(`synthetic eviction ${at} s${step}`);
       throw new Error("abort() returned");
@@ -148,9 +161,111 @@ function syntheticKind(state: DurableObjectState, kind: string, deadlineMs: numb
   };
 }
 
+// ── Engine-behaviour kinds ──────────────────────────────────────────────────
+
+/** s1 then an optional s2 that fails as the intent says (`crash` | `refuse` | `hang`). */
+export const SYNTHETIC_OPTIONAL_TAIL_KIND = "synthetic-optional-tail";
+/** One long step that works in chunks, checkpointing each, for longer than one execution may run. */
+export const SYNTHETIC_LONG_STEP_KIND = "synthetic-long-step";
+/** A step that hangs past its lease but honours its abort signal, then retries clean. */
+export const SYNTHETIC_COOPERATIVE_HANG_KIND = "synthetic-cooperative-hang";
+export const LONG_STEP_CHUNKS = 12;
+export const LONG_STEP_CHUNK_MS = 100;
+export const LONG_STEP_MAX_EXECUTION_MS = 450;
+
+function engineKinds(state: DurableObjectState): JobKindDefinition<{ mode?: string }>[] {
+  const { sql } = state.storage;
+  sql.exec(`CREATE TABLE IF NOT EXISTS synthetic_spans (seq INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT, started INTEGER, ended INTEGER)`);
+  const base = { leaseMs: SYNTHETIC_LEASE_MS, deadlineMs: SYNTHETIC_DEADLINE_MS, retryBaseMs: 10, defaultExpectedMs: 1_000 };
+  const parse = (raw: unknown) => (raw && typeof raw === "object" ? (raw as { mode?: string }) : {});
+  const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+  return [
+    {
+      kind: SYNTHETIC_OPTIONAL_TAIL_KIND,
+      parseInput: parse,
+      steps: [
+        { name: "s1", run: async () => ({ kind: "done" }) },
+        {
+          name: "s2",
+          optional: true,
+          run: async (ctx) => {
+            if (ctx.input.mode === "refuse") throw new PipelineRefusal("build_refused");
+            if (ctx.input.mode === "hang") await sleep(SYNTHETIC_LEASE_MS * 4);
+            throw new Error("optional tail crashed");
+          },
+        },
+      ],
+      ...base,
+      maxAttemptsPerStep: 2,
+    },
+    {
+      kind: SYNTHETIC_LONG_STEP_KIND,
+      parseInput: parse,
+      steps: [
+        {
+          name: "long",
+          run: async (ctx) => {
+            let next = (ctx.checkpoint as { next?: number } | null)?.next ?? 0;
+            while (next < LONG_STEP_CHUNKS) {
+              if (ctx.signal.aborted) throw new Error("aborted");
+              await sleep(LONG_STEP_CHUNK_MS);
+              next += 1;
+              ctx.saveCheckpoint({ next });
+              await ctx.renewLease();
+            }
+            return { kind: "done" };
+          },
+        },
+      ],
+      ...base,
+      // One attempt only: reaching the execution cap after checkpointing must not spend it.
+      maxAttemptsPerStep: 1,
+      maxExecutionMs: LONG_STEP_MAX_EXECUTION_MS,
+    },
+    {
+      kind: SYNTHETIC_COOPERATIVE_HANG_KIND,
+      parseInput: parse,
+      steps: [
+        {
+          name: "hang",
+          run: async (ctx) => {
+            const started = Date.now();
+            const seq = sql
+              .exec<{ seq: number }>(`INSERT INTO synthetic_spans (job_id, started) VALUES (?, ?) RETURNING seq`, ctx.jobId, started)
+              .one().seq;
+            const first = sql.exec<{ n: number }>(`SELECT COUNT(*) AS n FROM synthetic_spans WHERE job_id = ?`, ctx.jobId).one().n === 1;
+            try {
+              if (first) {
+                // Wait for the abort, then take a while to wind down (within one further lease).
+                await new Promise<void>((resolve) => ctx.signal.addEventListener("abort", () => resolve(), { once: true }));
+                await sleep(SYNTHETIC_LEASE_MS / 2);
+                throw new Error("stopped after abort");
+              }
+              return { kind: "done" };
+            } finally {
+              sql.exec(`UPDATE synthetic_spans SET ended = ? WHERE seq = ?`, Date.now(), seq);
+            }
+          },
+        },
+      ],
+      ...base,
+      maxAttemptsPerStep: 5,
+    },
+  ];
+}
+
 export function syntheticJobKinds(state: DurableObjectState): JobKindRegistry {
   harnessTables(state);
-  return new JobKindRegistry()
+  const registry = new JobKindRegistry()
     .register(syntheticKind(state, SYNTHETIC_KIND, SYNTHETIC_DEADLINE_MS))
     .register(syntheticKind(state, SYNTHETIC_SHORT_DEADLINE_KIND, SHORT_DEADLINE_MS));
+  for (const kind of engineKinds(state)) registry.register(kind as JobKindDefinition<unknown> as never);
+  return registry;
+}
+
+/** Execution spans of the cooperative-hang kind, for overlap checks. */
+export function syntheticSpans(state: DurableObjectState, jobId: string): { started: number; ended: number | null }[] {
+  return state.storage.sql
+    .exec<{ started: number; ended: number | null }>(`SELECT started, ended FROM synthetic_spans WHERE job_id = ? ORDER BY seq`, jobId)
+    .toArray();
 }

package/apps/storefront/vitest.workers.config.cts

@@ -28,6 +28,8 @@ const hostModule = esbuild.buildSync({
 const pipelineBinding = { TENANT_PIPELINE: { className: "TenantPipeline", scriptName: HOST } };
 
 module.exports = defineWorkersConfig({
+  // The source tree's `@/` import alias (tsconfig `paths`), for modules the tests import directly.
+  resolve: { alias: { "@": path.join(__dirname, "src") } },
   test: {
     name: "pipeline",
     include: ["test/pipeline/**/*.test.ts"],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tokenoftrust/storefront-runner",
-  "version": "2.4.1",
+  "version": "2.4.3",
   "license": "SEE LICENSE IN LICENSE",
   "description": "World-shareable storefront runner: multi-tenant renderer on Astro/Cloudflare. No control plane.",
   "packageManager": "pnpm@11.9.0",