@tokenoftrust/storefront-runner 2.2.126 → 2.3.1

package/apps/storefront/test/pipeline/synthetic-job.ts

@@ -0,0 +1,156 @@
+/**
+ * A synthetic 5-step job kind for the TenantPipeline fault harness. Test-only: the deployed
+ * registry (`src/lib/pipeline/job-kinds.ts`) never contains it.
+ *
+ * Steps s1..s5. s3 works in three chunks and checkpoints after each one. s2 records a commit
+ * fact, s4 evidence, s5 the live state, so retried fact writes are exercised too. Every step
+ * execution is logged to `synthetic_runs` (to prove resume-not-restart) and every effect to
+ * `synthetic_effects` with INSERT OR IGNORE (the step is idempotent, as the engine requires).
+ *
+ * The intent carries a fault plan: throw / hang past the lease / evict the object, at a step
+ * boundary (`before` the step's work, `after` it but before the engine records it) or `mid` s3
+ * (after its first chunk's checkpoint). A fault fires once unless `persistent`; its firing is
+ * persisted before it acts so it survives the eviction it causes.
+ */
+import type { DurableObjectState } from "cloudflare:workers";
+import {
+  InvalidJobInput,
+  JobKindRegistry,
+  StaleExecutionError,
+  type JobKindDefinition,
+  type StepContext,
+  type StepDefinition,
+  type StepOutcome,
+} from "../../src/lib/pipeline/job-runner";
+
+export type FaultMode = "throw" | "hang" | "evict";
+export interface Fault {
+  at: "before" | "after" | "mid";
+  step: number;
+  mode: FaultMode;
+  persistent?: boolean;
+}
+export interface SyntheticInput {
+  faults: Fault[];
+}
+
+export const SYNTHETIC_KIND = "synthetic-5-step";
+export const SYNTHETIC_SHORT_DEADLINE_KIND = "synthetic-5-step-short-deadline";
+export const SYNTHETIC_LEASE_MS = 250;
+export const SYNTHETIC_DEADLINE_MS = 30_000;
+export const SHORT_DEADLINE_MS = 1_500;
+export const S3_CHUNKS = 3;
+
+/** Executions abandoned by a lease expiry, kept so a test can try writing through them. */
+export const abandonedContexts: StepContext<SyntheticInput>[] = [];
+
+function parseInput(raw: unknown): SyntheticInput {
+  const faults = (raw as { faults?: unknown } | null)?.faults ?? [];
+  if (!Array.isArray(faults)) throw new InvalidJobInput("faults must be an array");
+  for (const f of faults as Fault[]) {
+    if (!["before", "after", "mid"].includes(f.at)) throw new InvalidJobInput("fault.at is invalid");
+    if (!["throw", "hang", "evict"].includes(f.mode)) throw new InvalidJobInput("fault.mode is invalid");
+    if (!Number.isInteger(f.step) || f.step < 1 || f.step > 5) throw new InvalidJobInput("fault.step is invalid");
+  }
+  return { faults: faults as Fault[] };
+}
+
+function harnessTables(state: DurableObjectState): void {
+  const { sql } = state.storage;
+  sql.exec(`CREATE TABLE IF NOT EXISTS synthetic_runs (seq INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT, step INTEGER, chunk INTEGER, attempt INTEGER)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS synthetic_effects (job_id TEXT, step INTEGER, chunk INTEGER, PRIMARY KEY (job_id, step, chunk))`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS synthetic_faults_fired (job_id TEXT, fault INTEGER, n INTEGER, PRIMARY KEY (job_id, fault))`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS synthetic_zombie_writes (job_id TEXT, outcome TEXT)`);
+}
+
+function syntheticKind(state: DurableObjectState, kind: string, deadlineMs: number): JobKindDefinition<SyntheticInput> {
+  const { sql } = state.storage;
+
+  async function maybeFault(ctx: StepContext<SyntheticInput>, at: Fault["at"], step: number): Promise<void> {
+    const index = ctx.input.faults.findIndex((f) => f.at === at && f.step === step);
+    if (index < 0) return;
+    const fault = ctx.input.faults[index]!;
+    const fired = sql
+      .exec<{ n: number }>(`SELECT n FROM synthetic_faults_fired WHERE job_id = ? AND fault = ?`, ctx.jobId, index)
+      .toArray()[0];
+    if (fired && !fault.persistent) return;
+    sql.exec(
+      `INSERT INTO synthetic_faults_fired (job_id, fault, n) VALUES (?, ?, 1)
+       ON CONFLICT(job_id, fault) DO UPDATE SET n = n + 1`,
+      ctx.jobId,
+      index,
+    );
+    await state.storage.sync();
+    if (fault.mode === "throw") throw new Error(`synthetic fault ${at} s${step}`);
+    if (fault.mode === "evict") {
+      state.abort(`synthetic eviction ${at} s${step}`);
+      throw new Error("abort() returned");
+    }
+    abandonedContexts.push(ctx);
+    await new Promise((resolve) => setTimeout(resolve, SYNTHETIC_LEASE_MS * 8));
+    try {
+      ctx.saveCheckpoint({ zombie: true });
+      sql.exec(`INSERT INTO synthetic_zombie_writes (job_id, outcome) VALUES (?, 'accepted')`, ctx.jobId);
+    } catch (err) {
+      sql.exec(
+        `INSERT INTO synthetic_zombie_writes (job_id, outcome) VALUES (?, ?)`,
+        ctx.jobId,
+        err instanceof StaleExecutionError ? "refused" : "error",
+      );
+    }
+    throw new Error("hung execution woke up");
+  }
+
+  function logRun(ctx: StepContext<SyntheticInput>, step: number, chunk: number): void {
+    sql.exec(`INSERT INTO synthetic_runs (job_id, step, chunk, attempt) VALUES (?, ?, ?, ?)`, ctx.jobId, step, chunk, ctx.attempt);
+    sql.exec(`INSERT OR IGNORE INTO synthetic_effects (job_id, step, chunk) VALUES (?, ?, ?)`, ctx.jobId, step, chunk);
+  }
+
+  const plainStep = (step: number, fact?: (ctx: StepContext<SyntheticInput>) => void): StepDefinition<SyntheticInput> => ({
+    name: `s${step}`,
+    async run(ctx): Promise<StepOutcome> {
+      await maybeFault(ctx, "before", step);
+      logRun(ctx, step, 0);
+      fact?.(ctx);
+      await maybeFault(ctx, "after", step);
+      return { kind: "done" };
+    },
+  });
+
+  const chunkedStep: StepDefinition<SyntheticInput> = {
+    name: "s3",
+    async run(ctx): Promise<StepOutcome> {
+      const next = (ctx.checkpoint as { next?: number } | null)?.next ?? 0;
+      if (next === 0) await maybeFault(ctx, "before", 3);
+      if (next === 1) await maybeFault(ctx, "mid", 3);
+      logRun(ctx, 3, next);
+      if (next < S3_CHUNKS - 1) return { kind: "continue", checkpoint: { next: next + 1 } };
+      await maybeFault(ctx, "after", 3);
+      return { kind: "done" };
+    },
+  };
+
+  return {
+    kind,
+    parseInput,
+    steps: [
+      plainStep(1),
+      plainStep(2, (ctx) => ctx.recordCommit(`sha-${ctx.jobId}`, [{ pr: 2, changeId: "c2", headSha: "h2" }])),
+      chunkedStep,
+      plainStep(4, (ctx) => ctx.recordEvidence(`sha-${ctx.jobId}`, `digest-${ctx.jobId}`, true)),
+      plainStep(5, (ctx) => ctx.recordLive("worker-r2-pointer", `sha-${ctx.jobId}`, `rcpt-${ctx.jobId}`, null)),
+    ],
+    leaseMs: SYNTHETIC_LEASE_MS,
+    deadlineMs,
+    maxAttemptsPerStep: kind === SYNTHETIC_SHORT_DEADLINE_KIND ? 1_000 : 5,
+    retryBaseMs: 10,
+    defaultExpectedMs: 2_000,
+  };
+}
+
+export function syntheticJobKinds(state: DurableObjectState): JobKindRegistry {
+  harnessTables(state);
+  return new JobKindRegistry()
+    .register(syntheticKind(state, SYNTHETIC_KIND, SYNTHETIC_DEADLINE_MS))
+    .register(syntheticKind(state, SYNTHETIC_SHORT_DEADLINE_KIND, SHORT_DEADLINE_MS));
+}

package/apps/storefront/vitest.workers.config.cts

@@ -0,0 +1,67 @@
+// The pipeline harness: tests that run INSIDE workerd (@cloudflare/vitest-pool-workers), so the
+// TenantPipeline Durable Object is exercised with real SQLite storage, real alarms and real
+// eviction (`ctx.abort()`). The Node unit suite (vitest.config.cts) never picks these up.
+//
+// The object lives in its own Worker, `pipeline-host`, not in the test runner's isolate: an
+// eviction must restart the object without touching its callers, as in production. Auxiliary
+// Workers take plain modules, so the host is bundled here, in memory, with esbuild.
+// .cts for the same reason as vitest.config.cts: compiled in memory, no sibling file written.
+process.env.VITE_CJS_IGNORE_WARNING = "1";
+const { defineWorkersConfig } = require("@cloudflare/vitest-pool-workers/config");
+const esbuild = require("esbuild");
+const path = require("node:path");
+
+const compatibilityDate = "2024-11-01";
+const compatibilityFlags = ["nodejs_compat"];
+const HOST = "pipeline-host";
+
+const hostModule = esbuild.buildSync({
+  entryPoints: [path.join(__dirname, "test/pipeline/pipeline-host.ts")],
+  bundle: true,
+  format: "esm",
+  platform: "neutral",
+  target: "es2022",
+  external: ["cloudflare:*"],
+  write: false,
+}).outputFiles[0].text;
+
+const pipelineBinding = { TENANT_PIPELINE: { className: "TenantPipeline", scriptName: HOST } };
+
+module.exports = defineWorkersConfig({
+  test: {
+    name: "pipeline",
+    include: ["test/pipeline/**/*.test.ts"],
+    cache: false,
+    testTimeout: 60_000,
+    hookTimeout: 60_000,
+    poolOptions: {
+      workers: {
+        isolatedStorage: false,
+        singleWorker: true,
+        miniflare: {
+          compatibilityDate,
+          compatibilityFlags,
+          durableObjects: pipelineBinding,
+          serviceBindings: { PIPELINE_HOST: HOST, INTAKE_B: "pipeline-intake-b" },
+          workers: [
+            {
+              name: HOST,
+              modules: [{ type: "ESModule", path: "pipeline-host.mjs", contents: hostModule }],
+              compatibilityDate,
+              compatibilityFlags,
+              durableObjects: { TENANT_PIPELINE: { className: "TenantPipeline", useSQLite: true } },
+            },
+            {
+              name: "pipeline-intake-b",
+              modules: true,
+              scriptPath: path.join(__dirname, "test/pipeline/intake-worker.mjs"),
+              compatibilityDate,
+              compatibilityFlags,
+              durableObjects: pipelineBinding,
+            },
+          ],
+        },
+      },
+    },
+  },
+});