@tokenoftrust/storefront-runner 4.12.0 → 4.12.1

package/apps/storefront/test/pipeline/matrix-faults.ts

@@ -39,10 +39,19 @@ export interface MatrixFault {
   mode: MatrixMode;
   /** `effect` mode: the effect after which the object is evicted. */
   effect?: string | null;
+  /** Test-only constructor delay after this eviction, to reproduce a late alarm. */
+  resumeDelayMs?: number;
 }
 
 /** What the object looked like at the instant a matrix fault evicted it. */
 export interface MatrixKill {
+  instanceId: string;
+  execution: number;
+  at: number;
+  stepStartSeq: number;
+  transitionSeq: number;
+  effectSeq: number;
+  resumeNotBefore: number;
   jobId: string;
   kind: string;
   step: string;
@@ -63,6 +72,10 @@ export interface MatrixKill {
 }
 
 export interface MatrixEffect {
+  seq: number;
+  at: number;
+  instanceId: string;
+  execution: number;
   jobId: string;
   kind: string;
   step: string;
@@ -102,6 +115,7 @@ export function previewServicesWithEffects(services: PreviewJobServices, landed:
 }
 
 interface Running {
+  execution: number;
   jobId: string;
   kind: string;
   step: string;
@@ -110,9 +124,9 @@ interface Running {
 function tables(state: DurableObjectState): void {
   const { sql } = state.storage;
   sql.exec(
-    `CREATE TABLE IF NOT EXISTS matrix_faults (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT, step TEXT, mode TEXT, effect TEXT, remaining INTEGER)`,
+    `CREATE TABLE IF NOT EXISTS matrix_faults (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT, step TEXT, mode TEXT, effect TEXT, remaining INTEGER, resume_delay INTEGER)`,
   );
-  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_effects (seq INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT, kind TEXT, step TEXT, effect TEXT)`);
+  sql.exec(`CREATE TABLE IF NOT EXISTS matrix_effects (seq INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT, kind TEXT, step TEXT, effect TEXT, at INTEGER, instance_id TEXT, execution INTEGER)`);
   sql.exec(`CREATE TABLE IF NOT EXISTS matrix_execs (job_id TEXT, step TEXT, n INTEGER, PRIMARY KEY (job_id, step))`);
   sql.exec(`CREATE TABLE IF NOT EXISTS matrix_kills (seq INTEGER PRIMARY KEY AUTOINCREMENT, kill TEXT)`);
   sql.exec(`CREATE TABLE IF NOT EXISTS matrix_holds (job_id TEXT, step TEXT, released INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (job_id, step))`);
@@ -122,6 +136,7 @@ function tables(state: DurableObjectState): void {
 const HOLD_LIMIT_MS = 30_000;
 
 export class MatrixFaults {
+  private readonly instanceId = crypto.randomUUID();
   private running: Running | null = null;
 
   constructor(
@@ -144,7 +159,7 @@ export class MatrixFaults {
           ? {
               onTerminal: async (info: Parameters<NonNullable<JobKindDefinition<unknown>["onTerminal"]>>[0]) => {
                 const previous = this.running;
-                this.running = { jobId: info.jobId, kind: name, step: "on-terminal" };
+                this.running = { jobId: info.jobId, kind: name, step: "on-terminal", execution: 0 };
                 try {
                   await onTerminal(info);
                 } finally {
@@ -171,7 +186,7 @@ export class MatrixFaults {
             step.name,
           )
           .one().n;
-        this.running = { jobId: ctx.jobId, kind, step: step.name };
+        this.running = { jobId: ctx.jobId, kind, step: step.name, execution: n };
         try {
           if (n === 1) await this.maybeHold(kind, step.name, ctx);
           if (n === 1) await this.maybeKill("before", null);
@@ -190,22 +205,24 @@ export class MatrixFaults {
   readonly landed: Landed = async (effect) => {
     const running = this.running;
     this.state.storage.sql.exec(
-      `INSERT INTO matrix_effects (job_id, kind, step, effect) VALUES (?, ?, ?, ?)`,
+      `INSERT INTO matrix_effects (job_id, kind, step, effect, at, instance_id, execution) VALUES (?, ?, ?, ?, ?, ?, ?)`,
       running?.jobId ?? "",
       running?.kind ?? "",
       running?.step ?? "",
       effect,
+      Date.now(), this.instanceId, running?.execution ?? 0,
     );
     if (running) await this.maybeKill("effect", effect);
   };
 
   arm(fault: MatrixFault): void {
     this.state.storage.sql.exec(
-      `INSERT INTO matrix_faults (kind, step, mode, effect, remaining) VALUES (?, ?, ?, ?, 1)`,
+      `INSERT INTO matrix_faults (kind, step, mode, effect, remaining, resume_delay) VALUES (?, ?, ?, ?, 1, ?)`,
       fault.kind,
       fault.step,
       fault.mode,
       fault.effect ?? null,
+      Math.max(0, Math.min(5_000, fault.resumeDelayMs ?? 0)),
     );
   }
 
@@ -249,12 +266,12 @@ export class MatrixFaults {
   effects(jobId?: string): MatrixEffect[] {
     const { sql } = this.state.storage;
     const rows = jobId
-      ? sql.exec<{ job_id: string; kind: string; step: string; effect: string }>(
-          `SELECT job_id, kind, step, effect FROM matrix_effects WHERE job_id = ? ORDER BY seq`,
+      ? sql.exec<{ seq: number; at: number; instance_id: string; execution: number; job_id: string; kind: string; step: string; effect: string }>(
+          `SELECT * FROM matrix_effects WHERE job_id = ? ORDER BY seq`,
           jobId,
         )
-      : sql.exec<{ job_id: string; kind: string; step: string; effect: string }>(`SELECT job_id, kind, step, effect FROM matrix_effects ORDER BY seq`);
-    return rows.toArray().map((r) => ({ jobId: r.job_id, kind: r.kind, step: r.step, effect: r.effect }));
+      : sql.exec<{ seq: number; at: number; instance_id: string; execution: number; job_id: string; kind: string; step: string; effect: string }>(`SELECT * FROM matrix_effects ORDER BY seq`);
+    return rows.toArray().map((r) => ({ jobId: r.job_id, kind: r.kind, step: r.step, effect: r.effect, seq: r.seq, at: r.at, instanceId: r.instance_id, execution: r.execution }));
   }
 
   execs(jobId: string): { step: string; n: number }[] {
@@ -286,8 +303,8 @@ export class MatrixFaults {
     if (!running) return;
     const { sql } = this.state.storage;
     const fault = sql
-      .exec<{ id: number }>(
-        `SELECT id FROM matrix_faults WHERE kind = ? AND step = ? AND mode = ? AND IFNULL(effect, '') = ? AND remaining > 0 ORDER BY id LIMIT 1`,
+      .exec<{ id: number; resume_delay: number }>(
+        `SELECT id, resume_delay FROM matrix_faults WHERE kind = ? AND step = ? AND mode = ? AND IFNULL(effect, '') = ? AND remaining > 0 ORDER BY id LIMIT 1`,
         running.kind,
         running.step,
         mode,
@@ -297,6 +314,7 @@ export class MatrixFaults {
     if (!fault) return;
     sql.exec(`UPDATE matrix_faults SET remaining = remaining - 1 WHERE id = ?`, fault.id);
     const kill = await this.snapshot(running, mode, effect);
+    kill.resumeNotBefore = fault.resume_delay ? Date.now() + fault.resume_delay : 0;
     sql.exec(`INSERT INTO matrix_kills (kill) VALUES (?)`, JSON.stringify(kill));
     // Everything written so far — the effect itself included — is durable, exactly as a real crash
     // after it would leave it.
@@ -327,6 +345,10 @@ export class MatrixFaults {
       }
     }
     return {
+      instanceId: this.instanceId, execution: running.execution, at: Date.now(), resumeNotBefore: 0,
+      stepStartSeq: sql.exec<{ seq: number }>(`SELECT MAX(seq) seq FROM transitions WHERE job_id = ? AND step = ? AND note LIKE 'step-start%'`, running.jobId, running.step).one().seq,
+      transitionSeq: sql.exec<{ seq: number }>(`SELECT MAX(seq) seq FROM transitions`).one().seq,
+      effectSeq: sql.exec<{ seq: number }>(`SELECT MAX(seq) seq FROM matrix_effects`).one().seq ?? 0,
       jobId: running.jobId,
       kind: running.kind,
       step: running.step,

package/apps/storefront/test/pipeline/pipeline-host.ts

@@ -80,6 +80,13 @@ function harnessSeedServices(
 const matrices = new WeakMap<DurableObjectState, { faults: MatrixFaults; registry: JobKindRegistry }>();
 
 export class TenantPipeline extends DeployedTenantPipeline {
+  constructor(ctx: DurableObjectState, env: unknown) {
+    super(ctx, env);
+    const resumeAt = matrices.get(ctx)?.faults.kills().at(-1)?.resumeNotBefore ?? 0;
+    if (resumeAt > Date.now()) void ctx.blockConcurrencyWhile(async () => {
+      await new Promise((resolve) => setTimeout(resolve, resumeAt - Date.now()));
+    });
+  }
   private commercePaused = false;
   private commerceResume: (() => void) | null = null;
   private commerceOutcome: string = "complete";

package/apps/storefront/test/pipeline/publish-kinds.ts

@@ -57,7 +57,7 @@ export const PUBLISH_HARNESS_TIMING: PublishTiming = {
 export const PUBLISH_HARNESS_TUNING: PublishKindTuning = {
   leaseMs: 3_000,
   retryBaseMs: 20,
-  deadlineMs: { synchronous: 30_000, asynchronous: 30_000 },
+  deadlineMs: { synchronous: 60_000, asynchronous: 60_000 },
   hotfixExtraMs: 0,
   compensationRetryBaseMs: 300,
 };

package/apps/storefront/test/pipeline/synthetic-job.ts

@@ -172,6 +172,10 @@ export const SYNTHETIC_COOPERATIVE_HANG_KIND = "synthetic-cooperative-hang";
 export const LONG_STEP_CHUNKS = 12;
 export const LONG_STEP_CHUNK_MS = 100;
 export const LONG_STEP_MAX_EXECUTION_MS = 450;
+export const SYNTHETIC_STEP_CLOCK_KIND = "synthetic-step-clock";
+export const STEP_CLOCK_BASE_MS = 1_000;
+export const STEP_CLOCK_CAPTURE_MS = 1_500;
+export const STEP_CLOCK_LIMIT_MS = 4_000;
 
 function engineKinds(state: DurableObjectState): JobKindDefinition<{ mode?: string }>[] {
   const { sql } = state.storage;
@@ -180,6 +184,34 @@ function engineKinds(state: DurableObjectState): JobKindDefinition<{ mode?: stri
   const parse = (raw: unknown) => (raw && typeof raw === "object" ? (raw as { mode?: string }) : {});
   const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
   return [
+    {
+      ...base,
+      kind: SYNTHETIC_STEP_CLOCK_KIND,
+      parseInput: parse,
+      deadlineMs: STEP_CLOCK_BASE_MS,
+      leaseMs: 10_000,
+      maxAttemptsPerStep: 3,
+      steps: [
+        { name: "capture", deadlineMs: STEP_CLOCK_LIMIT_MS, run: async (ctx) => {
+          const deadline = sql.exec<{ deadline: number }>(`SELECT deadline FROM jobs WHERE job_id = ?`, ctx.jobId).one().deadline;
+          ctx.setProduces({ captureStartedAt: ctx.now(), captureDeadline: deadline });
+          if (ctx.input.mode === "evict" && !ctx.checkpoint) {
+            ctx.saveCheckpoint({ evicted: true });
+            await ctx.flush();
+            state.abort("synthetic eviction during bounded step clock");
+          }
+          if (ctx.input.mode === "hang") await new Promise<void>((resolve) => ctx.signal.addEventListener("abort", () => resolve(), { once: true }));
+          else await sleep(STEP_CLOCK_CAPTURE_MS);
+          return { kind: "done" as const };
+        } },
+        { name: "after", run: async (ctx) => {
+          const deadline = sql.exec<{ deadline: number }>(`SELECT deadline FROM jobs WHERE job_id = ?`, ctx.jobId).one().deadline;
+          ctx.setProduces({ afterStartedAt: ctx.now(), afterDeadline: deadline });
+          await new Promise<void>((resolve) => ctx.signal.addEventListener("abort", () => resolve(), { once: true }));
+          return { kind: "done" as const };
+        } },
+      ],
+    },
     {
       kind: SYNTHETIC_OPTIONAL_TAIL_KIND,
       parseInput: parse,

package/apps/storefront/test/pipeline-faults/matrix-lib.ts

@@ -561,9 +561,15 @@ export function chainViolations(log: TransitionView[]): string[] {
 }
 
 /** Across every job of the tenant: between a step's start and its outcome, no other step starts. */
-export function overlapViolations(all: TransitionView[]): string[] {
+export function overlapViolations(all: TransitionView[], kills: readonly MatrixKill[] = [], effects: readonly MatrixEffect[] = []): string[] {
   let open: TransitionView | null = null;
   for (const t of all) {
+    // A recorded abort ends only its exact execution; any later effect keeps the overlap visible.
+    if (open && kills.some((kill) => ["before", "after", "resume", "effect"].includes(kill.mode) && !!kill.instanceId && kill.execution > 0 &&
+      kill.jobId === open!.jobId && kill.step === open!.step && kill.stepStartSeq === open!.seq &&
+      kill.transitionSeq >= open!.seq && kill.transitionSeq < t.seq &&
+      !effects.some((effect) => effect.jobId === kill.jobId && effect.step === kill.step &&
+        effect.instanceId === kill.instanceId && effect.execution === kill.execution && effect.seq > kill.effectSeq))) open = null;
     if (t.note?.startsWith("step-start")) {
       if (open && open.jobId !== t.jobId) return [`step ${t.step} of ${t.jobId} started while ${open.step} of ${open.jobId} was running`];
       open = t;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tokenoftrust/storefront-runner",
-  "version": "4.12.0",
+  "version": "4.12.1",
   "license": "SEE LICENSE IN LICENSE",
   "description": "World-shareable storefront runner: multi-tenant renderer on Astro/Cloudflare. No control plane.",
   "packageManager": "pnpm@11.9.0",