@cosmicdrift/kumiko-framework 0.327.0 → 0.328.1

package/src/pipeline/__tests__/event-dispatcher-multi-instance.integration.test.ts

@@ -1,424 +0,0 @@
-// E.5 — Multi-instance claims. The whole reason the dispatcher was designed
-// around SELECT FOR UPDATE SKIP LOCKED is that in production there are N
-// dispatcher processes, not 1. If two dispatchers try to advance the same
-// consumer in parallel, SKIP LOCKED must guarantee: **exactly one** drives
-// the pass, the other no-ops, zero duplicate delivery.
-//
-// The existing event-dispatcher.integration.ts runs everything single-
-// instance. These tests pin the cross-process claims:
-//
-//   1. Two dispatchers on the same DB + same consumer name: handler is
-//      called exactly once per event (no duplicate delivery).
-//   2. Two dispatchers with different consumer names: both progress
-//      independently — one slow consumer doesn't starve the other.
-//   3. A consumer joining with 2000 events in the backlog catches up
-//      across multiple passes without replaying events or exploding.
-//
-// Note: these tests assert on behaviour that is not observable in
-// single-instance runs. Any regression in the locking strategy shows up
-// here first.
-
-import { afterEach, beforeAll, describe, expect, test } from "bun:test";
-import { createEventStoreExecutor } from "../../db/event-store-executor";
-import { insertMany } from "../../db/query";
-import { createTenantDb, type TenantDb } from "../../db/tenant-db";
-import { defineFeature } from "../../engine";
-import { eventsTable, type StoredEvent } from "../../event-store";
-import {
-  createEventDispatcher,
-  type EventConsumer,
-  type EventDispatcher,
-  getConsumerState,
-} from "../../pipeline";
-import {
-  resetEventStore,
-  setupTestStack,
-  type TestStack,
-  TestUsers,
-  unsafeCreateEntityTable,
-} from "../../stack";
-import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
-import { generateId } from "../../utils";
-
-// --- Fixture ---
-
-const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
-  entityName: "widget",
-});
-
-// A trivial feature — the dispatchers built by the tests use the same
-// consumer name ("multi:consumer:echo") via direct createEventDispatcher
-// calls (no r.multiStreamProjection registration on this stack, since the
-// test-stack would then auto-wire a subscriber we don't want in the
-// multi-instance setup).
-const multiFeature = defineFeature("multi", (r) => {
-  r.entity("widget", sharedWidgetEntity);
-});
-
-const admin = TestUsers.admin;
-let stack: TestStack;
-let tdb: TenantDb;
-
-beforeAll(async () => {
-  stack = await setupTestStack({
-    features: [multiFeature],
-    systemHooks: [],
-  });
-  await unsafeCreateEntityTable(stack.db, sharedWidgetEntity, "widget");
-  tdb = createTenantDb(stack.db, admin.tenantId);
-});
-
-afterEach(async () => {
-  await resetEventStore(stack, ["read_widgets"]);
-});
-
-async function appendWidget(name: string): Promise<void> {
-  await executor.create({ name }, admin, tdb);
-}
-
-// Bulk-seed N widget.created events directly into the events table.
-// Used by the backlog test where the seed phase would otherwise dominate
-// runtime (2000 sequential executor.create = 2000 DB round-trips).
-// The dispatcher only reads from events — bypassing the projections-table
-// write is safe here; we're testing cursor catch-up, not the executor.
-async function bulkSeedWidgetCreated(count: number, namePrefix: string): Promise<void> {
-  const rows = Array.from({ length: count }, (_, i) => ({
-    aggregateId: generateId(),
-    aggregateType: "widget",
-    tenantId: admin.tenantId,
-    version: 1,
-    type: "widget.created",
-    payload: { name: `${namePrefix}${i}` },
-    metadata: { userId: admin.id },
-    createdBy: admin.id,
-  }));
-  await insertMany(stack.db, eventsTable, rows);
-}
-
-function buildDispatcherWith(consumers: readonly EventConsumer[]): EventDispatcher {
-  return createEventDispatcher({
-    db: stack.db,
-    consumers,
-    context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
-    // Tight batch + poll so the test doesn't hinge on timing in .start();
-    // we drive everything through runOnce() for determinism.
-    batchSize: 200,
-    pollIntervalMs: 5000,
-  });
-}
-
-// --- Tests ---
-
-describe("E.5 — SKIP LOCKED: exactly-once delivery across dispatchers", () => {
-  test("two dispatchers, same consumer name: each event delivered exactly once", async () => {
-    // Shared name — both dispatchers race for the same row in
-    // kumiko_event_consumers. SKIP LOCKED must ensure only one wins.
-    const name = "multi:consumer:echo-same";
-    const seen: StoredEvent[] = [];
-
-    // Two consumers with the SAME name but distinct capture sides —
-    // mimics two different processes running the same subscriber code.
-    const consumerA: EventConsumer = {
-      name,
-      handler: async (event) => {
-        seen.push(event);
-      },
-    };
-    const consumerB: EventConsumer = { ...consumerA };
-
-    const dispA = buildDispatcherWith([consumerA]);
-    const dispB = buildDispatcherWith([consumerB]);
-    // Strict pre-reg: both dispatchers share the same consumer name so
-    // ON-CONFLICT-DO-NOTHING collapses to a single row — which is exactly
-    // the shared state the SKIP-LOCKED race is about.
-    await dispA.ensureRegistered();
-    await dispB.ensureRegistered();
-
-    // Seed a known, large-enough batch.
-    const count = 30;
-    for (let i = 0; i < count; i++) {
-      await appendWidget(`event-${i}`);
-    }
-
-    // Race both dispatchers. One will acquire the lock; the other's
-    // SELECT FOR UPDATE SKIP LOCKED returns nothing and it bails out of
-    // this pass. Run a handful of passes so stragglers (if any) get
-    // another shot.
-    for (let pass = 0; pass < 5; pass++) {
-      await Promise.all([dispA.runOnce(), dispB.runOnce()]);
-    }
-
-    // Each of the 30 events should appear exactly once across the
-    // combined `seen` array. Duplicate delivery would show up as >30.
-    expect(seen).toHaveLength(count);
-    const names = seen.map((e) => e.payload["name"]).sort();
-    const expected = Array.from({ length: count }, (_, i) => `event-${i}`).sort();
-    expect(names).toEqual(expected);
-
-    const finalState = await getConsumerState(stack.db, name);
-    expect(finalState?.lastProcessedEventId).toBe(BigInt(count));
-    expect(finalState?.status).toBe("idle");
-  });
-
-  test("different consumer names: one slow consumer does not starve the other", async () => {
-    // Two independent consumer rows, both driven on the same DB via two
-    // dispatcher instances. A slow handler on one side must not block the
-    // fast side.
-    const fastName = "multi:consumer:fast";
-    const slowName = "multi:consumer:slow";
-    const fastSeen: StoredEvent[] = [];
-    const slowSeen: StoredEvent[] = [];
-
-    const fast: EventConsumer = {
-      name: fastName,
-      handler: async (event) => {
-        fastSeen.push(event);
-      },
-    };
-    const slow: EventConsumer = {
-      name: slowName,
-      handler: async (event) => {
-        await new Promise((r) => setTimeout(r, 30));
-        slowSeen.push(event);
-      },
-    };
-
-    const dispA = buildDispatcherWith([fast]);
-    const dispB = buildDispatcherWith([slow]);
-    await dispA.ensureRegistered();
-    await dispB.ensureRegistered();
-
-    const count = 10;
-    for (let i = 0; i < count; i++) {
-      await appendWidget(`concurrent-${i}`);
-    }
-
-    // A tight race: fast dispatcher finishes its pass quickly; slow
-    // dispatcher is still processing. Neither should see the other's
-    // events (different consumer names = different rows, different
-    // cursors).
-    await Promise.all([dispA.runOnce(), dispB.runOnce()]);
-
-    expect(fastSeen).toHaveLength(count);
-    expect(slowSeen).toHaveLength(count);
-
-    const fastState = await getConsumerState(stack.db, fastName);
-    const slowState = await getConsumerState(stack.db, slowName);
-    expect(fastState?.lastProcessedEventId).toBe(BigInt(count));
-    expect(slowState?.lastProcessedEventId).toBe(BigInt(count));
-  });
-});
-
-describe("E.5 — cursor-lag catch-up", () => {
-  test("a consumer joining with a 500-event backlog catches up across multiple passes", async () => {
-    const name = "multi:consumer:late-joiner";
-    const seen: StoredEvent[] = [];
-    const consumer: EventConsumer = {
-      name,
-      handler: async (event) => {
-        seen.push(event);
-      },
-    };
-
-    // Seed events BEFORE the consumer first runs. Matches a deploy scenario
-    // where a new subscriber is added to a feature and starts from cursor=0
-    // against a populated events table. 500 events × batchSize=100 = 5 passes
-    // — still "multiple" and exercises the cursor-advance loop.
-    const count = 500;
-    await bulkSeedWidgetCreated(count, "backlog-");
-    // State row does not exist yet — this dispatcher is constructed inside
-    // the test, not via setupTestStack's auto-ensureRegistered.
-    expect(await getConsumerState(stack.db, name)).toBeNull();
-
-    const disp = createEventDispatcher({
-      db: stack.db,
-      consumers: [consumer],
-      context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
-      batchSize: 100,
-      pollIntervalMs: 5000,
-    });
-    // Strict pre-reg before the first pass — mirrors a production boot
-    // where start() runs before any runOnce(). Post-ensureRegistered the
-    // cursor row exists at 0 with the 500-event backlog ahead.
-    await disp.ensureRegistered();
-    const bootState = await getConsumerState(stack.db, name);
-    expect(bootState?.lastProcessedEventId).toBe(0n);
-
-    // batchSize = 100 → 5 passes cover 500 events. Run 8 to leave headroom;
-    // the 6th+ passes should be no-ops.
-    for (let pass = 0; pass < 8; pass++) {
-      const result = await disp.runOnce();
-      if (result.byConsumer[name]?.processed === 0) break;
-    }
-
-    // All events delivered, in order, exactly once.
-    expect(seen).toHaveLength(count);
-    for (let i = 0; i < count; i++) {
-      expect(seen[i]?.payload["name"]).toBe(`backlog-${i}`);
-    }
-
-    const finalState = await getConsumerState(stack.db, name);
-    expect(finalState?.lastProcessedEventId).toBe(BigInt(count));
-    expect(finalState?.status).toBe("idle");
-  });
-});
-
-// Welle 2.7 — per-instance delivery. Inverse of the shared test above:
-// with delivery="per-instance", each dispatcher gets its OWN cursor row
-// (via instance_id), so both dispatchers MUST deliver every event. SSE
-// broadcast in split-deploy is the canonical use-case.
-describe("Welle 2.7 — per-instance delivery: every dispatcher sees every event", () => {
-  test("two dispatchers with different instanceIds, same consumer name: both deliver every event", async () => {
-    const name = "multi:consumer:per-instance-echo";
-    const seenA: StoredEvent[] = [];
-    const seenB: StoredEvent[] = [];
-
-    const makeConsumer = (seen: StoredEvent[]): EventConsumer => ({
-      name,
-      delivery: "per-instance",
-      handler: async (event) => {
-        seen.push(event);
-      },
-    });
-
-    const dispA = createEventDispatcher({
-      db: stack.db,
-      consumers: [makeConsumer(seenA)],
-      context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
-      instanceId: "instance-A",
-      batchSize: 200,
-      pollIntervalMs: 5000,
-    });
-    const dispB = createEventDispatcher({
-      db: stack.db,
-      consumers: [makeConsumer(seenB)],
-      context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
-      instanceId: "instance-B",
-      batchSize: 200,
-      pollIntervalMs: 5000,
-    });
-    await dispA.ensureRegistered();
-    await dispB.ensureRegistered();
-
-    const count = 20;
-    for (let i = 0; i < count; i++) {
-      await appendWidget(`pi-${i}`);
-    }
-
-    // Run both dispatchers. Unlike shared delivery (race → exactly one
-    // wins), per-instance means both cursors advance independently.
-    await Promise.all([dispA.runOnce(), dispB.runOnce()]);
-
-    expect(seenA).toHaveLength(count);
-    expect(seenB).toHaveLength(count);
-
-    // Each instance has its own row with its own cursor.
-    const stateA = await getConsumerState(stack.db, name, "instance-A");
-    const stateB = await getConsumerState(stack.db, name, "instance-B");
-    expect(stateA?.lastProcessedEventId).toBe(BigInt(count));
-    expect(stateB?.lastProcessedEventId).toBe(BigInt(count));
-    expect(stateA?.instanceId).toBe("instance-A");
-    expect(stateB?.instanceId).toBe("instance-B");
-
-    // The shared sentinel row MUST NOT exist — per-instance consumers
-    // never write the default shard. If a bug ever routed per-instance
-    // writes to `__shared__`, this would silently collapse N instances'
-    // cursors into one and regress to shared semantics.
-    const stateShared = await getConsumerState(stack.db, name);
-    expect(stateShared).toBeNull();
-  });
-
-  test("mixed delivery: shared consumer stays exactly-once, per-instance consumer delivers to every dispatcher", async () => {
-    const sharedName = "multi:consumer:mixed-shared";
-    const perInstanceName = "multi:consumer:mixed-per-instance";
-
-    const sharedSeen: StoredEvent[] = [];
-    const perInstA: StoredEvent[] = [];
-    const perInstB: StoredEvent[] = [];
-
-    // Shared consumer registered on BOTH dispatchers — SKIP LOCKED on the
-    // one sentinel row means exactly one of them wins each event.
-    const sharedA: EventConsumer = {
-      name: sharedName,
-      handler: async (e) => void sharedSeen.push(e),
-    };
-    const sharedB: EventConsumer = { ...sharedA };
-
-    const perInstanceA: EventConsumer = {
-      name: perInstanceName,
-      delivery: "per-instance",
-      handler: async (e) => void perInstA.push(e),
-    };
-    const perInstanceB: EventConsumer = {
-      name: perInstanceName,
-      delivery: "per-instance",
-      handler: async (e) => void perInstB.push(e),
-    };
-
-    const dispA = createEventDispatcher({
-      db: stack.db,
-      consumers: [sharedA, perInstanceA],
-      context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
-      instanceId: "mixed-A",
-      batchSize: 200,
-      pollIntervalMs: 5000,
-    });
-    const dispB = createEventDispatcher({
-      db: stack.db,
-      consumers: [sharedB, perInstanceB],
-      context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
-      instanceId: "mixed-B",
-      batchSize: 200,
-      pollIntervalMs: 5000,
-    });
-    await dispA.ensureRegistered();
-    await dispB.ensureRegistered();
-
-    const count = 15;
-    for (let i = 0; i < count; i++) {
-      await appendWidget(`mix-${i}`);
-    }
-
-    // Multiple pass rounds so slow-loser of the SKIP-LOCKED race on the
-    // shared consumer still gets a chance to run if the fast-winner left
-    // events behind.
-    for (let pass = 0; pass < 3; pass++) {
-      await Promise.all([dispA.runOnce(), dispB.runOnce()]);
-    }
-
-    // Shared: total across both sides == count (exactly-once globally).
-    expect(sharedSeen).toHaveLength(count);
-
-    // Per-instance: each side gets the FULL set.
-    expect(perInstA).toHaveLength(count);
-    expect(perInstB).toHaveLength(count);
-  });
-
-  test("creating a dispatcher with a per-instance consumer but no instanceId throws at construction", () => {
-    expect(() =>
-      createEventDispatcher({
-        db: stack.db,
-        consumers: [
-          {
-            name: "multi:consumer:no-instance-id",
-            delivery: "per-instance",
-            handler: async () => {},
-          },
-        ],
-        context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
-        // instanceId deliberately omitted
-      }),
-    ).toThrow(/delivery="per-instance".+instanceId/);
-  });
-
-  test("instanceId equal to the reserved sentinel is rejected at construction", () => {
-    expect(() =>
-      createEventDispatcher({
-        db: stack.db,
-        consumers: [{ name: "x", handler: async () => {} }],
-        context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
-        instanceId: "__shared__",
-      }),
-    ).toThrow(/reserved sentinel/);
-  });
-});

package/src/pipeline/__tests__/event-dispatcher-pass-failure.integration.test.ts

@@ -1,266 +0,0 @@
-// kumiko-framework#2616 — processConsumer's db.begin rolled back the WHOLE
-// pass (including markProcessing/persistConsumerOutcome) whenever anything
-// inside it threw AFTER delivery succeeded, silently reverting attempts/
-// last_error/cursor to their pre-pass values. Combined with `context.log?.`,
-// a process without a logger lost the error entirely — in prod this looked
-// like a healthy consumer (status=idle, attempts=0, last_error=NULL) that
-// was actually looping at 24 rollbacks/sec.
-//
-// The real prod trigger was never identified (see the issue), so this test
-// reproduces the SHAPE of the bug via a legitimate fault-injection point
-// instead of the unknown original cause: EventDispatcherOptions.meter is
-// dependency-injected, and emitLagFromTx (called INSIDE the same db.begin,
-// after delivery+persistConsumerOutcome already ran) is where a thrown
-// metric emission rolls back an otherwise-successful pass. No production
-// code is touched to make this reproducible.
-
-import { afterEach, beforeAll, describe, expect, test } from "bun:test";
-import { createEventStoreExecutor } from "../../db/event-store-executor";
-import { asRawClient } from "../../db/query";
-import { createTenantDb, type TenantDb } from "../../db/tenant-db";
-import { defineFeature } from "../../engine";
-import { type Gauge, getFallbackMeter, type Meter } from "../../observability";
-import {
-  createEventDispatcher,
-  type EventConsumer,
-  type EventDispatcher,
-  getConsumerState,
-} from "../../pipeline";
-import {
-  resetEventStore,
-  setupTestStack,
-  type TestStack,
-  TestUsers,
-  unsafeCreateEntityTable,
-} from "../../stack";
-import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
-
-const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
-  entityName: "widget",
-});
-
-const feature = defineFeature("passfailure", (r) => {
-  r.entity("widget", sharedWidgetEntity);
-});
-
-const admin = TestUsers.admin;
-let stack: TestStack;
-let tdb: TenantDb;
-
-beforeAll(async () => {
-  stack = await setupTestStack({ features: [feature], systemHooks: [] });
-  await unsafeCreateEntityTable(stack.db, sharedWidgetEntity, "widget");
-  tdb = createTenantDb(stack.db, admin.tenantId);
-});
-
-afterEach(async () => {
-  await resetEventStore(stack, ["read_widgets"]);
-});
-
-async function appendWidget(name: string): Promise<void> {
-  await executor.create({ name }, admin, tdb);
-}
-
-// A meter that throws on the lag-gauge emission for one specific consumer —
-// the same injection point prod's unidentified exception would have hit,
-// since emitLagFromTx runs last inside processConsumer's db.begin.
-function makeLagFailingMeter(shouldFail: (consumer: string) => boolean): Meter {
-  const base = getFallbackMeter();
-  return {
-    registerMetric: (def) => base.registerMetric(def),
-    counter: (name) => base.counter(name),
-    histogram: (name) => base.histogram(name),
-    gauge: (name): Gauge => {
-      const real = base.gauge(name);
-      if (name !== "kumiko_event_consumer_lag_events") return real;
-      return {
-        set: (value, labels) => {
-          const consumer = String(labels?.["consumer"] ?? "");
-          if (shouldFail(consumer)) {
-            throw new Error(`injected-lag-failure-for-${consumer}`);
-          }
-          real.set(value, labels);
-        },
-        inc: (value, labels) => real.inc(value, labels),
-        dec: (value, labels) => real.dec(value, labels),
-      };
-    },
-    definitions: () => base.definitions(),
-  };
-}
-
-function buildDispatcher(consumers: readonly EventConsumer[], meter: Meter): EventDispatcher {
-  return createEventDispatcher({
-    db: stack.db,
-    consumers,
-    // Deliberately no `log` field — proves the console.error fallback, not
-    // the pre-existing context.log?.error(...) path.
-    context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
-    meter,
-    batchSize: 200,
-    pollIntervalMs: 5000,
-  });
-}
-
-describe("event-dispatcher — pass-level throw survives the rolled-back tx (#2616)", () => {
-  test("logs via console.error when context.log is absent", async () => {
-    const flaky = "passfailure:flaky-log";
-    const consumer: EventConsumer = { name: flaky, handler: async () => {} };
-    const dispatcher = buildDispatcher(
-      [consumer],
-      makeLagFailingMeter((c) => c === flaky),
-    );
-    await dispatcher.ensureRegistered();
-    await appendWidget("one");
-
-    const originalError = console.error;
-    const logged: string[] = [];
-    console.error = (...args: unknown[]) => {
-      logged.push(args.map(String).join(" "));
-    };
-    try {
-      await dispatcher.runOnce();
-    } finally {
-      console.error = originalError;
-    }
-
-    expect(logged.some((l) => l.includes(flaky) && l.includes("injected-lag-failure"))).toBe(true);
-  });
-
-  test("last_error survives the rollback while attempts stays untouched", async () => {
-    const flaky = "passfailure:attempts";
-    const consumer: EventConsumer = { name: flaky, handler: async () => {} };
-    const dispatcher = buildDispatcher(
-      [consumer],
-      makeLagFailingMeter((c) => c === flaky),
-    );
-    await dispatcher.ensureRegistered();
-    await appendWidget("one");
-
-    const before = await getConsumerState(stack.db, flaky);
-    expect(before?.attempts).toBe(0);
-    expect(before?.lastError).toBeNull();
-
-    await dispatcher.runOnce();
-
-    const after = await getConsumerState(stack.db, flaky);
-    // Delivery itself succeeded (the handler never threw) — only the lag
-    // emission did, rolling back markProcessing/persistConsumerOutcome. The
-    // dedicated post-catch write in its own transaction lands last_error,
-    // but must NOT touch attempts: that counter is deliverEvents' dead-letter
-    // budget, and this is an infra-level pass failure, not a handler throw.
-    expect(after?.attempts).toBe(0);
-    expect(after?.lastError).toMatch(/injected-lag-failure/);
-    // The cursor update rolled back with the rest of the transaction — the
-    // consumer will redeliver "one" once its backoff clears.
-    expect(after?.lastProcessedEventId).toBe(0n);
-  });
-
-  test("backs off after a thrown pass instead of retrying every tick, other consumers unaffected", async () => {
-    const flaky = "passfailure:backoff";
-    const healthy = "passfailure:healthy";
-    const healthySeen: string[] = [];
-
-    const flakyConsumer: EventConsumer = { name: flaky, handler: async () => {} };
-    const healthyConsumer: EventConsumer = {
-      name: healthy,
-      handler: async (event) => {
-        healthySeen.push(String(event.payload["name"]));
-      },
-    };
-    const dispatcher = buildDispatcher(
-      [flakyConsumer, healthyConsumer],
-      makeLagFailingMeter((c) => c === flaky),
-    );
-    await dispatcher.ensureRegistered();
-    await appendWidget("first");
-
-    await dispatcher.runOnce();
-    const afterFirst = await getConsumerState(stack.db, flaky);
-    expect(afterFirst?.attempts).toBe(0);
-    expect(afterFirst?.lastProcessedEventId).toBe(0n);
-
-    // Retry on the very next tick: the backoff must skip the flaky consumer
-    // (no delivery attempt, so no change to attempts) while the healthy one
-    // keeps advancing.
-    await appendWidget("second-for-healthy");
-    const second = await dispatcher.runOnce();
-    const afterSecond = await getConsumerState(stack.db, flaky);
-    expect(afterSecond?.attempts).toBe(0);
-    expect(healthySeen).toEqual(["first", "second-for-healthy"]);
-    expect(second.byConsumer[healthy]?.processed).toBeGreaterThan(0);
-  });
-
-  test("a successful pass clears the backoff so a later failure isn't skipped", async () => {
-    const name = "passfailure:recovers";
-    let failOnce = true;
-    const consumer: EventConsumer = { name, handler: async () => {} };
-    const meter = makeLagFailingMeter((c) => {
-      if (c === name && failOnce) {
-        failOnce = false;
-        return true;
-      }
-      return false;
-    });
-
-    const dispatcher = buildDispatcher([consumer], meter);
-    await dispatcher.ensureRegistered();
-    await appendWidget("one");
-
-    await dispatcher.runOnce();
-    const afterFailure = await getConsumerState(stack.db, name);
-    expect(afterFailure?.attempts).toBe(0);
-    expect(afterFailure?.lastProcessedEventId).toBe(0n);
-
-    // Wait out the (short, base-case) backoff window rather than reaching
-    // into dispatcher internals.
-    await new Promise((r) => setTimeout(r, 1100));
-    await dispatcher.runOnce();
-
-    const afterSuccess = await getConsumerState(stack.db, name);
-    expect(afterSuccess?.status).toBe("idle");
-    expect(afterSuccess?.attempts).toBe(0);
-    expect(afterSuccess?.lastProcessedEventId).toBe(1n);
-
-    // A fresh failure right after the successful pass must NOT be skipped —
-    // proves the backoff entry was cleared, not left at its escalated delay.
-    // attempts stays at 0: this is the same infra-level pass failure as
-    // above, which never touches deliverEvents' dead-letter budget.
-    failOnce = true;
-    await appendWidget("two");
-    await dispatcher.runOnce();
-    const afterSecondFailure = await getConsumerState(stack.db, name);
-    expect(afterSecondFailure?.attempts).toBe(0);
-  });
-
-  test("repeated infra-level pass failures never spend deliverEvents' dead-letter budget", async () => {
-    const flaky = "passfailure:budget";
-    const consumer: EventConsumer = { name: flaky, handler: async () => {} };
-    const meter = makeLagFailingMeter((c) => c === flaky);
-    await buildDispatcher([consumer], meter).ensureRegistered();
-
-    // Seed as if 9 of the real deliverEvents budget (maxAttempts=10) were
-    // already spent on genuine handler throws.
-    await asRawClient(stack.db).unsafe(
-      `UPDATE "kumiko_event_consumers" SET "attempts" = 9 WHERE "name" = $1`,
-      [flaky],
-    );
-    await appendWidget("one");
-
-    // Three infra-level pass failures in a row. Each uses a fresh dispatcher
-    // instance so its in-memory backoff never delays the next runOnce() —
-    // the DB row (and its seeded attempts=9) is what's shared and under test.
-    for (let i = 0; i < 3; i++) {
-      const dispatcher = buildDispatcher([consumer], meter);
-      await dispatcher.ensureRegistered();
-      await dispatcher.runOnce();
-    }
-
-    const after = await getConsumerState(stack.db, flaky);
-    // A 10th genuine handler throw would now dead-letter the consumer if
-    // these infra failures had bumped attempts past the seeded 9 — they
-    // must not, since none of them ran a real delivery pass.
-    expect(after?.attempts).toBe(9);
-    expect(after?.status).not.toBe("dead");
-  });
-});