@cosmicdrift/kumiko-framework 0.328.0 → 0.329.0

package/src/pipeline/event-dispatcher.ts

@@ -1,817 +0,0 @@
-import type { DbTx, PgClient } from "../db/connection";
-import { selectSnapshotXmax, selectSnapshotXmin } from "../db/queries/event-consumer";
-import type { AppContext } from "../engine/types";
-import { SYSTEM_TENANT_ID } from "../engine/types/identifiers";
-import { EVENTS_PUBSUB_CHANNEL, type StoredEvent } from "../event-store";
-import {
-  emitEventConsumerPassOutcome,
-  emitEventConsumerRearmExhausted,
-  emitEventDispatcherListenConnected,
-  getFallbackMeter,
-  getFallbackTracer,
-  type Meter,
-  type Tracer,
-} from "../observability";
-import { type PendingGapEntry, SHARED_INSTANCE_SENTINEL } from "./event-consumer-state";
-import {
-  acquireConsumerState,
-  consumerInstanceId,
-  type DeliveryOutcome,
-  deliverEvents,
-  emitLagFromTx,
-  fetchPendingEvents,
-  markProcessing,
-  type PersistedConsumerOutcome,
-  persistConsumerOutcome,
-  persistConsumerPassFailure,
-  preRegisterConsumers,
-  selectIdleConsumerKeys,
-} from "./event-dispatcher-delivery";
-import { partitionBurntGaps, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges";
-
-// Async event-dispatcher — the "AsyncDaemon"-pendant for Kumiko.
-//
-// Consumers (SSE broadcast, search-index, cross-feature subscribers, and —
-// later — async projections) read the events-table via a persistent cursor
-// held in kumiko_event_consumers. One row per consumer, one independent
-// cursor. A stalled Meili consumer doesn't block SSE; a dead subscription
-// doesn't pause the others.
-//
-// Run loop, per consumer, per pass:
-//   1. BEGIN
-//   2. SELECT state row FOR UPDATE SKIP LOCKED
-//      — multi-instance-safe: if another poller holds the lock, this pass
-//        skips this consumer and tries the next. No duplicate delivery.
-//   3. SELECT events WHERE id > lastProcessedEventId, PLUS any ranges still
-//      tracked in pending_gaps (ids invisible on an earlier turn that may
-//      have committed since) — ORDER BY id ASC LIMIT batchSize
-//   4. For each event: call the consumer's handler
-//        - handler throws → increment attempts, mark status="dead" at
-//          maxAttempts, surface lastError, STOP this consumer's pass
-//          (later events aren't consumed out-of-order)
-//        - handler succeeds → advance cursor, reset attempts
-//   5. COMMIT — cursor update + dead-letter flag land atomic
-//
-// Order guarantee: per-consumer, events are applied in events.id order. We
-// don't skip past a failing event — ops has to fix it or mark the consumer
-// disabled. This matches Marten's subscription semantics (and EventStoreDB's
-// persistent subscriptions): strictly-ordered + halt-on-poison.
-//
-// Delivery semantics: **at-least-once**. If a handler runs but the cursor
-// update fails (crash mid-pass), the same event is delivered again next pass.
-// Handlers MUST be idempotent.
-//
-// Delivery-loop mechanics (acquire/fetch/deliver/persist) live in
-// event-dispatcher-delivery.ts; the ops recovery surface (restart/disable/
-// enable/skipPoisonEvent/progress) lives in event-dispatcher-admin.ts. This
-// file is the facade: public types + the createEventDispatcher() factory.
-
-export type EventConsumerHandler = (event: StoredEvent, ctx: AppContext) => Promise<void>;
-
-// Batch variant of EventConsumerHandler — receives all events of one
-// delivery turn in id order, meant for side-effect sinks (search index,
-// bulk fan-out) that can collapse a turn into one round-trip. `handler`
-// stays mandatory and is the fallback: if the batch throws, the dispatcher
-// re-delivers the same events one by one through `handler`, so errorPolicy
-// (maxAttempts, skipApplyErrors, dead-lettering) applies per event exactly
-// as before. The batch handler must therefore be replay-safe — a partially
-// applied batch gets re-applied event by event under the same idempotency
-// rule as `handler`. Events appended from inside a batch handler carry the
-// LAST event's causation id only.
-export type EventConsumerBatchHandler = (
-  events: readonly StoredEvent[],
-  ctx: AppContext,
-) => Promise<void>;
-
-// Per-consumer error policy. When skipApplyErrors is true and handler throws,
-// the dispatcher logs the error, advances the cursor past the offending event,
-// and keeps delivering — instead of the default retry + dead-letter flow.
-// Wire by copying MultiStreamProjectionDefinition.errorMode.continuous into
-// the EventConsumer (see api/server.ts MSP wiring).
-export type EventConsumerErrorPolicy = {
-  readonly skipApplyErrors?: boolean;
-  // Per-consumer override of EventDispatcherOptions.maxAttempts. A consumer
-  // that depends on infra which may still be provisioning at boot (search
-  // adapter, external APIs) needs more retry headroom than the dispatcher-
-  // wide default before it gets dead-lettered.
-  readonly maxAttempts?: number;
-};
-
-export type EventConsumer = {
-  readonly name: string;
-  readonly handler: EventConsumerHandler;
-  // Optional batch fast-path for this turn's events — see
-  // EventConsumerBatchHandler for the fallback and replay-safety contract.
-  readonly batchHandler?: EventConsumerBatchHandler;
-  readonly errorPolicy?: EventConsumerErrorPolicy;
-  // Owning feature — when present, the dispatcher skips this consumer's
-  // pass while the feature is globally disabled. Events remain in the store
-  // and the consumer resumes from the same cursor when the feature is
-  // re-enabled (no data loss, no replay). System consumers (SSE, search,
-  // framework-level plumbing) omit this and always run.
-  readonly featureName?: string;
-  // Delivery semantics across multi-instance deploys:
-  //   "shared"       (default) — one cursor across all instances. SKIP LOCKED
-  //                   serialises; each event delivered exactly once globally.
-  //   "per-instance" — one cursor per (name, dispatcher.instanceId) shard.
-  //                   Every process delivers every event independently. For
-  //                   push-to-local-subscribers (SSE broker, in-memory cache
-  //                   invalidators). Handler MUST be side-effect-free with
-  //                   respect to shared storage (no DB writes), otherwise
-  //                   each instance duplicates the effect.
-  readonly delivery?: "shared" | "per-instance";
-  // Where the consumer's cursor starts on its FIRST registration. Default
-  // "beginning" replays the full log. "now" seeds the cursor at the current
-  // MAX(events.id) so mounting the consumer into an existing app does not
-  // retroactively fire side effects for historical events. Only affects the
-  // first registration — an existing cursor row is never clobbered.
-  readonly startFrom?: "beginning" | "now";
-};
-
-// Result of a dispatcher pass (runOnce / doPass). Shared across the public
-// interface and the internal helpers so all three sites agree on the shape
-// — adding a counter in one place wouldn't have compiled on the others
-// when the type was inlined in each signature.
-export type DispatcherPassResult = {
-  readonly processed: number;
-  readonly failed: number;
-  readonly byConsumer: Record<string, { processed: number; failed: number }>;
-};
-
-export type EventDispatcher = {
-  start(): Promise<void>;
-  stop(): Promise<void>;
-  // Force one pass now (tests drain deterministically via this).
-  // Throws if start() was never called — pre-registration of consumer
-  // state rows is a precondition, not a side-effect of the pass itself.
-  runOnce(): Promise<DispatcherPassResult>;
-  // Idempotent re-pre-registration of consumer state rows. Exists as a
-  // test-teardown surface: after `TRUNCATE kumiko_event_consumers` the
-  // rows are gone, and strict acquire() would skip every consumer as
-  // "not_registered". Tests call ensureRegistered() to repopulate without
-  // a full stop/start cycle. Production never needs this — start() runs
-  // it once on boot and the rows survive dispatcher lifetime.
-  ensureRegistered(): Promise<void>;
-  // Waits out a pass already in flight, without touching the timer or the
-  // LISTEN subscription — unlike stop(), the dispatcher keeps running
-  // afterwards. Test-teardown surface: a beforeEach/afterEach that resets
-  // the events table (TRUNCATE) must not race a pass this same dispatcher
-  // started (timer tick or NOTIFY wake-up) that is still writing into the
-  // tables the reset is about to wipe. No-op when nothing is in flight.
-  drain(): Promise<void>;
-  // Read-only view of the consumers this dispatcher is wired with. Exists
-  // for lane-filter assertions (Welle 2.6.b split-deploy tests) and for
-  // the boot-validator (Welle 2.6.c coverage check: every registered MSP
-  // must appear in at least one process's dispatcher). No runtime semantics
-  // — the list doesn't change after construction.
-  readonly consumers: readonly EventConsumer[];
-};
-
-export type EventDispatcherOptions = {
-  readonly db: import("../db/connection").DbConnection;
-  readonly consumers: readonly EventConsumer[];
-  readonly context: AppContext;
-  readonly batchSize?: number;
-  readonly pollIntervalMs?: number;
-  readonly maxAttempts?: number;
-  // Bounded auto-revival of a "dead" consumer. A dead consumer whose last
-  // write is older than rearmCooldownMs gets reset to idle and retried.
-  // rearmCount tracks re-arms since the last delivery that actually
-  // advanced the cursor — a poison event never advances it, so it climbs
-  // 1→maxRearmCount and then stays dead permanently; a transient failure
-  // that later delivers successfully resets it back to 0, so an unrelated
-  // future outage gets its own fresh budget. restartConsumer()/
-  // enableConsumer()/skipPoisonEvent() also reset it (an operator vouching
-  // the consumer is healthy again).
-  readonly rearmCooldownMs?: number;
-  readonly maxRearmCount?: number;
-  readonly tracer?: Tracer;
-  readonly meter?: Meter;
-  // Identifies THIS dispatcher process in the consumer-state table. Used as
-  // the `instance_id` value for every per-instance consumer's cursor row.
-  // Shared-delivery consumers ignore this and always use
-  // SHARED_INSTANCE_SENTINEL. Default undefined — dispatchers without any
-  // per-instance consumers don't need it. Required when at least one
-  // consumer has delivery="per-instance"; createEventDispatcher throws on
-  // boot if the invariant is violated, avoiding a later runtime surprise.
-  readonly instanceId?: string;
-  // Optional raw postgres.js client for LISTEN/NOTIFY-based wake-up
-  // (Sprint E.4). When present, `.start()` subscribes to EVENTS_PUBSUB_CHANNEL
-  // and fires runOnce on each NOTIFY — delivery latency becomes TCP-round-
-  // trip instead of pollIntervalMs. The polling timer remains active as a
-  // safety net (missed NOTIFYs from crashes, subscription drops).
-  readonly pgClient?: PgClient;
-};
-
-const DEFAULT_BATCH_SIZE = 200;
-const DEFAULT_POLL_MS = 100;
-const DEFAULT_MAX_ATTEMPTS = 10;
-const DEFAULT_REARM_COOLDOWN_MS = 5 * 60_000;
-const DEFAULT_MAX_REARM_COUNT = 3;
-// Backoff for a consumer whose pass THREW (the processConsumer catch below)
-// — an infra-level failure (db.begin's callback rejected outside deliverEvents'
-// own per-event try/catch), not a poisoned event. Doubles per consecutive
-// failure, capped at PASS_FAILURE_MAX_BACKOFF_MS so a consumer stuck failing
-// still gets retried within a minute; a successful pass resets it to 0.
-const PASS_FAILURE_BASE_BACKOFF_MS = 1_000;
-const PASS_FAILURE_MAX_BACKOFF_MS = 60_000;
-
-export function createEventDispatcher(options: EventDispatcherOptions): EventDispatcher {
-  const {
-    db,
-    consumers,
-    context,
-    batchSize = DEFAULT_BATCH_SIZE,
-    pollIntervalMs = DEFAULT_POLL_MS,
-    maxAttempts = DEFAULT_MAX_ATTEMPTS,
-    rearmCooldownMs = DEFAULT_REARM_COOLDOWN_MS,
-    maxRearmCount = DEFAULT_MAX_REARM_COUNT,
-  } = options;
-
-  // Fail-fast on misconfigured per-instance wiring. Catching this at
-  // construction surfaces the problem in boot logs instead of first
-  // delivery attempt — where it would land as a confusing preRegister
-  // throw much later in the startup sequence.
-  for (const consumer of consumers) {
-    if (consumer.delivery === "per-instance" && !options.instanceId) {
-      throw new Error(
-        `EventConsumer "${consumer.name}" has delivery="per-instance" but EventDispatcherOptions.instanceId is missing. ` +
-          `Pass ServerOptions.instanceId (defaults to KUMIKO_INSTANCE_ID or a boot-time UUID) when any consumer uses per-instance delivery.`,
-      );
-    }
-  }
-  if (options.instanceId === SHARED_INSTANCE_SENTINEL) {
-    throw new Error(
-      `EventDispatcherOptions.instanceId cannot equal the reserved sentinel "${SHARED_INSTANCE_SENTINEL}". ` +
-        `Pick any other stable string (typically KUMIKO_INSTANCE_ID from the deploy env).`,
-    );
-  }
-  const tracer: Tracer = options.tracer ?? getFallbackTracer();
-  const meter: Meter = options.meter ?? getFallbackMeter();
-  // Tracks which (consumer, instanceId) pairs already fired
-  // kumiko_event_consumer_rearm_exhausted_total, so a consumer stuck dead
-  // across many poll passes emits the ops-signal once, not every pass
-  // (that would be log/metric spam for a state that hasn't changed).
-  // Process-lifetime only — restarting the dispatcher re-arms reporting,
-  // which is fine: a fresh process re-observing a still-dead consumer is
-  // exactly the "still needs a human" signal ops wants.
-  const reportedDeadConsumers = new Set<string>();
-
-  // Per-(consumer, instanceId) backoff after a thrown pass (see the catch in
-  // processConsumer). Keyed the same way as reportedDeadConsumers.
-  // consecutiveFailures drives the exponential delay; retryAtMs is the
-  // Date.now() timestamp doPass compares against to decide whether this
-  // consumer's turn is skipped this tick. A successful pass deletes the
-  // entry — the next failure (if any) starts a fresh backoff from scratch.
-  const consumerBackoff = new Map<string, { consecutiveFailures: number; retryAtMs: number }>();
-
-  // Dedup flag for the idle pre-check's error log (doPass) — a broken
-  // pre-check falls back to the expensive-but-correct path silently on
-  // every tick (see doPass), which must not mean silently in the logs too:
-  // ops needs to see it degrade once, not have every poll tick spam the
-  // same line. Cleared the moment the pre-check succeeds again, so a later,
-  // unrelated outage gets its own fresh log line instead of staying
-  // permanently suppressed by this process's flag.
-  let idlePreCheckErrorLogged = false;
-
-  let running = false;
-  // Separate from `running` on purpose: pre-registration of consumer state
-  // rows is a one-time boot action, while running/timer/LISTEN is a
-  // lifecycle toggle. stop() flips running back to false but leaves
-  // preRegistered true — a subsequent runOnce() is still safe because the
-  // state rows are in place. Production code never stops-then-runs-once;
-  // tests do (drain on-demand without a timer loop).
-  let preRegistered = false;
-  let timer: ReturnType<typeof setInterval> | null = null;
-  // LISTEN subscription handle. Set when .start() successfully subscribed
-  // to EVENTS_PUBSUB_CHANNEL; cleared by .stop(). The timer remains active
-  // even with LISTEN attached — it's a cheap safety net against missed
-  // NOTIFYs (subscription drop, crash mid-commit).
-  let pgUnlisten: (() => Promise<void>) | null = null;
-
-  // Bounds how many consumer turns hold a DB transaction at once. Each turn's
-  // db.begin() checks out one pool connection, plus any writes the handler
-  // itself makes via context.db — the app db pool defaults to postgres-js's
-  // max=10 (db/connection.ts, no DATABASE_POOL_MAX override) and is shared
-  // with HTTP request handlers, so turns can't be allowed to claim it all.
-  // A pool no larger than this limit can deadlock: every connection held by
-  // an open turn TX while each handler waits for one more.
-  const MAX_CONCURRENT_CONSUMER_TURNS = 4;
-  let activeConsumerTurns = 0;
-  const consumerTurnWaiters: Array<() => void> = [];
-
-  async function acquireConsumerTurnSlot(): Promise<() => void> {
-    if (activeConsumerTurns >= MAX_CONCURRENT_CONSUMER_TURNS) {
-      // The releasing turn hands its slot over directly (count unchanged),
-      // so a synchronous acquire in between cannot overshoot the limit.
-      await new Promise<void>((resolve) => consumerTurnWaiters.push(resolve));
-    } else {
-      activeConsumerTurns++;
-    }
-    let released = false;
-    return () => {
-      // skip: already released — finally-blocks calling this twice must not
-      // free the same slot twice
-      if (released) return;
-      released = true;
-      const nextWaiter = consumerTurnWaiters.shift();
-      if (nextWaiter) nextWaiter();
-      else activeConsumerTurns--;
-    };
-  }
-
-  // Per-(consumer, instanceId) turn guard, so one slow consumer never holds
-  // back the others. Keyed like consumerBackoff: a consumer already running
-  // its turn hands back that same promise instead of starting a second one.
-  const inFlightTurns = new Map<string, Promise<{ processed: number; failed: number }>>();
-
-  async function runConsumerTurn(
-    consumer: EventConsumer,
-    effective: ReadonlySet<string> | undefined,
-    idleKeys: ReadonlySet<string>,
-  ): Promise<{ processed: number; failed: number }> {
-    const key = `${consumer.name}:${consumerInstanceId(consumer, options.instanceId)}`;
-    const existing = inFlightTurns.get(key);
-    if (existing) return existing;
-
-    // Feature-gate, backoff-gate and idle-gate are resolved synchronously,
-    // before any promise is registered — a gated consumer must resolve
-    // without ever occupying inFlightTurns, or it would look "still
-    // running" forever.
-    if (effective && consumer.featureName && !effective.has(consumer.featureName)) {
-      return { processed: 0, failed: 0 };
-    }
-    const backoff = consumerBackoff.get(key);
-    if (backoff && backoff.retryAtMs > Date.now()) {
-      return { processed: 0, failed: 0 };
-    }
-    // Idle-gate: doPass already proved this consumer has nothing to do
-    // (see selectIdleConsumerKeys) — skip the turn without ever opening a
-    // db.begin/FOR UPDATE SKIP LOCKED (see doPass for why that matters).
-    // Clear any prior backoff same as a successful pass would: the
-    // backoff-gate above already confirmed retryAtMs has elapsed, and
-    // "provably nothing to deliver" is itself proof the consumer isn't
-    // presently failing.
-    if (idleKeys.has(key)) {
-      consumerBackoff.delete(key);
-      return { processed: 0, failed: 0 };
-    }
-
-    // Registered before awaiting the slot — a turn queued behind the
-    // concurrency limit still counts as in-flight, so it isn't started twice.
-    const turn = (async () => {
-      const releaseSlot = await acquireConsumerTurnSlot();
-      try {
-        return await processConsumer(consumer);
-      } finally {
-        releaseSlot();
-      }
-    })();
-    inFlightTurns.set(key, turn);
-    try {
-      return await turn;
-    } finally {
-      if (inFlightTurns.get(key) === turn) inFlightTurns.delete(key);
-    }
-  }
-
-  async function drainInFlightTurns(): Promise<void> {
-    await Promise.all(
-      [...inFlightTurns.values()].map((turn) =>
-        turn.catch(() => {
-          // skip: errors already recorded per-consumer inside the pass
-        }),
-      ),
-    );
-  }
-
-  // A pass still in doPass's idle pre-check (selectIdleConsumerKeys) hasn't
-  // registered any consumer turn in inFlightTurns yet — stop() must wait for
-  // it too, or it keeps querying the pool stop() is about to close.
-  const inFlightPasses = new Set<Promise<DispatcherPassResult>>();
-
-  async function drainInFlightPasses(): Promise<void> {
-    await Promise.all(
-      [...inFlightPasses].map((pass) =>
-        pass.catch(() => {
-          // skip: errors already recorded per-consumer inside the pass
-        }),
-      ),
-    );
-  }
-
-  async function runOnce(): Promise<DispatcherPassResult> {
-    if (!preRegistered) {
-      throw new Error(
-        "EventDispatcher.runOnce() called before start() — consumer state rows are not registered. Call start() first (production) or ensureRegistered() (tests after truncating kumiko_event_consumers).",
-      );
-    }
-    const pass = doPass();
-    inFlightPasses.add(pass);
-    try {
-      return await pass;
-    } finally {
-      inFlightPasses.delete(pass);
-    }
-  }
-
-  async function doPass(): Promise<DispatcherPassResult> {
-    let totalProcessed = 0;
-    let totalFailed = 0;
-    const byConsumer: Record<string, { processed: number; failed: number }> = {};
-
-    // Feature-toggle snapshot taken once per pass (not per consumer): all
-    // consumers see the same disabled-set even if an operator flips a
-    // toggle mid-pass, so "this event batch" decisions stay consistent.
-    //
-    // Sprint-8a tier-composition: per-tenant resolver per-pass-konsultiert
-    // mit SYSTEM_TENANT_ID. Async-events sind tier-agnostic — wenn ein
-    // Tenant downgrade'd, sollen seine queued events trotzdem verarbeitet
-    // werden (events sind immutable, projection ist eventually-consistent).
-    // Tier-cuts wirken request-time im sync-dispatcher + lifecycle-pipeline,
-    // nicht im async-replay. App-level resolver entscheidet was er bei
-    // SYSTEM_TENANT_ID returnt (typisch: union-of-all-tier-features).
-    const effective = context.effectiveFeatures?.(SYSTEM_TENANT_ID);
-
-    // Lock-free pre-check, once per pass: which consumers are provably idle
-    // (no locking, no xid, no WAL — see selectIdleConsumerKeys). Read
-    // outside any transaction, before a single db.begin/FOR UPDATE SKIP
-    // LOCKED opens — an idle deployment (most consumers, most ticks)
-    // otherwise pays a row lock + WAL record + commit fsync per consumer
-    // per tick for nothing.
-    //
-    // A failure here (e.g. a transient DB blip) must not abort the whole
-    // pass — timer/LISTEN callers swallow doPass()'s rejection silently
-    // (see start()), which would mean no log, no persistConsumerPassFailure,
-    // and no backoff for every consumer this tick. Falling back to "nothing
-    // proven idle" instead routes every consumer through its normal
-    // acquireConsumerState path, whose own try/catch already covers this.
-    let idleKeys: ReadonlySet<string>;
-    try {
-      idleKeys = await selectIdleConsumerKeys(
-        db,
-        consumers.map((consumer) => ({
-          name: consumer.name,
-          instanceId: consumerInstanceId(consumer, options.instanceId),
-        })),
-      );
-      if (idlePreCheckErrorLogged) {
-        idlePreCheckErrorLogged = false;
-        const recoveredMsg =
-          "[event-dispatcher] idle pre-check recovered, database reachable again";
-        if (context.log) {
-          context.log.info(recoveredMsg);
-        } else {
-          console.log(recoveredMsg);
-        }
-      }
-    } catch (e) {
-      idleKeys = new Set();
-      // Log once per outage, not once per poll tick — see
-      // idlePreCheckErrorLogged above. Every consumer still gets delivered
-      // correctly this tick (fallback above), so this is a degradation
-      // signal for ops, not a delivery failure.
-      if (!idlePreCheckErrorLogged) {
-        idlePreCheckErrorLogged = true;
-        // The LISTEN peer talks to the same DB, so it is gone too; only a
-        // later onlisten may flip the gauge back to 1.
-        emitEventDispatcherListenConnected(meter, false);
-        const msg = e instanceof Error ? e.message : String(e);
-        const logMsg = `[event-dispatcher] idle pre-check failed, falling back to per-consumer locking every tick until it recovers: ${msg}`;
-        if (context.log) {
-          context.log.error(logMsg);
-        } else {
-          console.error(logMsg);
-        }
-      }
-    }
-
-    // Every consumer runs its own turn concurrently (own TX, own cursor,
-    // own backoff/feature-gate/idle-gate) — a slow consumer no longer
-    // blocks the others' delivery until it commits, see runConsumerTurn.
-    const results = await Promise.all(
-      consumers.map(async (consumer) => {
-        const perConsumer = await runConsumerTurn(consumer, effective, idleKeys);
-        return [consumer.name, perConsumer] as const;
-      }),
-    );
-    for (const [name, perConsumer] of results) {
-      byConsumer[name] = perConsumer;
-      totalProcessed += perConsumer.processed;
-      totalFailed += perConsumer.failed;
-    }
-
-    return { processed: totalProcessed, failed: totalFailed, byConsumer };
-  }
-
-  async function processConsumer(
-    consumer: EventConsumer,
-  ): Promise<{ processed: number; failed: number }> {
-    let processed = 0;
-    let failed = 0;
-
-    const instanceId = consumerInstanceId(consumer, options.instanceId);
-
-    const span = tracer.startSpan("events.consumer.pass", {
-      attributes: {
-        "consumer.name": consumer.name,
-        "consumer.instance_id": instanceId,
-      },
-    });
-
-    try {
-      await db.begin(async (tx: DbTx) => {
-        const acquired = await acquireConsumerState(
-          tx,
-          consumer.name,
-          instanceId,
-          rearmCooldownMs,
-          maxRearmCount,
-        );
-        // skip: another instance holds the lock, or the consumer is
-        // disabled/dead. Nothing to deliver this pass.
-        if (acquired.skip !== null) {
-          span.setAttribute("consumer.skip_reason", acquired.skip);
-          if (acquired.skip === "dead") {
-            const reportKey = `${consumer.name}:${instanceId}`;
-            if (!reportedDeadConsumers.has(reportKey)) {
-              reportedDeadConsumers.add(reportKey);
-              emitEventConsumerRearmExhausted(meter, { consumer: consumer.name, instanceId });
-            }
-          }
-          // skip: skip reason already recorded on the span above (and, for
-          // "dead", already emitted as a metric) — nothing left to deliver.
-          return;
-        }
-        // Acquired normally (including via a successful auto-rearm) — clear
-        // any prior dead-report so a future exhaustion re-emits instead of
-        // staying permanently suppressed by this process's Set.
-        reportedDeadConsumers.delete(`${consumer.name}:${instanceId}`);
-
-        const oldCursor = acquired.state.lastProcessedEventId;
-        const pendingGaps = acquired.state.pendingGaps;
-        // xmin BEFORE the fetch: a range is provably burnt only once this has
-        // passed the xmax recorded with it. Only needed when ranges exist.
-        const xminNow = pendingGaps.length > 0 ? await selectSnapshotXmin(tx) : "0";
-        const events = await fetchPendingEvents(tx, oldCursor, batchSize, toIdRanges(pendingGaps));
-        const fetchedIds = events.map((e) => e.id);
-        const truncated = events.length === batchSize;
-
-        // Burnt: no visible id this turn, the fetch is proven to have
-        // covered the range (LIMIT can otherwise cut it off early), and
-        // xmin passed its recorded xmax — every xact that could still
-        // produce a row has finished without one ever appearing.
-        const { burnt: burntGaps, surviving: survivingGaps } = partitionBurntGaps(
-          pendingGaps,
-          fetchedIds,
-          truncated,
-          xminNow,
-        );
-
-        // skip: nothing to deliver and no burnt gap to clean up. The
-        // doPass idle pre-check (selectIdleConsumerKeys) already keeps the
-        // common idle case out of this transaction entirely — reaching
-        // here with an empty fetch means the pre-check couldn't prove
-        // idleness (pending gaps forced a fetch, or a race window). By this
-        // point acquireConsumerState's FOR UPDATE has already taken the
-        // lock, so skipping markProcessing/persistConsumerOutcome only
-        // avoids extra writes, not that lock's own WAL record.
-        if (events.length === 0 && burntGaps.length === 0) {
-          span.setAttribute("consumer.skip_reason", "no_pending_events");
-          return;
-        }
-        await markProcessing(tx, consumer.name, instanceId);
-
-        const outcome: DeliveryOutcome =
-          events.length > 0
-            ? await deliverEvents(consumer, events, context, maxAttempts, acquired.state)
-            : {
-                cursor: oldCursor,
-                attempts: acquired.state.attempts,
-                lastError: acquired.state.lastError,
-                deadLettered: false,
-                processed: 0,
-                failed: 0,
-                resolvedPendingIds: [],
-              };
-        processed = outcome.processed;
-        failed = outcome.failed;
-
-        // Carve delivered/skip-applied ids out of the surviving ranges —
-        // an id still fetched-but-unresolved (halt-on-poison stopped before
-        // it) stays put, since it's below the cursor and only pending_gaps
-        // will ever retry it.
-        const keptGaps = survivingGaps.flatMap((gap) =>
-          splitRangeExcludingIds(gap, outcome.resolvedPendingIds),
-        );
-        // New gaps as ranges between consecutive fetched ids, so a huge id
-        // jump (retention prune) costs one entry, not one per missing id.
-        // Their xmax is read after the fetch; any later read is only more
-        // conservative, so it is fetched lazily.
-        const newWindowIds = fetchedIds.filter((id) => id > oldCursor && id <= outcome.cursor);
-        const newGapBounds: Array<readonly [bigint, bigint]> = [];
-        let prev = oldCursor;
-        for (const id of newWindowIds) {
-          if (id > prev + 1n) newGapBounds.push([prev + 1n, id - 1n]);
-          prev = id;
-        }
-        const xmaxNow = newGapBounds.length > 0 ? await selectSnapshotXmax(tx) : "";
-        const newGaps: PendingGapEntry[] = newGapBounds.map(([from, to]) => ({
-          from: from.toString(),
-          to: to.toString(),
-          xmax: xmaxNow,
-        }));
-        const persistedOutcome: PersistedConsumerOutcome = {
-          ...outcome,
-          pendingGaps: [...keptGaps, ...newGaps],
-        };
-
-        await persistConsumerOutcome(tx, consumer.name, instanceId, persistedOutcome);
-        await emitLagFromTx(tx, consumer.name, instanceId, outcome.cursor, meter);
-      });
-
-      // Pass completed without throwing — clear any backoff from a prior
-      // failure so the next unrelated outage gets its own fresh delay
-      // instead of inheriting a partially-escalated one.
-      consumerBackoff.delete(`${consumer.name}:${instanceId}`);
-
-      emitEventConsumerPassOutcome(
-        meter,
-        { consumer: consumer.name, instanceId },
-        processed,
-        failed,
-      );
-      span.setAttribute("consumer.processed", processed);
-      span.setAttribute("consumer.failed", failed);
-      span.setStatus(failed === 0 ? "ok" : "error");
-    } catch (e) {
-      // Unexpected: a handler error is caught inside deliverEvents and
-      // surfaces via `failed`, so anything landing here is infrastructure
-      // (db connection lost, serialization, standard-metrics not registered
-      // on this meter). Don't let one consumer's outage stall the others,
-      // but do log — a silent rollback here looks like "at-most-once" to
-      // callers and at-least-once-with-duplicate-delivery on the next pass;
-      // neither is what we want, so ops needs to see it.
-      const msg = e instanceof Error ? e.message : String(e);
-      if (context.log) {
-        context.log.error(`[event-dispatcher] ${consumer.name} pass failed: ${msg}`);
-      } else {
-        // No logger wired onto this context (e.g. createEventDispatcher used
-        // standalone) — console.error is the only way ops still sees this in
-        // pod logs instead of the failure vanishing with the rolled-back tx.
-        console.error(`[event-dispatcher] ${consumer.name} pass failed: ${msg}`);
-      }
-      span.setStatus("error", msg);
-
-      // The whole pass rolled back, so markProcessing/persistConsumerOutcome
-      // never committed — attempts/last_error are stuck at their pre-pass
-      // values. Record the failure in its OWN transaction so it survives.
-      // Best-effort: this write must never itself take down the dispatcher,
-      // so any failure here is swallowed after logging.
-      try {
-        await db.begin(async (tx: DbTx) => {
-          await persistConsumerPassFailure(tx, consumer.name, instanceId, msg);
-        });
-      } catch (persistErr) {
-        const persistMsg = persistErr instanceof Error ? persistErr.message : String(persistErr);
-        const failureMsg = `[event-dispatcher] ${consumer.name} failed to persist pass-failure state: ${persistMsg}`;
-        if (context.log) {
-          context.log.error(failureMsg);
-        } else {
-          console.error(failureMsg);
-        }
-      }
-
-      // Escalate the backoff so this consumer isn't retried on the very
-      // next tick — see PASS_FAILURE_BASE_BACKOFF_MS/MAX_MS above.
-      const key = `${consumer.name}:${instanceId}`;
-      const consecutiveFailures = (consumerBackoff.get(key)?.consecutiveFailures ?? 0) + 1;
-      const delayMs = Math.min(
-        PASS_FAILURE_BASE_BACKOFF_MS * 2 ** (consecutiveFailures - 1),
-        PASS_FAILURE_MAX_BACKOFF_MS,
-      );
-      consumerBackoff.set(key, { consecutiveFailures, retryAtMs: Date.now() + delayMs });
-    } finally {
-      span.end();
-    }
-
-    return { processed, failed };
-  }
-
-  return {
-    consumers,
-    async start() {
-      // skip: already running, idempotent
-      if (running) return;
-      running = true;
-
-      // Pre-register consumer state rows. Without this, a consumer first
-      // bootstraps lazily on its first runOnce — and if prune runs between
-      // "process came up" and "first pass landed", prune wouldn't see the
-      // consumer in the state table, would delete events past its (absent)
-      // cursor, and the consumer's first pass would silently skip them.
-      //
-      // Pre-registering turns every consumer into a row-with-cursor-0 the
-      // moment the dispatcher starts — so the retention guard
-      // (pruneEvents → ConsumerLagError) correctly refuses to prune past
-      // any consumer that exists, including freshly-deployed ones.
-      await preRegisterConsumers(db, consumers, options.instanceId);
-      preRegistered = true;
-
-      timer = setInterval(() => {
-        void runOnce().catch(() => {
-          // skip: per-consumer errors already recorded in the state row
-        });
-      }, pollIntervalMs);
-
-      // NOTIFY-based wake-up: subscribe on the same channel that
-      // event-store.append fires on commit. Fires runOnce directly, no
-      // polling round-trip. The timer stays on as a belt-and-braces
-      // fallback (dropped subscriptions, missed commits under load).
-      //
-      // Observability: the gauge kumiko_event_dispatcher_listen_connected
-      // flips to 1 on initial subscribe and again on every reconnect that
-      // succeeds (onlisten callback). It drops to 0 on the initial connect
-      // failure, on stop() and when the idle pre-check detects a DB outage.
-      // An outage shorter than one poll tick is not detected; the gauge then
-      // stays at 1 until postgres.js's delayed re-LISTEN lands.
-      emitEventDispatcherListenConnected(meter, false);
-      if (options.pgClient) {
-        try {
-          const sub = await options.pgClient.listen(
-            EVENTS_PUBSUB_CHANNEL,
-            () => {
-              void runOnce().catch(() => {
-                // skip: per-consumer errors already recorded in the state row
-              });
-            },
-            () => {
-              // Fires on initial connect and again when postgres.js's own
-              // re-LISTEN lands after a drop; during an outage that re-LISTEN
-              // waits in its connection backoff (measured 10-30s after the DB
-              // returns), polling covers delivery meanwhile. Never unlisten()
-              // + listen() by hand: that leaves duplicate listeners.
-              // ponytail: no own LISTEN reconnect, the backoff gap is polled;
-              // add a dedicated LISTEN connection if prod wires a pgClient
-              // and that latency matters.
-              emitEventDispatcherListenConnected(meter, true);
-            },
-          );
-          pgUnlisten = sub.unlisten;
-        } catch (e) {
-          emitEventDispatcherListenConnected(meter, false);
-          const msg = e instanceof Error ? e.message : String(e);
-          context.log?.error(`[event-dispatcher] pg LISTEN failed: ${msg}`);
-        }
-      }
-    },
-
-    async stop() {
-      // skip: already stopped, idempotent
-      if (!running) return;
-      running = false;
-
-      if (timer) {
-        clearInterval(timer);
-        timer = null;
-      }
-
-      if (pgUnlisten) {
-        await pgUnlisten().catch(() => {
-          // skip: unlisten failure only matters during shutdown — the
-          // subscription is being torn down anyway.
-        });
-        pgUnlisten = null;
-        emitEventDispatcherListenConnected(meter, false);
-      }
-
-      // Drain any in-flight pass so shutdown observes consistent state —
-      // including one still stuck in the idle pre-check above inFlightTurns.
-      await drainInFlightPasses();
-      await drainInFlightTurns();
-      // preRegistered stays true — the rows survive stop(). runOnce()
-      // after a stop() still works (tests stop the timer and then drain
-      // deterministically).
-    },
-
-    async ensureRegistered() {
-      await preRegisterConsumers(db, consumers, options.instanceId);
-      preRegistered = true;
-    },
-
-    drain: drainInFlightTurns,
-
-    runOnce,
-  };
-}
-
-export type { ConsumerProgress, ConsumerRecoveryState } from "./event-dispatcher-admin";
-export {
-  disableConsumer,
-  enableConsumer,
-  getAllConsumerProgress,
-  getConsumerState,
-  listConsumersWithState,
-  restartConsumer,
-  skipPoisonEvent,
-} from "./event-dispatcher-admin";