@cosmicdrift/kumiko-framework 0.307.0 → 0.309.0

package/src/pipeline/event-dispatcher.ts

@@ -25,6 +25,7 @@ import {
   persistConsumerOutcome,
   persistConsumerPassFailure,
   preRegisterConsumers,
+  selectIdleConsumerKeys,
 } from "./event-dispatcher-delivery";
 import { partitionBurntGaps, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges";
 
@@ -67,6 +68,21 @@ import { partitionBurntGaps, splitRangeExcludingIds, toIdRanges } from "./pendin
 
 export type EventConsumerHandler = (event: StoredEvent, ctx: AppContext) => Promise<void>;
 
+// Batch variant of EventConsumerHandler — receives all events of one
+// delivery turn in id order, meant for side-effect sinks (search index,
+// bulk fan-out) that can collapse a turn into one round-trip. `handler`
+// stays mandatory and is the fallback: if the batch throws, the dispatcher
+// re-delivers the same events one by one through `handler`, so errorPolicy
+// (maxAttempts, skipApplyErrors, dead-lettering) applies per event exactly
+// as before. The batch handler must therefore be replay-safe — a partially
+// applied batch gets re-applied event by event under the same idempotency
+// rule as `handler`. Events appended from inside a batch handler carry the
+// LAST event's causation id only.
+export type EventConsumerBatchHandler = (
+  events: readonly StoredEvent[],
+  ctx: AppContext,
+) => Promise<void>;
+
 // Per-consumer error policy. When skipApplyErrors is true and handler throws,
 // the dispatcher logs the error, advances the cursor past the offending event,
 // and keeps delivering — instead of the default retry + dead-letter flow.
@@ -84,6 +100,9 @@ export type EventConsumerErrorPolicy = {
 export type EventConsumer = {
   readonly name: string;
   readonly handler: EventConsumerHandler;
+  // Optional batch fast-path for this turn's events — see
+  // EventConsumerBatchHandler for the fallback and replay-safety contract.
+  readonly batchHandler?: EventConsumerBatchHandler;
   readonly errorPolicy?: EventConsumerErrorPolicy;
   // Owning feature — when present, the dispatcher skips this consumer's
   // pass while the feature is globally disabled. Events remain in the store
@@ -246,6 +265,15 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
   // entry — the next failure (if any) starts a fresh backoff from scratch.
   const consumerBackoff = new Map<string, { consecutiveFailures: number; retryAtMs: number }>();
 
+  // Dedup flag for the idle pre-check's error log (doPass) — a broken
+  // pre-check falls back to the expensive-but-correct path silently on
+  // every tick (see doPass), which must not mean silently in the logs too:
+  // ops needs to see it degrade once, not have every poll tick spam the
+  // same line. Cleared the moment the pre-check succeeds again, so a later,
+  // unrelated outage gets its own fresh log line instead of staying
+  // permanently suppressed by this process's flag.
+  let idlePreCheckErrorLogged = false;
+
   let running = false;
   // Separate from `running` on purpose: pre-registration of consumer state
   // rows is a one-time boot action, while running/timer/LISTEN is a
@@ -300,14 +328,16 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
   async function runConsumerTurn(
     consumer: EventConsumer,
     effective: ReadonlySet<string> | undefined,
+    idleKeys: ReadonlySet<string>,
   ): Promise<{ processed: number; failed: number }> {
     const key = `${consumer.name}:${consumerInstanceId(consumer, options.instanceId)}`;
     const existing = inFlightTurns.get(key);
     if (existing) return existing;
 
-    // Feature-gate and backoff-gate are resolved synchronously, before any
-    // promise is registered — a gated consumer must resolve without ever
-    // occupying inFlightTurns, or it would look "still running" forever.
+    // Feature-gate, backoff-gate and idle-gate are resolved synchronously,
+    // before any promise is registered — a gated consumer must resolve
+    // without ever occupying inFlightTurns, or it would look "still
+    // running" forever.
     if (effective && consumer.featureName && !effective.has(consumer.featureName)) {
       return { processed: 0, failed: 0 };
     }
@@ -315,6 +345,17 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
     if (backoff && backoff.retryAtMs > Date.now()) {
       return { processed: 0, failed: 0 };
     }
+    // Idle-gate: doPass already proved this consumer has nothing to do
+    // (see selectIdleConsumerKeys) — skip the turn without ever opening a
+    // db.begin/FOR UPDATE SKIP LOCKED (see doPass for why that matters).
+    // Clear any prior backoff same as a successful pass would: the
+    // backoff-gate above already confirmed retryAtMs has elapsed, and
+    // "provably nothing to deliver" is itself proof the consumer isn't
+    // presently failing.
+    if (idleKeys.has(key)) {
+      consumerBackoff.delete(key);
+      return { processed: 0, failed: 0 };
+    }
 
     // Registered before awaiting the slot — a turn queued behind the
     // concurrency limit still counts as in-flight, so it isn't started twice.
@@ -371,12 +412,65 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
     // SYSTEM_TENANT_ID returnt (typisch: union-of-all-tier-features).
     const effective = context.effectiveFeatures?.(SYSTEM_TENANT_ID);
 
+    // Lock-free pre-check, once per pass: which consumers are provably idle
+    // (no locking, no xid, no WAL — see selectIdleConsumerKeys). Read
+    // outside any transaction, before a single db.begin/FOR UPDATE SKIP
+    // LOCKED opens — an idle deployment (most consumers, most ticks)
+    // otherwise pays a row lock + WAL record + commit fsync per consumer
+    // per tick for nothing.
+    //
+    // A failure here (e.g. a transient DB blip) must not abort the whole
+    // pass — timer/LISTEN callers swallow doPass()'s rejection silently
+    // (see start()), which would mean no log, no persistConsumerPassFailure,
+    // and no backoff for every consumer this tick. Falling back to "nothing
+    // proven idle" instead routes every consumer through its normal
+    // acquireConsumerState path, whose own try/catch already covers this.
+    let idleKeys: ReadonlySet<string>;
+    try {
+      idleKeys = await selectIdleConsumerKeys(
+        db,
+        consumers.map((consumer) => ({
+          name: consumer.name,
+          instanceId: consumerInstanceId(consumer, options.instanceId),
+        })),
+      );
+      if (idlePreCheckErrorLogged) {
+        idlePreCheckErrorLogged = false;
+        const recoveredMsg =
+          "[event-dispatcher] idle pre-check recovered, database reachable again";
+        if (context.log) {
+          context.log.info(recoveredMsg);
+        } else {
+          console.log(recoveredMsg);
+        }
+      }
+    } catch (e) {
+      idleKeys = new Set();
+      // Log once per outage, not once per poll tick — see
+      // idlePreCheckErrorLogged above. Every consumer still gets delivered
+      // correctly this tick (fallback above), so this is a degradation
+      // signal for ops, not a delivery failure.
+      if (!idlePreCheckErrorLogged) {
+        idlePreCheckErrorLogged = true;
+        // The LISTEN peer talks to the same DB, so it is gone too; only a
+        // later onlisten may flip the gauge back to 1.
+        emitEventDispatcherListenConnected(meter, false);
+        const msg = e instanceof Error ? e.message : String(e);
+        const logMsg = `[event-dispatcher] idle pre-check failed, falling back to per-consumer locking every tick until it recovers: ${msg}`;
+        if (context.log) {
+          context.log.error(logMsg);
+        } else {
+          console.error(logMsg);
+        }
+      }
+    }
+
     // Every consumer runs its own turn concurrently (own TX, own cursor,
-    // own backoff/feature-gate) — a slow consumer no longer blocks the
-    // others' delivery until it commits, see runConsumerTurn.
+    // own backoff/feature-gate/idle-gate) — a slow consumer no longer
+    // blocks the others' delivery until it commits, see runConsumerTurn.
     const results = await Promise.all(
       consumers.map(async (consumer) => {
-        const perConsumer = await runConsumerTurn(consumer, effective);
+        const perConsumer = await runConsumerTurn(consumer, effective, idleKeys);
         return [consumer.name, perConsumer] as const;
       }),
     );
@@ -453,9 +547,14 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
           xminNow,
         );
 
-        // skip: nothing to deliver and no burnt gap to clean up — no
-        // markProcessing/persistConsumerOutcome write, so an idle consumer
-        // doesn't burn a WAL record on every poll tick.
+        // skip: nothing to deliver and no burnt gap to clean up. The
+        // doPass idle pre-check (selectIdleConsumerKeys) already keeps the
+        // common idle case out of this transaction entirely — reaching
+        // here with an empty fetch means the pre-check couldn't prove
+        // idleness (pending gaps forced a fetch, or a race window). By this
+        // point acquireConsumerState's FOR UPDATE has already taken the
+        // lock, so skipping markProcessing/persistConsumerOutcome only
+        // avoids extra writes, not that lock's own WAL record.
         if (events.length === 0 && burntGaps.length === 0) {
           span.setAttribute("consumer.skip_reason", "no_pending_events");
           return;
@@ -610,10 +709,11 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
       // fallback (dropped subscriptions, missed commits under load).
       //
       // Observability: the gauge kumiko_event_dispatcher_listen_connected
-      // flips to 1 on initial subscribe AND on every postgres.js silent
-      // reconnect (via the onlisten callback). A drop to 0 while running
-      // means delivery latency regressed from TCP-round-trip to
-      // pollIntervalMs — ops-visible.
+      // flips to 1 on initial subscribe and again on every reconnect that
+      // succeeds (onlisten callback). It drops to 0 on the initial connect
+      // failure, on stop() and when the idle pre-check detects a DB outage.
+      // An outage shorter than one poll tick is not detected; the gauge then
+      // stays at 1 until postgres.js's delayed re-LISTEN lands.
       emitEventDispatcherListenConnected(meter, false);
       if (options.pgClient) {
         try {
@@ -625,10 +725,14 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
               });
             },
             () => {
-              // Fires on initial connect AND on each reconnect. postgres.js
-              // reconnects transparently if the TCP connection drops, so the
-              // only way to see the recovery window is to flip the gauge
-              // every time this callback lands.
+              // Fires on initial connect and again when postgres.js's own
+              // re-LISTEN lands after a drop; during an outage that re-LISTEN
+              // waits in its connection backoff (measured 10-30s after the DB
+              // returns), polling covers delivery meanwhile. Never unlisten()
+              // + listen() by hand: that leaves duplicate listeners.
+              // ponytail: no own LISTEN reconnect, the backoff gap is polled;
+              // add a dedicated LISTEN connection if prod wires a pgClient
+              // and that latency matters.
               emitEventDispatcherListenConnected(meter, true);
             },
           );

package/src/pipeline/index.ts

@@ -28,6 +28,7 @@ export type {
   ConsumerRecoveryState,
   DispatcherPassResult,
   EventConsumer,
+  EventConsumerBatchHandler,
   EventConsumerHandler,
   EventDispatcher,
   EventDispatcherOptions,

package/src/pipeline/msp-rebuild.ts

@@ -3,6 +3,8 @@ import {
   markConsumerRebuildFailed,
   resetConsumerForMspRebuild,
   selectConsumerForUpdate,
+  selectEventIdHorizonWithMissingRanges,
+  selectSnapshotXmax,
   updateConsumerRebuildCursor,
 } from "../db/queries/event-consumer";
 import {
@@ -28,6 +30,7 @@ import { emitProjectionRebuild } from "../observability/standard-metrics";
 import type { Meter } from "../observability/types/metric";
 import { SHARED_INSTANCE_SENTINEL } from "./event-consumer-state";
 import type { MultiStreamApplyContext } from "./multi-stream-apply-context";
+import { capPendingGapsBelowCursor, subtractSortedIdsFromRanges } from "./pending-gap-ranges";
 import type { RebuildResult } from "./projection-rebuild";
 
 // Rebuild a multi-stream projection (MSP) from the event log. Symmetric to
@@ -61,6 +64,20 @@ import type { RebuildResult } from "./projection-rebuild";
 //   - Advance cursor to last processed event id, status=idle.
 //   - Swap the shadow into public (brief ACCESS EXCLUSIVE, not the replay).
 //
+// Handoff to the live dispatcher (#3231): the replay horizon (MAX(id) at the
+// time selectEventIdHorizonWithMissingRanges ran) plus the id gaps open below
+// it in that same snapshot are read up front and the replay is bounded to
+// `id <= horizon`. A transaction holding a lower id can still be in-flight
+// during replay — its row is invisible here and, once committed, would sit
+// below the cursor with no live-dispatcher gap tracking to ever revisit it.
+// Seeding pending_gaps closes that: ids actually read during replay are
+// carved out (delivered here, so a live pass must never redeliver them), and
+// whatever survives is capped below the written cursor — the live dispatcher
+// owns everything from `id > cursor` onward on its own. Those surviving
+// ranges are the shape of the snapshot taken before the replay, but their
+// xmax is read after the replay — a later xmax can only delay when the live
+// dispatcher burns the gap, never make it burn a still-in-flight range early.
+//
 // Failure: outer catch writes status="dead" + lastError so ops sees the
 // failure after the TX rolled back. Use restartConsumer to clear dead.
 //
@@ -145,10 +162,13 @@ export async function rebuildMultiStreamProjection(
     // Outside the rebuild tx, like the schema: idempotent DDL colliding
     // inside the tx would roll the whole replay back.
     if (skipApplyErrors) await createRebuildDeadLetterTable(db);
+    const replayedIds: bigint[] = [];
     await db.begin(async (tx: DbTx) => {
       await resetConsumerForMspRebuild(tx, mspName, SHARED_INSTANCE_SENTINEL);
       await selectConsumerForUpdate(tx, mspName, SHARED_INSTANCE_SENTINEL);
 
+      const { horizon, gaps: seededGaps } = await selectEventIdHorizonWithMissingRanges(tx);
+
       await assertLiveColumnsMatchMeta(tx, meta, mspName);
       await assertLiveTableHasNoRowLevelSecurity(tx, meta.tableName, mspName);
       await buildShadowTable(tx, meta);
@@ -168,10 +188,13 @@ export async function rebuildMultiStreamProjection(
           createdAt: Temporal.Instant;
           createdBy: string;
         };
+        // Events past `horizon` are the live dispatcher's to deliver via
+        // `id > cursor` — bounding the replay here is what keeps the two
+        // paths from ever double-applying the same event.
         const events = await selectMany<EventRow>(
           tx,
           eventsTable,
-          { type: [...subscribedTypes] },
+          { type: [...subscribedTypes], id: { lte: horizon } },
           { orderBy: { col: "id", direction: "asc" } },
         );
 
@@ -194,6 +217,7 @@ export async function rebuildMultiStreamProjection(
             db: tx,
             tenantId: row.tenantId,
           });
+          replayedIds.push(row.id);
           const applyFn = msp.apply[row.type];
           if (!applyFn) continue;
           if (skipApplyErrors) {
@@ -221,11 +245,25 @@ export async function rebuildMultiStreamProjection(
       if (skipped.length > 0) {
         await recordRebuildDeadLetters(tx, mspName, skipped);
       }
+      // Every id actually read this replay (applied or skip-applied) is
+      // resolved — a live pass must never redeliver it. Anything surviving
+      // that subtraction is capped below the written cursor: a gap at or
+      // above it is the live dispatcher's own `id > cursor` territory, not
+      // pending_gaps'. No events replayed (cursor stays 0) means no gap can
+      // sit below it either.
+      const survivingGaps = subtractSortedIdsFromRanges(seededGaps, replayedIds);
+      const cappedGaps = capPendingGapsBelowCursor(survivingGaps, lastProcessedEventId);
+      const postReplayXmax = cappedGaps.length > 0 ? await selectSnapshotXmax(tx) : null;
+      const pendingGaps =
+        postReplayXmax === null
+          ? cappedGaps
+          : cappedGaps.map((gap) => ({ ...gap, xmax: postReplayXmax }));
       await updateConsumerRebuildCursor(
         tx,
         mspName,
         SHARED_INSTANCE_SENTINEL,
         lastProcessedEventId,
+        pendingGaps,
       );
       await swapShadowIntoLive(tx, meta.tableName);
     });

package/src/pipeline/pending-gap-ranges.ts

@@ -70,3 +70,51 @@ export function splitRangeExcludingIds(
   if (cursor <= to) parts.push({ from: cursor.toString(), to: to.toString(), xmax: range.xmax });
   return parts;
 }
+
+// Same carve-out as splitRangeExcludingIds, but for many ranges against many
+// ids at once — an MSP rebuild's replay reads events in ascending id order,
+// so a single forward pointer over `sortedIds` (both inputs already sorted,
+// ranges non-overlapping) avoids scanning the full id list per range.
+export function subtractSortedIdsFromRanges(
+  ranges: readonly PendingGapEntry[],
+  sortedIds: readonly bigint[],
+): PendingGapEntry[] {
+  if (sortedIds.length === 0) return [...ranges];
+  const result: PendingGapEntry[] = [];
+  let cursor = 0;
+  for (const range of ranges) {
+    const from = BigInt(range.from);
+    const to = BigInt(range.to);
+    let id = sortedIds[cursor];
+    while (id !== undefined && id < from) {
+      cursor++;
+      id = sortedIds[cursor];
+    }
+    const idsInRange: bigint[] = [];
+    while (id !== undefined && id <= to) {
+      idsInRange.push(id);
+      cursor++;
+      id = sortedIds[cursor];
+    }
+    result.push(...splitRangeExcludingIds(range, idsInRange));
+  }
+  return result;
+}
+
+// Ranges above the written cursor belong to the live dispatcher's own
+// `id > cursor` territory; a range straddling it is truncated at cursor - 1.
+export function capPendingGapsBelowCursor(
+  gaps: readonly PendingGapEntry[],
+  cursor: bigint,
+): PendingGapEntry[] {
+  if (cursor <= 0n) return [];
+  const capped: PendingGapEntry[] = [];
+  for (const gap of gaps) {
+    const from = BigInt(gap.from);
+    if (from >= cursor) continue;
+    const to = BigInt(gap.to);
+    const cappedTo = to >= cursor ? cursor - 1n : to;
+    capped.push({ ...gap, to: cappedTo.toString() });
+  }
+  return capped;
+}