@cosmicdrift/kumiko-framework 0.306.0 → 0.308.0

package/src/pipeline/event-dispatcher-delivery.ts

@@ -2,7 +2,7 @@
 // `Temporal` TYPE that ConsumerStateRow.updatedAt/StoredEventRow.createdAt
 // resolve against (same #1438 dual-package-hazard pattern as event-store.ts).
 import { Temporal as TemporalPolyfill } from "temporal-polyfill";
-import { requestContext } from "../api/request-context";
+import { type RequestContextData, requestContext } from "../api/request-context";
 import type { DbConnection, DbTx } from "../db/connection";
 import {
   insertConsumerIfAbsent,
@@ -10,13 +10,18 @@ import {
   rearmDeadConsumer,
   recordConsumerPassFailure,
   selectConsumerForUpdateSkipLocked,
+  selectProvablyIdleConsumerPairs,
   updateConsumerDeliveryOutcome,
 } from "../db/queries/event-consumer";
-import { selectEventsHeadId } from "../db/queries/event-store";
-import { coerceRow, extractTableInfo, selectMany } from "../db/query";
+import {
+  type PendingIdRange,
+  selectEventsHeadId,
+  selectPendingAndNewEventRows,
+} from "../db/queries/event-store";
+import { coerceRow, extractTableInfo } from "../db/query";
 import { qnScope } from "../engine/qualified-name";
 import type { AppContext } from "../engine/types";
-import { eventsTable, toStoredEvent as rowToStoredEvent } from "../event-store";
+import { eventsTable, toStoredEvent as rowToStoredEvent, type StoredEvent } from "../event-store";
 import {
   emitDispatcherError,
   emitEventConsumerLag,
@@ -26,9 +31,18 @@ import {
 import {
   ConsumerStatuses,
   eventConsumerStateTable,
+  type PendingGapEntry,
   SHARED_INSTANCE_SENTINEL,
 } from "./event-consumer-state";
 import type { EventConsumer } from "./event-dispatcher";
+import { parseWriteOrigin } from "./write-origin";
+
+// Fails closed without throwing: a throw would poison the event for every consumer.
+const UNPARSEABLE_STORED_WRITE_ORIGIN = {
+  rootHandler: "<unknown>",
+  anonymousRoot: true,
+  publicIntake: false,
+} as const;
 
 // Per-consumer pass mechanics: acquire the state row, fetch pending events,
 // hand them to the consumer's handler in order, persist the outcome. Split
@@ -46,6 +60,7 @@ export type ConsumerStateRowShape = {
   readonly status: string;
   readonly attempts: number;
   readonly rearmCount: number;
+  readonly pendingGaps: readonly PendingGapEntry[];
   readonly lastError: string | null;
   readonly updatedAt: Temporal.Instant;
 };
@@ -145,6 +160,39 @@ export async function acquireConsumerState(
   return { state, skip: null };
 }
 
+// Read-only pre-check run once per doPass, before any consumer's turn opens
+// a transaction (event-dispatcher.ts's doPass/runConsumerTurn). Keyed like
+// consumerBackoff/inFlightTurns: `${name}:${instanceId}`. Consumers not
+// provably idle (missing row, dead, pending gaps, or an event past cursor)
+// simply don't appear in the result — the caller falls through to the
+// existing acquireConsumerState/FOR UPDATE SKIP LOCKED path unchanged.
+//
+// Race: an event that commits between this query and the next tick CAN
+// make this snapshot stale — that's expected, not a bug. What it can't do
+// is get silently lost: the append fires NOTIFY (or the next poll tick
+// fires regardless), triggering a fresh doPass. That pass's own pre-check
+// sees the new head id and excludes the consumer from "idle"; a new
+// pending_gap only ever gets created when a later turn's fetch (id > the
+// old cursor) walks past the row that committed late, and this same
+// pre-check reads pending_gaps live on every pass, so it observes that gap
+// too. In short: this can under-report idleness for one tick, never
+// over-report it.
+export async function selectIdleConsumerKeys(
+  db: DbConnection,
+  pairs: ReadonlyArray<{ readonly name: string; readonly instanceId: string }>,
+): Promise<ReadonlySet<string>> {
+  if (pairs.length === 0) return new Set();
+  const names = pairs.map((p) => p.name);
+  const instanceIds = pairs.map((p) => p.instanceId);
+  const idlePairs = await selectProvablyIdleConsumerPairs(
+    db,
+    names,
+    instanceIds,
+    ConsumerStatuses.dead,
+  );
+  return new Set(idlePairs.map((p) => `${p.name}:${p.instanceId}`));
+}
+
 // Shared pre-registration: one row per (consumer, shard), cursor = 0,
 // status = idle. Shared-delivery consumers use SHARED_INSTANCE_SENTINEL;
 // per-instance consumers use the dispatcher's instanceId. Idempotent
@@ -186,17 +234,20 @@ export async function markProcessing(tx: DbTx, name: string, instanceId: string)
   await markConsumerProcessing(tx, name, instanceId);
 }
 
+// `pendingRanges` are id ranges below `cursor` the consumer is still
+// watching as gaps (invisible on an earlier turn — see event-dispatcher.ts's
+// processConsumer). Fetching them alongside the plain `id > cursor` window
+// means a row that committed late becomes visible and deliverable the next
+// time this consumer's turn runs, instead of being permanently skipped.
 export async function fetchPendingEvents(
   tx: DbTx,
   cursor: bigint,
   batchSize: number,
+  pendingRanges: readonly PendingIdRange[] = [],
 ): Promise<ReadonlyArray<StoredEventRow>> {
-  return (await selectMany(
-    tx,
-    eventsTable,
-    { id: { gt: cursor } },
-    { orderBy: { col: "id", direction: "asc" }, limit: batchSize },
-  )) as ReadonlyArray<StoredEventRow>; // @cast-boundary db-row
+  const rawRows = await selectPendingAndNewEventRows(tx, cursor, pendingRanges, batchSize);
+  const info = extractTableInfo(eventsTable);
+  return rawRows.map((row) => coerceRow(row, info) as StoredEventRow); // @cast-boundary db-row
 }
 
 export type DeliveryOutcome = {
@@ -206,14 +257,22 @@ export type DeliveryOutcome = {
   readonly deadLettered: boolean;
   readonly processed: number;
   readonly failed: number;
+  // Which of the *pending* ids in `events` (id <= the cursor this delivery
+  // started from) got resolved this pass — delivered or skip-applied. The
+  // caller (event-dispatcher.ts) splits exactly these out of pending_gaps;
+  // everything else in the input batch was a "new" row past the old cursor.
+  readonly resolvedPendingIds: readonly bigint[];
 };
 
-// Deliver events to the consumer's handler in events.id order. Halt-on-
-// poison: a throw breaks the loop, the cursor stays at the last successful
-// event, and attempts climb. At the consumer's effectiveMaxAttempts
-// (errorPolicy.maxAttempts ?? maxAttempts) the caller persists status=
-// "dead" and the consumer is parked until ops intervenes (see
-// restartConsumer / skipPoisonEvent).
+// Deliver events to the consumer's handler in events.id order, or — when
+// the consumer wires a batchHandler — through it in one call first. Halt-
+// on-poison: a throw breaks the per-event loop, the cursor stays at the
+// last successful event, and attempts climb. At the consumer's
+// effectiveMaxAttempts (errorPolicy.maxAttempts ?? maxAttempts) the caller
+// persists status="dead" and the consumer is parked until ops intervenes
+// (see restartConsumer / skipPoisonEvent). A batch that throws falls back
+// to the same per-event loop for the same events — the batch failure
+// itself does not bump attempts, only a per-event failure does.
 export async function deliverEvents(
   consumer: EventConsumer,
   events: ReadonlyArray<StoredEventRow>,
@@ -221,83 +280,171 @@ export async function deliverEvents(
   maxAttempts: number,
   state: ConsumerStateRow,
 ): Promise<DeliveryOutcome> {
-  let cursor = state.lastProcessedEventId;
+  const startCursor = state.lastProcessedEventId;
+  let cursor = startCursor;
   let attempts = state.attempts;
   let lastError: string | null = state.lastError ?? null;
   let deadLettered = false;
   const effectiveMaxAttempts = consumer.errorPolicy?.maxAttempts ?? maxAttempts;
   let processed = 0;
   let failed = 0;
+  const resolvedPendingIds: bigint[] = [];
+
+  // A pending row sits below startCursor: it resolves its gap but never moves
+  // the cursor backward. ORDER BY id walks all pending rows first.
+  const resolve = (id: bigint): void => {
+    if (id > cursor) cursor = id;
+    if (id <= startCursor) resolvedPendingIds.push(id);
+    attempts = 0;
+    lastError = null;
+  };
+
+  if (consumer.batchHandler && events.length > 0) {
+    const batchSucceeded = await tryApplyBatch(consumer, events, context);
+    if (batchSucceeded) {
+      for (const row of events) resolve(row.id);
+      processed = events.length;
+      return { cursor, attempts, lastError, deadLettered, processed, failed, resolvedPendingIds };
+    }
+    // fall through to the per-event loop below; the batch failure itself
+    // does NOT bump attempts — only a per-event failure does.
+  }
 
   for (const row of events) {
     try {
-      // Propagate causation: if the handler calls ctx.appendEvent, the new
-      // event should record THIS event as its cause. correlationId is
-      // inherited unchanged — it survives the hop across streams by design.
-      // requestId falls back to a fresh id because the dispatcher runs
-      // outside any HTTP request (background poll), and a stable log-
-      // correlation handle is still useful for debugging.
-      const stored = rowToStoredEvent(row);
-      const correlationId = stored.metadata.correlationId ?? requestContext.generateId();
-      const causationId = String(stored.id);
-      const requestId = requestContext.generateId();
-      // #3043 — an event this apply writes is attributed to the consumer, not
-      // to whatever wrote the triggering event; causationId already links back.
-      await requestContext.run(
-        {
-          requestId,
-          correlationId,
-          causationId,
-          handler: consumer.name,
-          feature: consumer.featureName ?? qnScope(consumer.name),
-        },
-        async () => {
-          await consumer.handler(stored, context);
-        },
-      );
-      cursor = row.id;
-      attempts = 0;
-      lastError = null;
+      await applyEvent(consumer, row, context);
+      resolve(row.id);
       processed += 1;
     } catch (e) {
       const errMessage = e instanceof Error ? e.message : String(e);
+      failed += 1;
       if (consumer.errorPolicy?.skipApplyErrors) {
-        // Best-effort mode: record the error on the skip counter so ops
-        // can alert on a spike of skipped events, advance the cursor past
-        // the bad event, keep going. The consumer stays "idle", not "dead".
-        // Also emit a warn-level log line — the metric tells ops THAT events
-        // are being dropped, the log tells them WHICH events. Without this
-        // a poisoned-then-skipped event is invisible to forensic search.
-        const errorClass = e instanceof Error ? e.constructor.name : "UnknownError";
-        emitDispatcherError(context.meter ?? getFallbackMeter(), {
-          handler: consumer.name,
-          errorClass,
-        });
-        context.log?.warn(
-          `event-dispatcher: ${consumer.name} skipped event ${row.id} (${errorClass}): ${errMessage}`,
-        );
-        cursor = row.id;
-        attempts = 0;
-        lastError = null;
-        failed += 1;
+        reportSkippedEvent(consumer, row.id, e, errMessage, context);
+        resolve(row.id);
         continue;
       }
       attempts += 1;
       lastError = errMessage;
-      failed += 1;
       if (attempts >= effectiveMaxAttempts) deadLettered = true;
       break;
     }
   }
 
-  return { cursor, attempts, lastError, deadLettered, processed, failed };
+  return { cursor, attempts, lastError, deadLettered, processed, failed, resolvedPendingIds };
+}
+
+// Shared requestContext scope for one apply — event-derived fields
+// (correlationId, causationId, writeOrigin) come from `causationSource`;
+// applyEvent uses the event itself, applyBatch uses the LAST event of the
+// turn (see applyBatch for why). requestId falls back to a fresh id
+// because the dispatcher runs outside any HTTP request (background poll),
+// and a stable log-correlation handle is still useful for debugging.
+// #3043 — an event this apply writes is attributed to the consumer, not
+// to whatever wrote the triggering event; causationId already links back.
+function buildConsumerRequestScope(
+  consumer: EventConsumer,
+  causationSource: StoredEvent,
+): RequestContextData {
+  const correlationId = causationSource.metadata.correlationId ?? requestContext.generateId();
+  const causationId = String(causationSource.id);
+  const requestId = requestContext.generateId();
+  // The job-trigger consumer's handleEvent stamps event-triggered jobs from this.
+  const rawStoredWriteOrigin = causationSource.metadata.writeOrigin;
+  const writeOrigin =
+    rawStoredWriteOrigin === undefined
+      ? undefined
+      : (parseWriteOrigin(rawStoredWriteOrigin) ?? UNPARSEABLE_STORED_WRITE_ORIGIN);
+  return {
+    requestId,
+    correlationId,
+    causationId,
+    handler: consumer.name,
+    feature: consumer.featureName ?? qnScope(consumer.name),
+    writeOrigin,
+  };
+}
+
+async function applyEvent(
+  consumer: EventConsumer,
+  row: StoredEventRow,
+  context: AppContext,
+): Promise<void> {
+  const stored = rowToStoredEvent(row);
+  await requestContext.run(buildConsumerRequestScope(consumer, stored), async () => {
+    await consumer.handler(stored, context);
+  });
 }
 
+// Runs the consumer's batchHandler for the whole turn under one
+// requestContext scope, attempted before the per-event loop. Returns
+// false on failure instead of rethrowing — the caller falls back to
+// per-event delivery for the exact same events, so a batch failure must
+// not itself count as an applyEvent failure or bump attempts.
+async function tryApplyBatch(
+  consumer: EventConsumer,
+  events: ReadonlyArray<StoredEventRow>,
+  context: AppContext,
+): Promise<boolean> {
+  try {
+    await applyBatch(consumer, events, context);
+    return true;
+  } catch (e) {
+    const errMessage = e instanceof Error ? e.message : String(e);
+    context.log?.warn(
+      `event-dispatcher: ${consumer.name} batch of ${events.length} failed, falling back to per-event delivery: ${errMessage}`,
+    );
+    return false;
+  }
+}
+
+async function applyBatch(
+  consumer: EventConsumer,
+  events: ReadonlyArray<StoredEventRow>,
+  context: AppContext,
+): Promise<void> {
+  const batchHandler = consumer.batchHandler;
+  // skip: deliverEvents only calls this for consumers that wire a batchHandler.
+  if (!batchHandler) return;
+  const stored = events.map((row) => rowToStoredEvent(row));
+  // correlationId/causationId/writeOrigin come from the LAST row — a
+  // batch-appended event's causation is attributed to the turn's most
+  // recent input, same as chaining N applyEvent calls would leave the
+  // requestContext at after the final one.
+  const lastStored = stored.at(-1);
+  // skip: deliverEvents never calls this with an empty turn.
+  if (!lastStored) return;
+  await requestContext.run(buildConsumerRequestScope(consumer, lastStored), async () => {
+    await batchHandler(stored, context);
+  });
+}
+
+// Best-effort mode: record the error on the skip counter so ops can alert on
+// a spike of skipped events; the consumer stays "idle", not "dead". The
+// warn-level log line tells them WHICH events — without it a
+// poisoned-then-skipped event is invisible to forensic search.
+function reportSkippedEvent(
+  consumer: EventConsumer,
+  eventId: bigint,
+  e: unknown,
+  errMessage: string,
+  context: AppContext,
+): void {
+  const errorClass = e instanceof Error ? e.constructor.name : "UnknownError";
+  emitDispatcherError(context.meter ?? getFallbackMeter(), { handler: consumer.name, errorClass });
+  context.log?.warn(
+    `event-dispatcher: ${consumer.name} skipped event ${eventId} (${errorClass}): ${errMessage}`,
+  );
+}
+
+export type PersistedConsumerOutcome = DeliveryOutcome & {
+  readonly pendingGaps: readonly PendingGapEntry[];
+};
+
 export async function persistConsumerOutcome(
   tx: DbTx,
   name: string,
   instanceId: string,
-  outcome: DeliveryOutcome,
+  outcome: PersistedConsumerOutcome,
 ): Promise<void> {
   await updateConsumerDeliveryOutcome(tx, name, instanceId, outcome);
 }