@cosmicdrift/kumiko-framework 1.0.0 → 2.0.0

package/src/pipeline/entity-cache.ts

@@ -1,3 +1,4 @@
+import type { EntityCache } from "@cosmicdrift/kumiko-types/entity-cache";
 import type Redis from "ioredis";
 import type { EntityId, TenantId } from "../engine/types/identifiers";
 import { RedisKeys } from "./redis-keys";
@@ -23,39 +24,7 @@ function parseCached(raw: string): Record<string, unknown> | null {
   }
 }
 
-export type EntityCache = {
-  /** Get a single cached entity. Returns null on miss. */
-  get(
-    tenantId: TenantId,
-    entityName: string,
-    id: EntityId,
-  ): Promise<Record<string, unknown> | null>;
-
-  /** Get multiple cached entities. Returns a Map of id → data (misses are absent). */
-  mget(
-    tenantId: TenantId,
-    entityName: string,
-    ids: readonly EntityId[],
-  ): Promise<Map<EntityId, Record<string, unknown>>>;
-
-  /** Cache a single entity. */
-  set(
-    tenantId: TenantId,
-    entityName: string,
-    id: EntityId,
-    data: Record<string, unknown>,
-  ): Promise<void>;
-
-  /** Cache multiple entities at once. */
-  mset(
-    tenantId: TenantId,
-    entityName: string,
-    entries: ReadonlyArray<{ id: EntityId; data: Record<string, unknown> }>,
-  ): Promise<void>;
-
-  /** Invalidate a single cached entity. */
-  del(tenantId: TenantId, entityName: string, id: EntityId): Promise<void>;
-};
+export type { EntityCache } from "@cosmicdrift/kumiko-types/entity-cache";
 
 export type EntityCacheOptions = {
   ttlSeconds?: number;

package/src/pipeline/event-consumer-state.ts

@@ -10,6 +10,7 @@ import {
   sql,
   text,
 } from "../db/dialect";
+import { alterTableAddColumn } from "../db/queries/ddl";
 import { tableExists } from "../db/schema-inspection";
 import { unsafePushTables } from "../stack";
 
@@ -72,6 +73,14 @@ export const eventConsumerStateTable = pgTable(
       .default(sql`0`),
     status: text("status").notNull().default("idle"),
     attempts: integer("attempts").notNull().default(0),
+    // Counts automatic dead→idle revivals since the last delivery that
+    // advanced the cursor (event-dispatcher.ts's cooldown re-arm, see
+    // acquireConsumerState). A poison event never advances the cursor, so
+    // this climbs to maxRearmCount and then stays dead permanently. A
+    // successful delivery resets it to 0 (proof the consumer isn't
+    // poisoned), as does a manual restartConsumer()/enableConsumer()/
+    // skipPoisonEvent() — an operator vouching the consumer is healthy.
+    rearmCount: integer("rearm_count").notNull().default(0),
     lastError: text("last_error"),
     updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql`now()`),
   },
@@ -99,7 +108,26 @@ export type ConsumerStatus = (typeof ConsumerStatuses)[keyof typeof ConsumerStat
 //
 // guard:dup-ok — intentionale Parallele zu createProjectionStateTable; symmetrische State-Tabellen by design
 export async function createEventConsumerStateTable(db: DbConnection): Promise<void> {
-  // skip: table already exists — bootstrap is called from multiple paths
-  if (await tableExists(db, "public.kumiko_event_consumers")) return;
+  // skip: table already exists — bootstrap is called from multiple paths.
+  // Still check for columns added after the table's first deploy (e.g.
+  // rearm_count) so an older DB catches up without a dedicated migration.
+  if (await tableExists(db, "public.kumiko_event_consumers")) {
+    // ADD COLUMN IF NOT EXISTS is idempotent and race-safe on its own —
+    // no need to pre-check columnNamesOf first. Two instances booting
+    // concurrently against the same DB (this table is explicitly
+    // multi-instance) both ALTER, neither crashes on "column already
+    // exists" (#1362).
+    await alterTableAddColumn(
+      db,
+      "kumiko_event_consumers",
+      "rearm_count",
+      "integer",
+      " DEFAULT 0",
+      " NOT NULL",
+      /* ifNotExists */ true,
+    );
+    // skip: table (+ any missing column) is already up to date
+    return;
+  }
   await unsafePushTables(db, { kumikoEventConsumers: eventConsumerStateTable });
 }

package/src/pipeline/event-dispatcher-admin.ts

@@ -0,0 +1,293 @@
+import type { DbConnection, DbTx } from "../db/connection";
+import {
+  advanceConsumerPastEventReturning,
+  updateConsumerStatusReturning,
+} from "../db/queries/event-consumer";
+import { selectNextEventIdAfter } from "../db/queries/event-store";
+import { coerceRow, extractTableInfo, selectMany } from "../db/query";
+import { getEventsHighWaterMark } from "../event-store";
+import { eventConsumerStateTable, SHARED_INSTANCE_SENTINEL } from "./event-consumer-state";
+import type { ConsumerStateRow, ConsumerStateRowShape } from "./event-dispatcher-delivery";
+
+// --- Ops recovery surface ---
+//
+// These are intentionally verb-distinct; each maps to a CLI sub-command.
+// They all target a single consumer row by name. Every call returns the
+// state after the write so the CLI can echo what actually changed.
+//
+// Semantics:
+//   restartConsumer   status="dead" → "idle", attempts=0, lastError=null.
+//                     Cursor unchanged → next pass retries the SAME event
+//                     that poisoned the consumer. For transient failures.
+//   disableConsumer   status=* → "disabled". Dispatcher skips this consumer
+//                     until enableConsumer() flips it back.
+//   enableConsumer    status="disabled" → "idle". No-op on any other state.
+//   skipPoisonEvent   cursor advances past the first event after the
+//                     current cursor (the one that's failing). attempts=0,
+//                     lastError=null, status="idle". For events that will
+//                     never succeed (broken payload, removed feature code).
+
+function normalizeConsumerState(row: ConsumerStateRowShape): ConsumerRecoveryState {
+  return {
+    name: row.name,
+    instanceId: row.instanceId,
+    status: row.status,
+    lastProcessedEventId: row.lastProcessedEventId,
+    attempts: row.attempts,
+    rearmCount: row.rearmCount,
+    lastError: row.lastError,
+    updatedAt: row.updatedAt,
+  };
+}
+
+export type ConsumerRecoveryState = {
+  readonly name: string;
+  readonly instanceId: string;
+  readonly status: string;
+  readonly lastProcessedEventId: bigint;
+  readonly attempts: number;
+  readonly rearmCount: number;
+  readonly lastError: string | null;
+  readonly updatedAt: Temporal.Instant;
+};
+
+// Ops calls default to the SHARED_INSTANCE_SENTINEL row — that's the only
+// row shared-delivery consumers have, so legacy CLI invocations without
+// --instance-id keep working. Per-instance consumers require an explicit
+// instanceId: picking one of N shards arbitrarily ("first row wins") or
+// mutating all shards simultaneously ("bounce every instance") are both
+// worse than a loud missing-arg error on the CLI.
+async function requireConsumerRow(
+  db: DbConnection,
+  name: string,
+  instanceId: string,
+): Promise<ConsumerStateRowShape> {
+  const [row] = await selectMany<ConsumerStateRow>(db, eventConsumerStateTable, {
+    name,
+    instanceId,
+  });
+  if (!row) {
+    throw new Error(
+      `Consumer "${name}" (instance_id="${instanceId}") has no state row — it hasn't run yet, the name is misspelled, or the instance is misspelled. ` +
+        `For per-instance consumers pass the instance_id explicitly; shared consumers use the default.`,
+    );
+  }
+  return row;
+}
+
+async function applyConsumerStatusTransition(
+  db: DbConnection,
+  name: string,
+  instanceId: string,
+  targetStatus: "idle" | "disabled",
+): Promise<ConsumerRecoveryState> {
+  const raw = await updateConsumerStatusReturning(db, name, instanceId, targetStatus);
+  const updated =
+    raw && (coerceRow(raw, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow);
+  if (!updated) {
+    throw new Error(
+      `Consumer "${name}" (instance_id="${instanceId}") vanished between read and write — retry.`,
+    );
+  }
+  return normalizeConsumerState(updated);
+}
+
+export async function restartConsumer(
+  db: DbConnection,
+  name: string,
+  instanceId: string = SHARED_INSTANCE_SENTINEL,
+): Promise<ConsumerRecoveryState> {
+  const before = await requireConsumerRow(db, name, instanceId);
+  if (before.status !== "dead") {
+    throw new Error(
+      `Consumer "${name}" (instance_id="${instanceId}") is not dead (status="${before.status}"). Restart only applies to dead consumers; use "enable" for a disabled one.`,
+    );
+  }
+  return applyConsumerStatusTransition(db, name, instanceId, "idle");
+}
+
+export async function disableConsumer(
+  db: DbConnection,
+  name: string,
+  instanceId: string = SHARED_INSTANCE_SENTINEL,
+): Promise<ConsumerRecoveryState> {
+  await requireConsumerRow(db, name, instanceId);
+  return applyConsumerStatusTransition(db, name, instanceId, "disabled");
+}
+
+export async function enableConsumer(
+  db: DbConnection,
+  name: string,
+  instanceId: string = SHARED_INSTANCE_SENTINEL,
+): Promise<ConsumerRecoveryState> {
+  const before = await requireConsumerRow(db, name, instanceId);
+  if (before.status !== "disabled") {
+    throw new Error(
+      `Consumer "${name}" (instance_id="${instanceId}") is not disabled (status="${before.status}"). Enable only flips disabled → idle; use "restart" for a dead consumer.`,
+    );
+  }
+  return applyConsumerStatusTransition(db, name, instanceId, "idle");
+}
+
+// skipPoisonEvent advances the cursor past the first event after the
+// current cursor. Single TX so concurrent dispatcher passes can't double-
+// advance. If no event exists past the cursor, there is nothing to skip —
+// treat as idempotent no-op (cursor already at head).
+export async function skipPoisonEvent(
+  db: DbConnection,
+  name: string,
+  instanceId: string = SHARED_INSTANCE_SENTINEL,
+): Promise<ConsumerRecoveryState & { readonly skippedEventId: bigint | null }> {
+  const before = await requireConsumerRow(db, name, instanceId);
+  return db.begin(async (tx: DbTx) => {
+    const poisonId = await selectNextEventIdAfter(tx, before.lastProcessedEventId);
+    if (poisonId === null) {
+      const [unchanged] = await selectMany<ConsumerStateRow>(tx, eventConsumerStateTable, {
+        name,
+        instanceId,
+      });
+      if (!unchanged)
+        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished — retry.`);
+      return { ...normalizeConsumerState(unchanged), skippedEventId: null };
+    }
+    const raw = await advanceConsumerPastEventReturning(tx, name, instanceId, poisonId);
+    const updated =
+      raw && (coerceRow(raw, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow);
+    if (!updated)
+      throw new Error(
+        `Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`,
+      );
+    return { ...normalizeConsumerState(updated), skippedEventId: poisonId };
+  });
+}
+
+// Read-only status for one consumer shard — CLI surface.
+export async function getConsumerState(
+  db: DbConnection,
+  name: string,
+  instanceId: string = SHARED_INSTANCE_SENTINEL,
+): Promise<{
+  readonly name: string;
+  readonly instanceId: string;
+  readonly status: string;
+  readonly lastProcessedEventId: bigint;
+  readonly attempts: number;
+  readonly rearmCount: number;
+  readonly lastError: string | null;
+  readonly updatedAt: Temporal.Instant;
+} | null> {
+  const [row] = await selectMany<ConsumerStateRow>(db, eventConsumerStateTable, {
+    name,
+    instanceId,
+  });
+  if (!row) return null;
+  return {
+    name: row.name,
+    instanceId: row.instanceId,
+    status: row.status,
+    lastProcessedEventId: row.lastProcessedEventId,
+    attempts: row.attempts,
+    rearmCount: row.rearmCount,
+    lastError: row.lastError,
+    updatedAt: row.updatedAt,
+  };
+}
+
+// List every consumer the registry knows about, joined with all shard rows
+// from the state table. One entry per (name, instance_id) shard. Consumers
+// that have never run appear with status="never-run" and instance_id =
+// SHARED_INSTANCE_SENTINEL — a placeholder, because without a running
+// dispatcher we can't know the instance-ids of per-instance consumers yet.
+// Mirrors listProjectionsWithState — the registry (not the DB) is the
+// source-of-truth for which consumer-names exist; the DB is the source-
+// of-truth for which instance-shards have been seen.
+export async function listConsumersWithState(
+  db: DbConnection,
+  registeredNames: readonly string[],
+): Promise<
+  ReadonlyArray<{
+    readonly name: string;
+    readonly instanceId: string;
+    readonly status: string;
+    readonly lastProcessedEventId: bigint;
+    readonly attempts: number;
+    readonly lastError: string | null;
+  }>
+> {
+  const stateRows = await selectMany<ConsumerStateRow>(db, eventConsumerStateTable);
+  const registered = new Set(registeredNames);
+
+  // Materialize one output row per (name, instance_id). Registered names
+  // without any shard (never-run) get a placeholder row so ops can still
+  // see the name exists.
+  const out: Array<{
+    name: string;
+    instanceId: string;
+    status: string;
+    lastProcessedEventId: bigint;
+    attempts: number;
+    lastError: string | null;
+  }> = [];
+
+  const seenNames = new Set<string>();
+  for (const r of stateRows) {
+    if (!registered.has(r.name)) continue; // stale row from an older deploy
+    seenNames.add(r.name);
+    out.push({
+      name: r.name,
+      instanceId: r.instanceId,
+      status: r.status,
+      lastProcessedEventId: r.lastProcessedEventId,
+      attempts: r.attempts,
+      lastError: r.lastError,
+    });
+  }
+  for (const name of registeredNames) {
+    if (seenNames.has(name)) continue;
+    out.push({
+      name,
+      instanceId: SHARED_INSTANCE_SENTINEL,
+      status: "never-run",
+      lastProcessedEventId: 0n,
+      attempts: 0,
+      lastError: null,
+    });
+  }
+  return out;
+}
+
+export type ConsumerProgress = {
+  readonly name: string;
+  readonly instanceId: string;
+  readonly status: string;
+  readonly lastProcessedEventId: bigint;
+  readonly attempts: number;
+  readonly lastError: string | null;
+  // Global MAX(events.id) at query time.
+  readonly highWaterMark: bigint;
+  // HWM - cursor. 0n when caught-up. Disabled consumers often show high
+  // lag intentionally (ops parks them before pruning).
+  readonly lag: bigint;
+};
+
+// Like listConsumersWithState, but also returns HWM + lag per consumer.
+// Async consumers (MSPs) lag behind inline projections because they run
+// post-commit — lag is the primary signal for backpressure, dead consumers,
+// or dispatcher stalls. Programmatic callers can map the result to a
+// `kumiko_consumer_lag{name}` Prometheus gauge.
+// guard:dup-ok — intentionale Parallele zu getAllProjectionProgress; Consumer ≠ Projection (verschiedene Subsysteme)
+export async function getAllConsumerProgress(
+  db: DbConnection,
+  registeredNames: readonly string[],
+): Promise<readonly ConsumerProgress[]> {
+  const [consumers, highWaterMark] = await Promise.all([
+    listConsumersWithState(db, registeredNames),
+    getEventsHighWaterMark(db),
+  ]);
+
+  return consumers.map((c) => ({
+    ...c,
+    highWaterMark,
+    lag: highWaterMark - c.lastProcessedEventId,
+  }));
+}

package/src/pipeline/event-dispatcher-delivery.ts

@@ -0,0 +1,305 @@
+// Value-only import, aliased to avoid shadowing the ambient global
+// `Temporal` TYPE that ConsumerStateRow.updatedAt/StoredEventRow.createdAt
+// resolve against (same #1438 dual-package-hazard pattern as event-store.ts).
+import { Temporal as TemporalPolyfill } from "temporal-polyfill";
+import { requestContext } from "../api/request-context";
+import type { DbConnection, DbTx } from "../db/connection";
+import {
+  insertConsumerIfAbsent,
+  markConsumerProcessing,
+  rearmDeadConsumer,
+  selectConsumerForUpdateSkipLocked,
+  updateConsumerDeliveryOutcome,
+} from "../db/queries/event-consumer";
+import { selectEventsHeadId } from "../db/queries/event-store";
+import { coerceRow, extractTableInfo, selectMany } from "../db/query";
+import type { AppContext } from "../engine/types";
+import { eventsTable, toStoredEvent as rowToStoredEvent } from "../event-store";
+import {
+  emitDispatcherError,
+  emitEventConsumerLag,
+  getFallbackMeter,
+  type Meter,
+} from "../observability";
+import {
+  ConsumerStatuses,
+  eventConsumerStateTable,
+  SHARED_INSTANCE_SENTINEL,
+} from "./event-consumer-state";
+import type { EventConsumer } from "./event-dispatcher";
+
+// Per-consumer pass mechanics: acquire the state row, fetch pending events,
+// hand them to the consumer's handler in order, persist the outcome. Split
+// out of event-dispatcher.ts so the delivery loop is independently readable
+// from the public lifecycle surface (start/stop/runOnce) and the ops
+// recovery surface (event-dispatcher-admin.ts).
+//
+// Free functions (not closures) — every helper takes an explicit `tx`, none
+// use the outer dispatcher's closure state.
+
+export type ConsumerStateRowShape = {
+  readonly name: string;
+  readonly instanceId: string;
+  readonly lastProcessedEventId: bigint;
+  readonly status: string;
+  readonly attempts: number;
+  readonly rearmCount: number;
+  readonly lastError: string | null;
+  readonly updatedAt: Temporal.Instant;
+};
+export type ConsumerStateRow = ConsumerStateRowShape;
+
+export type StoredEventRow = {
+  readonly id: bigint;
+  readonly aggregateId: string;
+  readonly aggregateType: string;
+  readonly tenantId: string;
+  readonly version: number;
+  readonly type: string;
+  readonly eventVersion: number;
+  readonly payload: Record<string, unknown>;
+  readonly metadata: import("../event-store/event-store").EventMetadata;
+  readonly createdAt: Temporal.Instant;
+  readonly createdBy: string;
+};
+
+export type AcquireOutcome =
+  | { readonly state: ConsumerStateRow; readonly skip: null }
+  | {
+      readonly state: null;
+      readonly skip: "locked_by_other_instance" | "disabled" | "dead" | "not_registered";
+    };
+
+// Lock the consumer's state row with SKIP LOCKED. Strict: no in-tx bootstrap.
+// The row must exist — start() pre-registers every consumer up front so
+// prune (event-retention) sees their cursors as soon as the process is up,
+// closing the race where a lazy-bootstrapped consumer's cursor is absent
+// during prune and its events are silently deleted.
+//
+// skip="not_registered" signals a row-missing-despite-start condition.
+// Production shouldn't hit this — it means either start() wasn't called
+// (runOnce() guards against that) or the state row was deleted externally
+// (a test TRUNCATE without subsequent ensureRegistered(), or an operator
+// intervention). Skipping quietly preserves the dispatcher's other
+// consumers and surfaces the issue via the metrics pass-outcome.
+export async function acquireConsumerState(
+  tx: DbTx,
+  name: string,
+  instanceId: string,
+  rearmCooldownMs: number,
+  maxRearmCount: number,
+): Promise<AcquireOutcome> {
+  const rawState = await selectConsumerForUpdateSkipLocked(tx, name, instanceId);
+
+  if (!rawState) {
+    return { state: null, skip: "not_registered" };
+  }
+
+  const state = coerceRow(rawState, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow;
+
+  if (!state) {
+    // Either the row never existed (no pre-reg, no ensureRegistered) or
+    // another instance currently holds the lock with SKIP LOCKED filtering
+    // us out. We can't distinguish here in a single query, so return
+    // "not_registered" — ops sees a skip-reason instead of silent delivery
+    // loss. Under normal operation (start() called, no external tampering)
+    // this path is never taken.
+    return { state: null, skip: "not_registered" };
+  }
+
+  if (state.status === ConsumerStatuses.disabled) return { state: null, skip: "disabled" };
+  if (state.status === ConsumerStatuses.dead) {
+    // Bounded auto-revival: a transient failure (e.g. a Meilisearch blip)
+    // shouldn't need an operator to notice and run restartConsumer() once
+    // the cause is long gone. Cooldown since the last write (the death or
+    // a prior re-arm) gates the retry; maxRearmCount stops a poison event
+    // from looping forever (re-arm → same event fails → dead → re-arm →
+    // ...) — after the cap it stays dead until a human intervenes.
+    const cooldownDeadline = TemporalPolyfill.Now.instant().subtract({
+      milliseconds: rearmCooldownMs,
+    });
+    // @cast-boundary temporal-polyfill-vs-ambient: same TC39 Temporal.Instant
+    // at runtime — state.updatedAt is DB-row-typed against the ambient
+    // global, two distinct nominal types across the two .d.ts sources (see
+    // event-store.ts).
+    const cooldownElapsed =
+      TemporalPolyfill.Instant.compare(
+        state.updatedAt as unknown as InstanceType<typeof TemporalPolyfill.Instant>,
+        cooldownDeadline,
+      ) <= 0;
+    if (cooldownElapsed && state.rearmCount < maxRearmCount) {
+      const rearmed = await rearmDeadConsumer(tx, name, instanceId);
+      const rearmedState =
+        rearmed &&
+        (coerceRow(rearmed, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow);
+      if (rearmedState) return { state: rearmedState, skip: null };
+    }
+    // Caller (event-dispatcher.ts's processConsumer) emits
+    // kumiko_event_consumer_rearm_exhausted_total once per (consumer,
+    // instance) transition into this branch — it has the process-lifetime
+    // state to dedupe across poll passes that this pure function doesn't.
+    return { state: null, skip: "dead" };
+  }
+  return { state, skip: null };
+}
+
+// Shared pre-registration: one row per (consumer, shard), cursor = 0,
+// status = idle. Shared-delivery consumers use SHARED_INSTANCE_SENTINEL;
+// per-instance consumers use the dispatcher's instanceId. Idempotent
+// under restart and concurrent start-calls via ON CONFLICT DO NOTHING
+// on the composite PK — never clobbers an existing cursor.
+export async function preRegisterConsumers(
+  db: DbConnection,
+  consumers: readonly EventConsumer[],
+  dispatcherInstanceId: string | undefined,
+): Promise<void> {
+  for (const consumer of consumers) {
+    const instanceId = consumerInstanceId(consumer, dispatcherInstanceId);
+    await insertConsumerIfAbsent(db, consumer.name, instanceId);
+  }
+}
+
+// Resolve the instance_id column value for one consumer on this dispatcher.
+// Shared stays at the sentinel; per-instance rides the dispatcher's id.
+// Throws when a per-instance consumer is registered without an instanceId
+// — missing at boot is the sharp-edge to catch, not at first delivery.
+export function consumerInstanceId(
+  consumer: EventConsumer,
+  dispatcherInstanceId: string | undefined,
+): string {
+  if (consumer.delivery !== "per-instance") return SHARED_INSTANCE_SENTINEL;
+  if (!dispatcherInstanceId) {
+    throw new Error(
+      `EventConsumer "${consumer.name}" has delivery="per-instance" but the dispatcher was created without an instanceId — ` +
+        `pass EventDispatcherOptions.instanceId (typically from ServerOptions.instanceId / KUMIKO_INSTANCE_ID).`,
+    );
+  }
+  return dispatcherInstanceId;
+}
+
+// Mark the consumer row as "processing" for ops visibility. The SKIP LOCKED
+// lock already guarantees single-writer semantics; this is purely
+// informational (and resets on commit to idle/dead via persistConsumerOutcome).
+export async function markProcessing(tx: DbTx, name: string, instanceId: string): Promise<void> {
+  await markConsumerProcessing(tx, name, instanceId);
+}
+
+export async function fetchPendingEvents(
+  tx: DbTx,
+  cursor: bigint,
+  batchSize: number,
+): Promise<ReadonlyArray<StoredEventRow>> {
+  return (await selectMany(
+    tx,
+    eventsTable,
+    { id: { gt: cursor } },
+    { orderBy: { col: "id", direction: "asc" }, limit: batchSize },
+  )) as ReadonlyArray<StoredEventRow>; // @cast-boundary db-row
+}
+
+export type DeliveryOutcome = {
+  readonly cursor: bigint;
+  readonly attempts: number;
+  readonly lastError: string | null;
+  readonly deadLettered: boolean;
+  readonly processed: number;
+  readonly failed: number;
+};
+
+// Deliver events to the consumer's handler in events.id order. Halt-on-
+// poison: a throw breaks the loop, the cursor stays at the last successful
+// event, and attempts climb. At the consumer's effectiveMaxAttempts
+// (errorPolicy.maxAttempts ?? maxAttempts) the caller persists status=
+// "dead" and the consumer is parked until ops intervenes (see
+// restartConsumer / skipPoisonEvent).
+export async function deliverEvents(
+  consumer: EventConsumer,
+  events: ReadonlyArray<StoredEventRow>,
+  context: AppContext,
+  maxAttempts: number,
+  state: ConsumerStateRow,
+): Promise<DeliveryOutcome> {
+  let cursor = state.lastProcessedEventId;
+  let attempts = state.attempts;
+  let lastError: string | null = state.lastError ?? null;
+  let deadLettered = false;
+  const effectiveMaxAttempts = consumer.errorPolicy?.maxAttempts ?? maxAttempts;
+  let processed = 0;
+  let failed = 0;
+
+  for (const row of events) {
+    try {
+      // Propagate causation: if the handler calls ctx.appendEvent, the new
+      // event should record THIS event as its cause. correlationId is
+      // inherited unchanged — it survives the hop across streams by design.
+      // requestId falls back to a fresh id because the dispatcher runs
+      // outside any HTTP request (background poll), and a stable log-
+      // correlation handle is still useful for debugging.
+      const stored = rowToStoredEvent(row);
+      const correlationId = stored.metadata.correlationId ?? requestContext.generateId();
+      const causationId = String(stored.id);
+      const requestId = requestContext.generateId();
+      await requestContext.run({ requestId, correlationId, causationId }, async () => {
+        await consumer.handler(stored, context);
+      });
+      cursor = row.id;
+      attempts = 0;
+      lastError = null;
+      processed += 1;
+    } catch (e) {
+      const errMessage = e instanceof Error ? e.message : String(e);
+      if (consumer.errorPolicy?.skipApplyErrors) {
+        // Best-effort mode: record the error on the skip counter so ops
+        // can alert on a spike of skipped events, advance the cursor past
+        // the bad event, keep going. The consumer stays "idle", not "dead".
+        // Also emit a warn-level log line — the metric tells ops THAT events
+        // are being dropped, the log tells them WHICH events. Without this
+        // a poisoned-then-skipped event is invisible to forensic search.
+        const errorClass = e instanceof Error ? e.constructor.name : "UnknownError";
+        emitDispatcherError(context.meter ?? getFallbackMeter(), {
+          handler: consumer.name,
+          errorClass,
+        });
+        context.log?.warn(
+          `event-dispatcher: ${consumer.name} skipped event ${row.id} (${errorClass}): ${errMessage}`,
+        );
+        cursor = row.id;
+        attempts = 0;
+        lastError = null;
+        failed += 1;
+        continue;
+      }
+      attempts += 1;
+      lastError = errMessage;
+      failed += 1;
+      if (attempts >= effectiveMaxAttempts) deadLettered = true;
+      break;
+    }
+  }
+
+  return { cursor, attempts, lastError, deadLettered, processed, failed };
+}
+
+export async function persistConsumerOutcome(
+  tx: DbTx,
+  name: string,
+  instanceId: string,
+  outcome: DeliveryOutcome,
+): Promise<void> {
+  await updateConsumerDeliveryOutcome(tx, name, instanceId, outcome);
+}
+
+// Emit the lag gauge inside the consumer pass's tx so ops sees a snapshot
+// consistent with the cursor we just advanced to. `MAX(id)` on the events
+// table is an O(1) reverse-index scan — cheap even under load.
+export async function emitLagFromTx(
+  tx: DbTx,
+  consumerName: string,
+  instanceId: string,
+  cursor: bigint,
+  meter: Meter,
+): Promise<void> {
+  const head = await selectEventsHeadId(tx);
+  const lag = head > cursor ? Number(head - cursor) : 0;
+  emitEventConsumerLag(meter, { consumer: consumerName, instanceId }, lag);
+}