@cosmicdrift/kumiko-framework 0.328.0 → 0.329.0

package/dist/pipeline/event-dispatcher-admin.js

@@ -0,0 +1,196 @@
+import { advanceConsumerPastEventReturning, removePendingGapReturning, updateConsumerStatusReturning, } from "../db/queries/event-consumer.js";
+import { selectNextEventIdAfter, selectSmallestVisibleIdInRanges, } from "../db/queries/event-store.js";
+import { coerceRow, extractTableInfo, selectMany } from "../db/query.js";
+import { getEventsHighWaterMark } from "../event-store/index.js";
+import { eventConsumerStateTable, SHARED_INSTANCE_SENTINEL } from "./event-consumer-state.js";
+import { rangeContainsId, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges.js";
+// --- Ops recovery surface ---
+//
+// These are intentionally verb-distinct; each maps to a CLI sub-command.
+// They all target a single consumer row by name. Every call returns the
+// state after the write so the CLI can echo what actually changed.
+//
+// Semantics:
+//   restartConsumer   status="dead" → "idle", attempts=0, lastError=null.
+//                     Cursor unchanged → next pass retries the SAME event
+//                     that poisoned the consumer. For transient failures.
+//   disableConsumer   status=* → "disabled". Dispatcher skips this consumer
+//                     until enableConsumer() flips it back.
+//   enableConsumer    status="disabled" → "idle". No-op on any other state.
+//   skipPoisonEvent   cursor advances past the first event after the
+//                     current cursor (the one that's failing). attempts=0,
+//                     lastError=null, status="idle". For events that will
+//                     never succeed (broken payload, removed feature code).
+function normalizeConsumerState(row) {
+    return {
+        name: row.name,
+        instanceId: row.instanceId,
+        status: row.status,
+        lastProcessedEventId: row.lastProcessedEventId,
+        attempts: row.attempts,
+        rearmCount: row.rearmCount,
+        lastError: row.lastError,
+        updatedAt: row.updatedAt,
+    };
+}
+// Ops calls default to the SHARED_INSTANCE_SENTINEL row — that's the only
+// row shared-delivery consumers have, so legacy CLI invocations without
+// --instance-id keep working. Per-instance consumers require an explicit
+// instanceId: picking one of N shards arbitrarily ("first row wins") or
+// mutating all shards simultaneously ("bounce every instance") are both
+// worse than a loud missing-arg error on the CLI.
+async function requireConsumerRow(db, name, instanceId) {
+    const [row] = await selectMany(db, eventConsumerStateTable, {
+        name,
+        instanceId,
+    });
+    if (!row) {
+        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") has no state row — it hasn't run yet, the name is misspelled, or the instance is misspelled. ` +
+            `For per-instance consumers pass the instance_id explicitly; shared consumers use the default.`);
+    }
+    return row;
+}
+async function applyConsumerStatusTransition(db, name, instanceId, targetStatus) {
+    const raw = await updateConsumerStatusReturning(db, name, instanceId, targetStatus);
+    const updated = raw && coerceRow(raw, extractTableInfo(eventConsumerStateTable));
+    if (!updated) {
+        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished between read and write — retry.`);
+    }
+    return normalizeConsumerState(updated);
+}
+export async function restartConsumer(db, name, instanceId = SHARED_INSTANCE_SENTINEL) {
+    const before = await requireConsumerRow(db, name, instanceId);
+    if (before.status !== "dead") {
+        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") is not dead (status="${before.status}"). Restart only applies to dead consumers; use "enable" for a disabled one.`);
+    }
+    return applyConsumerStatusTransition(db, name, instanceId, "idle");
+}
+export async function disableConsumer(db, name, instanceId = SHARED_INSTANCE_SENTINEL) {
+    await requireConsumerRow(db, name, instanceId);
+    return applyConsumerStatusTransition(db, name, instanceId, "disabled");
+}
+export async function enableConsumer(db, name, instanceId = SHARED_INSTANCE_SENTINEL) {
+    const before = await requireConsumerRow(db, name, instanceId);
+    if (before.status !== "disabled") {
+        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") is not disabled (status="${before.status}"). Enable only flips disabled → idle; use "restart" for a dead consumer.`);
+    }
+    return applyConsumerStatusTransition(db, name, instanceId, "idle");
+}
+// skipPoisonEvent advances past whatever is currently blocking the consumer.
+// If the smallest visible id inside pending_gaps exists, THAT'S the poison —
+// halt-on-poison walks gap ids before any id past the cursor — so it's split
+// out of its range directly; the cursor stays put (it's already above this
+// id). Otherwise the poison is the usual next event after the cursor. Single
+// TX so concurrent dispatcher passes can't double-advance. Neither exists →
+// idempotent no-op.
+export async function skipPoisonEvent(db, name, instanceId = SHARED_INSTANCE_SENTINEL) {
+    const before = await requireConsumerRow(db, name, instanceId);
+    return db.begin(async (tx) => {
+        const pendingGaps = before.pendingGaps;
+        const smallestVisiblePending = await selectSmallestVisibleIdInRanges(tx, toIdRanges(pendingGaps));
+        if (smallestVisiblePending !== null) {
+            const newPendingGaps = pendingGaps.flatMap((gap) => rangeContainsId(gap, smallestVisiblePending)
+                ? splitRangeExcludingIds(gap, [smallestVisiblePending])
+                : [gap]);
+            const raw = await removePendingGapReturning(tx, name, instanceId, newPendingGaps);
+            const updated = raw && coerceRow(raw, extractTableInfo(eventConsumerStateTable));
+            if (!updated)
+                throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`);
+            return { ...normalizeConsumerState(updated), skippedEventId: smallestVisiblePending };
+        }
+        const poisonId = await selectNextEventIdAfter(tx, before.lastProcessedEventId);
+        if (poisonId === null) {
+            const [unchanged] = await selectMany(tx, eventConsumerStateTable, {
+                name,
+                instanceId,
+            });
+            if (!unchanged)
+                throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished — retry.`);
+            return { ...normalizeConsumerState(unchanged), skippedEventId: null };
+        }
+        const raw = await advanceConsumerPastEventReturning(tx, name, instanceId, poisonId);
+        const updated = raw && coerceRow(raw, extractTableInfo(eventConsumerStateTable));
+        if (!updated)
+            throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`);
+        return { ...normalizeConsumerState(updated), skippedEventId: poisonId };
+    });
+}
+// Read-only status for one consumer shard — CLI surface.
+export async function getConsumerState(db, name, instanceId = SHARED_INSTANCE_SENTINEL) {
+    const [row] = await selectMany(db, eventConsumerStateTable, {
+        name,
+        instanceId,
+    });
+    if (!row)
+        return null;
+    return {
+        name: row.name,
+        instanceId: row.instanceId,
+        status: row.status,
+        lastProcessedEventId: row.lastProcessedEventId,
+        attempts: row.attempts,
+        rearmCount: row.rearmCount,
+        lastError: row.lastError,
+        updatedAt: row.updatedAt,
+    };
+}
+// List every consumer the registry knows about, joined with all shard rows
+// from the state table. One entry per (name, instance_id) shard. Consumers
+// that have never run appear with status="never-run" and instance_id =
+// SHARED_INSTANCE_SENTINEL — a placeholder, because without a running
+// dispatcher we can't know the instance-ids of per-instance consumers yet.
+// Mirrors listProjectionsWithState — the registry (not the DB) is the
+// source-of-truth for which consumer-names exist; the DB is the source-
+// of-truth for which instance-shards have been seen.
+export async function listConsumersWithState(db, registeredNames) {
+    const stateRows = await selectMany(db, eventConsumerStateTable);
+    const registered = new Set(registeredNames);
+    // Materialize one output row per (name, instance_id). Registered names
+    // without any shard (never-run) get a placeholder row so ops can still
+    // see the name exists.
+    const out = [];
+    const seenNames = new Set();
+    for (const r of stateRows) {
+        if (!registered.has(r.name))
+            continue; // stale row from an older deploy
+        seenNames.add(r.name);
+        out.push({
+            name: r.name,
+            instanceId: r.instanceId,
+            status: r.status,
+            lastProcessedEventId: r.lastProcessedEventId,
+            attempts: r.attempts,
+            lastError: r.lastError,
+        });
+    }
+    for (const name of registeredNames) {
+        if (seenNames.has(name))
+            continue;
+        out.push({
+            name,
+            instanceId: SHARED_INSTANCE_SENTINEL,
+            status: "never-run",
+            lastProcessedEventId: 0n,
+            attempts: 0,
+            lastError: null,
+        });
+    }
+    return out;
+}
+// Like listConsumersWithState, but also returns HWM + lag per consumer.
+// Async consumers (MSPs) lag behind inline projections because they run
+// post-commit — lag is the primary signal for backpressure, dead consumers,
+// or dispatcher stalls. Programmatic callers can map the result to a
+// `kumiko_consumer_lag{name}` Prometheus gauge.
+// guard:dup-ok — intentionale Parallele zu getAllProjectionProgress; Consumer ≠ Projection (verschiedene Subsysteme)
+export async function getAllConsumerProgress(db, registeredNames) {
+    const [consumers, highWaterMark] = await Promise.all([
+        listConsumersWithState(db, registeredNames),
+        getEventsHighWaterMark(db),
+    ]);
+    return consumers.map((c) => ({
+        ...c,
+        highWaterMark,
+        lag: highWaterMark - c.lastProcessedEventId,
+    }));
+}

package/dist/pipeline/event-dispatcher-delivery.d.ts

@@ -0,0 +1,68 @@
+import type { DbConnection, DbRunner, DbTx } from "../db/connection.js";
+import { type PendingIdRange } from "../db/queries/event-store.js";
+import type { AppContext } from "../engine/types/index.js";
+import { type Meter } from "../observability/index.js";
+import { type PendingGapEntry } from "./event-consumer-state.js";
+import type { EventConsumer } from "./event-dispatcher.js";
+export type ConsumerStateRowShape = {
+    readonly name: string;
+    readonly instanceId: string;
+    readonly lastProcessedEventId: bigint;
+    readonly status: string;
+    readonly attempts: number;
+    readonly rearmCount: number;
+    readonly pendingGaps: readonly PendingGapEntry[];
+    readonly lastError: string | null;
+    readonly updatedAt: Temporal.Instant;
+};
+export type ConsumerStateRow = ConsumerStateRowShape;
+export type StoredEventRow = {
+    readonly id: bigint;
+    readonly aggregateId: string;
+    readonly aggregateType: string;
+    readonly tenantId: string;
+    readonly version: number;
+    readonly type: string;
+    readonly eventVersion: number;
+    readonly payload: Record<string, unknown>;
+    readonly metadata: import("../event-store/event-store.js").EventMetadata;
+    readonly createdAt: Temporal.Instant;
+    readonly createdBy: string;
+};
+export type AcquireOutcome = {
+    readonly state: ConsumerStateRow;
+    readonly skip: null;
+} | {
+    readonly state: null;
+    readonly skip: "locked_by_other_instance" | "disabled" | "dead" | "not_registered";
+};
+export declare function acquireConsumerState(tx: DbTx, name: string, instanceId: string, rearmCooldownMs: number, maxRearmCount: number): Promise<AcquireOutcome>;
+export type ConsumerCursor = {
+    readonly lastProcessedEventId: bigint;
+    readonly pendingGaps: readonly PendingGapEntry[];
+};
+export declare function selectConsumerCursorForUpdate(db: DbRunner, name: string, instanceId: string): Promise<ConsumerCursor | undefined>;
+export declare function selectIdleConsumerKeys(db: DbConnection, pairs: ReadonlyArray<{
+    readonly name: string;
+    readonly instanceId: string;
+}>): Promise<ReadonlySet<string>>;
+export declare function preRegisterConsumers(db: DbConnection, consumers: readonly EventConsumer[], dispatcherInstanceId: string | undefined): Promise<void>;
+export declare function consumerInstanceId(consumer: EventConsumer, dispatcherInstanceId: string | undefined): string;
+export declare function markProcessing(tx: DbTx, name: string, instanceId: string): Promise<void>;
+export declare function fetchPendingEvents(tx: DbTx, cursor: bigint, batchSize: number, pendingRanges?: readonly PendingIdRange[]): Promise<ReadonlyArray<StoredEventRow>>;
+export type DeliveryOutcome = {
+    readonly cursor: bigint;
+    readonly attempts: number;
+    readonly lastError: string | null;
+    readonly deadLettered: boolean;
+    readonly processed: number;
+    readonly failed: number;
+    readonly resolvedPendingIds: readonly bigint[];
+};
+export declare function deliverEvents(consumer: EventConsumer, events: ReadonlyArray<StoredEventRow>, context: AppContext, maxAttempts: number, state: ConsumerStateRow): Promise<DeliveryOutcome>;
+export type PersistedConsumerOutcome = DeliveryOutcome & {
+    readonly pendingGaps: readonly PendingGapEntry[];
+};
+export declare function persistConsumerOutcome(tx: DbTx, name: string, instanceId: string, outcome: PersistedConsumerOutcome): Promise<void>;
+export declare function persistConsumerPassFailure(tx: DbTx, name: string, instanceId: string, errorMessage: string): Promise<void>;
+export declare function emitLagFromTx(tx: DbTx, consumerName: string, instanceId: string, cursor: bigint, meter: Meter): Promise<void>;

package/dist/pipeline/event-dispatcher-delivery.js

@@ -0,0 +1,310 @@
+// Value-only import, aliased to avoid shadowing the ambient global
+// `Temporal` TYPE that ConsumerStateRow.updatedAt/StoredEventRow.createdAt
+// resolve against (same #1438 dual-package-hazard pattern as event-store.ts).
+import { Temporal as TemporalPolyfill } from "temporal-polyfill";
+import { requestContext } from "../api/request-context.js";
+import { insertConsumerIfAbsent, markConsumerProcessing, rearmDeadConsumer, recordConsumerPassFailure, selectConsumerForUpdate, selectConsumerForUpdateSkipLocked, selectProvablyIdleConsumerPairs, updateConsumerDeliveryOutcome, } from "../db/queries/event-consumer.js";
+import { selectEventsHeadId, selectPendingAndNewEventRows, } from "../db/queries/event-store.js";
+import { coerceRow, extractTableInfo } from "../db/query.js";
+import { qnScope } from "../engine/qualified-name.js";
+import { eventsTable, toStoredEvent as rowToStoredEvent, } from "../event-store/index.js";
+import { emitDispatcherError, emitEventConsumerLag, getFallbackMeter, } from "../observability/index.js";
+import { ConsumerStatuses, eventConsumerStateTable, SHARED_INSTANCE_SENTINEL, } from "./event-consumer-state.js";
+import { parseWriteOrigin } from "./write-origin.js";
+// Fails closed without throwing: a throw would poison the event for every consumer.
+const UNPARSEABLE_STORED_WRITE_ORIGIN = {
+    rootHandler: "<unknown>",
+    anonymousRoot: true,
+    publicIntake: false,
+};
+// Lock the consumer's state row with SKIP LOCKED. Strict: no in-tx bootstrap.
+// The row must exist — start() pre-registers every consumer up front so
+// prune (event-retention) sees their cursors as soon as the process is up,
+// closing the race where a lazy-bootstrapped consumer's cursor is absent
+// during prune and its events are silently deleted.
+//
+// skip="not_registered" signals a row-missing-despite-start condition.
+// Production shouldn't hit this — it means either start() wasn't called
+// (runOnce() guards against that) or the state row was deleted externally
+// (a test TRUNCATE without subsequent ensureRegistered(), or an operator
+// intervention). Skipping quietly preserves the dispatcher's other
+// consumers and surfaces the issue via the metrics pass-outcome.
+export async function acquireConsumerState(tx, name, instanceId, rearmCooldownMs, maxRearmCount) {
+    const rawState = await selectConsumerForUpdateSkipLocked(tx, name, instanceId);
+    if (!rawState) {
+        return { state: null, skip: "not_registered" };
+    }
+    const state = coerceRow(rawState, extractTableInfo(eventConsumerStateTable));
+    if (!state) {
+        // Either the row never existed (no pre-reg, no ensureRegistered) or
+        // another instance currently holds the lock with SKIP LOCKED filtering
+        // us out. We can't distinguish here in a single query, so return
+        // "not_registered" — ops sees a skip-reason instead of silent delivery
+        // loss. Under normal operation (start() called, no external tampering)
+        // this path is never taken.
+        return { state: null, skip: "not_registered" };
+    }
+    if (state.status === ConsumerStatuses.disabled)
+        return { state: null, skip: "disabled" };
+    if (state.status === ConsumerStatuses.dead) {
+        // Bounded auto-revival: a transient failure (e.g. a Meilisearch blip)
+        // shouldn't need an operator to notice and run restartConsumer() once
+        // the cause is long gone. Cooldown since the last write (the death or
+        // a prior re-arm) gates the retry; maxRearmCount stops a poison event
+        // from looping forever (re-arm → same event fails → dead → re-arm →
+        // ...) — after the cap it stays dead until a human intervenes.
+        const cooldownDeadline = TemporalPolyfill.Now.instant().subtract({
+            milliseconds: rearmCooldownMs,
+        });
+        // @cast-boundary temporal-polyfill-vs-ambient: same TC39 Temporal.Instant
+        // at runtime — state.updatedAt is DB-row-typed against the ambient
+        // global, two distinct nominal types across the two .d.ts sources (see
+        // event-store.ts).
+        const cooldownElapsed = TemporalPolyfill.Instant.compare(state.updatedAt, cooldownDeadline) <= 0;
+        if (cooldownElapsed && state.rearmCount < maxRearmCount) {
+            const rearmed = await rearmDeadConsumer(tx, name, instanceId);
+            const rearmedState = rearmed &&
+                coerceRow(rearmed, extractTableInfo(eventConsumerStateTable));
+            if (rearmedState)
+                return { state: rearmedState, skip: null };
+        }
+        // Caller (event-dispatcher.ts's processConsumer) emits
+        // kumiko_event_consumer_rearm_exhausted_total once per (consumer,
+        // instance) transition into this branch — it has the process-lifetime
+        // state to dedupe across poll passes that this pure function doesn't.
+        return { state: null, skip: "dead" };
+    }
+    return { state, skip: null };
+}
+// Blocking FOR UPDATE (not SKIP LOCKED): callers of this one need to actually
+// wait for the dispatcher's own turn to finish, not skip past it, so their
+// read of the already-applied cursor is guaranteed to be up to date rather
+// than possibly stale.
+export async function selectConsumerCursorForUpdate(db, name, instanceId) {
+    const rawState = await selectConsumerForUpdate(db, name, instanceId);
+    if (!rawState)
+        return undefined;
+    const state = coerceRow(rawState, extractTableInfo(eventConsumerStateTable));
+    return { lastProcessedEventId: state.lastProcessedEventId, pendingGaps: state.pendingGaps };
+}
+// Read-only pre-check run once per doPass, before any consumer's turn opens
+// a transaction (event-dispatcher.ts's doPass/runConsumerTurn). Keyed like
+// consumerBackoff/inFlightTurns: `${name}:${instanceId}`. Consumers not
+// provably idle (missing row, dead, pending gaps, or an event past cursor)
+// simply don't appear in the result — the caller falls through to the
+// existing acquireConsumerState/FOR UPDATE SKIP LOCKED path unchanged.
+//
+// Race: an event that commits between this query and the next tick CAN
+// make this snapshot stale — that's expected, not a bug. What it can't do
+// is get silently lost: the append fires NOTIFY (or the next poll tick
+// fires regardless), triggering a fresh doPass. That pass's own pre-check
+// sees the new head id and excludes the consumer from "idle"; a new
+// pending_gap only ever gets created when a later turn's fetch (id > the
+// old cursor) walks past the row that committed late, and this same
+// pre-check reads pending_gaps live on every pass, so it observes that gap
+// too. In short: this can under-report idleness for one tick, never
+// over-report it.
+export async function selectIdleConsumerKeys(db, pairs) {
+    if (pairs.length === 0)
+        return new Set();
+    const names = pairs.map((p) => p.name);
+    const instanceIds = pairs.map((p) => p.instanceId);
+    const idlePairs = await selectProvablyIdleConsumerPairs(db, names, instanceIds, ConsumerStatuses.dead);
+    return new Set(idlePairs.map((p) => `${p.name}:${p.instanceId}`));
+}
+// Shared pre-registration: one row per (consumer, shard), cursor = 0,
+// status = idle. Shared-delivery consumers use SHARED_INSTANCE_SENTINEL;
+// per-instance consumers use the dispatcher's instanceId. Idempotent
+// under restart and concurrent start-calls via ON CONFLICT DO NOTHING
+// on the composite PK — never clobbers an existing cursor.
+export async function preRegisterConsumers(db, consumers, dispatcherInstanceId) {
+    for (const consumer of consumers) {
+        const instanceId = consumerInstanceId(consumer, dispatcherInstanceId);
+        await insertConsumerIfAbsent(db, consumer.name, instanceId, consumer.startFrom);
+    }
+}
+// Resolve the instance_id column value for one consumer on this dispatcher.
+// Shared stays at the sentinel; per-instance rides the dispatcher's id.
+// Throws when a per-instance consumer is registered without an instanceId
+// — missing at boot is the sharp-edge to catch, not at first delivery.
+export function consumerInstanceId(consumer, dispatcherInstanceId) {
+    if (consumer.delivery !== "per-instance")
+        return SHARED_INSTANCE_SENTINEL;
+    if (!dispatcherInstanceId) {
+        throw new Error(`EventConsumer "${consumer.name}" has delivery="per-instance" but the dispatcher was created without an instanceId — ` +
+            `pass EventDispatcherOptions.instanceId (typically from ServerOptions.instanceId / KUMIKO_INSTANCE_ID).`);
+    }
+    return dispatcherInstanceId;
+}
+// Mark the consumer row as "processing" for ops visibility. The SKIP LOCKED
+// lock already guarantees single-writer semantics; this is purely
+// informational (and resets on commit to idle/dead via persistConsumerOutcome).
+export async function markProcessing(tx, name, instanceId) {
+    await markConsumerProcessing(tx, name, instanceId);
+}
+// `pendingRanges` are id ranges below `cursor` the consumer is still
+// watching as gaps (invisible on an earlier turn — see event-dispatcher.ts's
+// processConsumer). Fetching them alongside the plain `id > cursor` window
+// means a row that committed late becomes visible and deliverable the next
+// time this consumer's turn runs, instead of being permanently skipped.
+export async function fetchPendingEvents(tx, cursor, batchSize, pendingRanges = []) {
+    const rawRows = await selectPendingAndNewEventRows(tx, cursor, pendingRanges, batchSize);
+    const info = extractTableInfo(eventsTable);
+    return rawRows.map((row) => coerceRow(row, info)); // @cast-boundary db-row
+}
+// Deliver events to the consumer's handler in events.id order, or — when
+// the consumer wires a batchHandler — through it in one call first. Halt-
+// on-poison: a throw breaks the per-event loop, the cursor stays at the
+// last successful event, and attempts climb. At the consumer's
+// effectiveMaxAttempts (errorPolicy.maxAttempts ?? maxAttempts) the caller
+// persists status="dead" and the consumer is parked until ops intervenes
+// (see restartConsumer / skipPoisonEvent). A batch that throws falls back
+// to the same per-event loop for the same events — the batch failure
+// itself does not bump attempts, only a per-event failure does.
+export async function deliverEvents(consumer, events, context, maxAttempts, state) {
+    const startCursor = state.lastProcessedEventId;
+    let cursor = startCursor;
+    let attempts = state.attempts;
+    let lastError = state.lastError ?? null;
+    let deadLettered = false;
+    const effectiveMaxAttempts = consumer.errorPolicy?.maxAttempts ?? maxAttempts;
+    let processed = 0;
+    let failed = 0;
+    const resolvedPendingIds = [];
+    // A pending row sits below startCursor: it resolves its gap but never moves
+    // the cursor backward. ORDER BY id walks all pending rows first.
+    const resolve = (id) => {
+        if (id > cursor)
+            cursor = id;
+        if (id <= startCursor)
+            resolvedPendingIds.push(id);
+        attempts = 0;
+        lastError = null;
+    };
+    if (consumer.batchHandler && events.length > 0) {
+        const batchSucceeded = await tryApplyBatch(consumer, events, context);
+        if (batchSucceeded) {
+            for (const row of events)
+                resolve(row.id);
+            processed = events.length;
+            return { cursor, attempts, lastError, deadLettered, processed, failed, resolvedPendingIds };
+        }
+        // fall through to the per-event loop below; the batch failure itself
+        // does NOT bump attempts — only a per-event failure does.
+    }
+    for (const row of events) {
+        try {
+            await applyEvent(consumer, row, context);
+            resolve(row.id);
+            processed += 1;
+        }
+        catch (e) {
+            const errMessage = e instanceof Error ? e.message : String(e);
+            failed += 1;
+            if (consumer.errorPolicy?.skipApplyErrors) {
+                reportSkippedEvent(consumer, row.id, e, errMessage, context);
+                resolve(row.id);
+                continue;
+            }
+            attempts += 1;
+            lastError = errMessage;
+            if (attempts >= effectiveMaxAttempts)
+                deadLettered = true;
+            break;
+        }
+    }
+    return { cursor, attempts, lastError, deadLettered, processed, failed, resolvedPendingIds };
+}
+// Shared requestContext scope for one apply — event-derived fields
+// (correlationId, causationId, writeOrigin) come from `causationSource`;
+// applyEvent uses the event itself, applyBatch uses the LAST event of the
+// turn (see applyBatch for why). requestId falls back to a fresh id
+// because the dispatcher runs outside any HTTP request (background poll),
+// and a stable log-correlation handle is still useful for debugging.
+// #3043 — an event this apply writes is attributed to the consumer, not
+// to whatever wrote the triggering event; causationId already links back.
+function buildConsumerRequestScope(consumer, causationSource) {
+    const correlationId = causationSource.metadata.correlationId ?? requestContext.generateId();
+    const causationId = String(causationSource.id);
+    const requestId = requestContext.generateId();
+    // The job-trigger consumer's handleEvent stamps event-triggered jobs from this.
+    const rawStoredWriteOrigin = causationSource.metadata.writeOrigin;
+    const writeOrigin = rawStoredWriteOrigin === undefined
+        ? undefined
+        : (parseWriteOrigin(rawStoredWriteOrigin) ?? UNPARSEABLE_STORED_WRITE_ORIGIN);
+    return {
+        requestId,
+        correlationId,
+        causationId,
+        handler: consumer.name,
+        feature: consumer.featureName ?? qnScope(consumer.name),
+        writeOrigin,
+    };
+}
+async function applyEvent(consumer, row, context) {
+    const stored = rowToStoredEvent(row);
+    await requestContext.run(buildConsumerRequestScope(consumer, stored), async () => {
+        await consumer.handler(stored, context);
+    });
+}
+// Runs the consumer's batchHandler for the whole turn under one
+// requestContext scope, attempted before the per-event loop. Returns
+// false on failure instead of rethrowing — the caller falls back to
+// per-event delivery for the exact same events, so a batch failure must
+// not itself count as an applyEvent failure or bump attempts.
+async function tryApplyBatch(consumer, events, context) {
+    try {
+        await applyBatch(consumer, events, context);
+        return true;
+    }
+    catch (e) {
+        const errMessage = e instanceof Error ? e.message : String(e);
+        context.log?.warn(`event-dispatcher: ${consumer.name} batch of ${events.length} failed, falling back to per-event delivery: ${errMessage}`);
+        return false;
+    }
+}
+async function applyBatch(consumer, events, context) {
+    const batchHandler = consumer.batchHandler;
+    // skip: deliverEvents only calls this for consumers that wire a batchHandler.
+    if (!batchHandler)
+        return;
+    const stored = events.map((row) => rowToStoredEvent(row));
+    // correlationId/causationId/writeOrigin come from the LAST row — a
+    // batch-appended event's causation is attributed to the turn's most
+    // recent input, same as chaining N applyEvent calls would leave the
+    // requestContext at after the final one.
+    const lastStored = stored.at(-1);
+    // skip: deliverEvents never calls this with an empty turn.
+    if (!lastStored)
+        return;
+    await requestContext.run(buildConsumerRequestScope(consumer, lastStored), async () => {
+        await batchHandler(stored, context);
+    });
+}
+// Best-effort mode: record the error on the skip counter so ops can alert on
+// a spike of skipped events; the consumer stays "idle", not "dead". The
+// warn-level log line tells them WHICH events — without it a
+// poisoned-then-skipped event is invisible to forensic search.
+function reportSkippedEvent(consumer, eventId, e, errMessage, context) {
+    const errorClass = e instanceof Error ? e.constructor.name : "UnknownError";
+    emitDispatcherError(context.meter ?? getFallbackMeter(), { handler: consumer.name, errorClass });
+    context.log?.warn(`event-dispatcher: ${consumer.name} skipped event ${eventId} (${errorClass}): ${errMessage}`);
+}
+export async function persistConsumerOutcome(tx, name, instanceId, outcome) {
+    await updateConsumerDeliveryOutcome(tx, name, instanceId, outcome);
+}
+// Best-effort record of a pass that threw before persistConsumerOutcome
+// could run (event-dispatcher.ts's processConsumer catch). The caller opens
+// its OWN transaction for this — the one that rolled back never committed
+// anything, including this write, if it happened inside the same tx.
+export async function persistConsumerPassFailure(tx, name, instanceId, errorMessage) {
+    await recordConsumerPassFailure(tx, name, instanceId, errorMessage);
+}
+// Emit the lag gauge inside the consumer pass's tx so ops sees a snapshot
+// consistent with the cursor we just advanced to. `MAX(id)` on the events
+// table is an O(1) reverse-index scan — cheap even under load.
+export async function emitLagFromTx(tx, consumerName, instanceId, cursor, meter) {
+    const head = await selectEventsHeadId(tx);
+    const lag = head > cursor ? Number(head - cursor) : 0;
+    emitEventConsumerLag(meter, { consumer: consumerName, instanceId }, lag);
+}

package/dist/pipeline/event-dispatcher.d.ts

@@ -0,0 +1,52 @@
+import type { PgClient } from "../db/connection.js";
+import type { AppContext } from "../engine/types/index.js";
+import { type StoredEvent } from "../event-store/index.js";
+import { type Meter, type Tracer } from "../observability/index.js";
+export type EventConsumerHandler = (event: StoredEvent, ctx: AppContext) => Promise<void>;
+export type EventConsumerBatchHandler = (events: readonly StoredEvent[], ctx: AppContext) => Promise<void>;
+export type EventConsumerErrorPolicy = {
+    readonly skipApplyErrors?: boolean;
+    readonly maxAttempts?: number;
+};
+export type EventConsumer = {
+    readonly name: string;
+    readonly handler: EventConsumerHandler;
+    readonly batchHandler?: EventConsumerBatchHandler;
+    readonly errorPolicy?: EventConsumerErrorPolicy;
+    readonly featureName?: string;
+    readonly delivery?: "shared" | "per-instance";
+    readonly startFrom?: "beginning" | "now";
+};
+export type DispatcherPassResult = {
+    readonly processed: number;
+    readonly failed: number;
+    readonly byConsumer: Record<string, {
+        processed: number;
+        failed: number;
+    }>;
+};
+export type EventDispatcher = {
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    runOnce(): Promise<DispatcherPassResult>;
+    ensureRegistered(): Promise<void>;
+    drain(): Promise<void>;
+    readonly consumers: readonly EventConsumer[];
+};
+export type EventDispatcherOptions = {
+    readonly db: import("../db/connection.js").DbConnection;
+    readonly consumers: readonly EventConsumer[];
+    readonly context: AppContext;
+    readonly batchSize?: number;
+    readonly pollIntervalMs?: number;
+    readonly maxAttempts?: number;
+    readonly rearmCooldownMs?: number;
+    readonly maxRearmCount?: number;
+    readonly tracer?: Tracer;
+    readonly meter?: Meter;
+    readonly instanceId?: string;
+    readonly pgClient?: PgClient;
+};
+export declare function createEventDispatcher(options: EventDispatcherOptions): EventDispatcher;
+export type { ConsumerProgress, ConsumerRecoveryState } from "./event-dispatcher-admin.js";
+export { disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher-admin.js";