@cosmicdrift/kumiko-framework 0.328.0 → 0.329.0

package/dist/pipeline/event-dispatcher.js

@@ -0,0 +1,523 @@
+import { selectSnapshotXmax, selectSnapshotXmin } from "../db/queries/event-consumer.js";
+import { SYSTEM_TENANT_ID } from "../engine/types/identifiers.js";
+import { EVENTS_PUBSUB_CHANNEL } from "../event-store/index.js";
+import { emitEventConsumerPassOutcome, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, getFallbackMeter, getFallbackTracer, } from "../observability/index.js";
+import { SHARED_INSTANCE_SENTINEL } from "./event-consumer-state.js";
+import { acquireConsumerState, consumerInstanceId, deliverEvents, emitLagFromTx, fetchPendingEvents, markProcessing, persistConsumerOutcome, persistConsumerPassFailure, preRegisterConsumers, selectIdleConsumerKeys, } from "./event-dispatcher-delivery.js";
+import { partitionBurntGaps, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges.js";
+const DEFAULT_BATCH_SIZE = 200;
+const DEFAULT_POLL_MS = 100;
+const DEFAULT_MAX_ATTEMPTS = 10;
+const DEFAULT_REARM_COOLDOWN_MS = 5 * 60_000;
+const DEFAULT_MAX_REARM_COUNT = 3;
+// Backoff for a consumer whose pass THREW (the processConsumer catch below)
+// — an infra-level failure (db.begin's callback rejected outside deliverEvents'
+// own per-event try/catch), not a poisoned event. Doubles per consecutive
+// failure, capped at PASS_FAILURE_MAX_BACKOFF_MS so a consumer stuck failing
+// still gets retried within a minute; a successful pass resets it to 0.
+const PASS_FAILURE_BASE_BACKOFF_MS = 1_000;
+const PASS_FAILURE_MAX_BACKOFF_MS = 60_000;
+export function createEventDispatcher(options) {
+    const { db, consumers, context, batchSize = DEFAULT_BATCH_SIZE, pollIntervalMs = DEFAULT_POLL_MS, maxAttempts = DEFAULT_MAX_ATTEMPTS, rearmCooldownMs = DEFAULT_REARM_COOLDOWN_MS, maxRearmCount = DEFAULT_MAX_REARM_COUNT, } = options;
+    // Fail-fast on misconfigured per-instance wiring. Catching this at
+    // construction surfaces the problem in boot logs instead of first
+    // delivery attempt — where it would land as a confusing preRegister
+    // throw much later in the startup sequence.
+    for (const consumer of consumers) {
+        if (consumer.delivery === "per-instance" && !options.instanceId) {
+            throw new Error(`EventConsumer "${consumer.name}" has delivery="per-instance" but EventDispatcherOptions.instanceId is missing. ` +
+                `Pass ServerOptions.instanceId (defaults to KUMIKO_INSTANCE_ID or a boot-time UUID) when any consumer uses per-instance delivery.`);
+        }
+    }
+    if (options.instanceId === SHARED_INSTANCE_SENTINEL) {
+        throw new Error(`EventDispatcherOptions.instanceId cannot equal the reserved sentinel "${SHARED_INSTANCE_SENTINEL}". ` +
+            `Pick any other stable string (typically KUMIKO_INSTANCE_ID from the deploy env).`);
+    }
+    const tracer = options.tracer ?? getFallbackTracer();
+    const meter = options.meter ?? getFallbackMeter();
+    // Tracks which (consumer, instanceId) pairs already fired
+    // kumiko_event_consumer_rearm_exhausted_total, so a consumer stuck dead
+    // across many poll passes emits the ops-signal once, not every pass
+    // (that would be log/metric spam for a state that hasn't changed).
+    // Process-lifetime only — restarting the dispatcher re-arms reporting,
+    // which is fine: a fresh process re-observing a still-dead consumer is
+    // exactly the "still needs a human" signal ops wants.
+    const reportedDeadConsumers = new Set();
+    // Per-(consumer, instanceId) backoff after a thrown pass (see the catch in
+    // processConsumer). Keyed the same way as reportedDeadConsumers.
+    // consecutiveFailures drives the exponential delay; retryAtMs is the
+    // Date.now() timestamp doPass compares against to decide whether this
+    // consumer's turn is skipped this tick. A successful pass deletes the
+    // entry — the next failure (if any) starts a fresh backoff from scratch.
+    const consumerBackoff = new Map();
+    // Dedup flag for the idle pre-check's error log (doPass) — a broken
+    // pre-check falls back to the expensive-but-correct path silently on
+    // every tick (see doPass), which must not mean silently in the logs too:
+    // ops needs to see it degrade once, not have every poll tick spam the
+    // same line. Cleared the moment the pre-check succeeds again, so a later,
+    // unrelated outage gets its own fresh log line instead of staying
+    // permanently suppressed by this process's flag.
+    let idlePreCheckErrorLogged = false;
+    let running = false;
+    // Separate from `running` on purpose: pre-registration of consumer state
+    // rows is a one-time boot action, while running/timer/LISTEN is a
+    // lifecycle toggle. stop() flips running back to false but leaves
+    // preRegistered true — a subsequent runOnce() is still safe because the
+    // state rows are in place. Production code never stops-then-runs-once;
+    // tests do (drain on-demand without a timer loop).
+    let preRegistered = false;
+    let timer = null;
+    // LISTEN subscription handle. Set when .start() successfully subscribed
+    // to EVENTS_PUBSUB_CHANNEL; cleared by .stop(). The timer remains active
+    // even with LISTEN attached — it's a cheap safety net against missed
+    // NOTIFYs (subscription drop, crash mid-commit).
+    let pgUnlisten = null;
+    // Bounds how many consumer turns hold a DB transaction at once. Each turn's
+    // db.begin() checks out one pool connection, plus any writes the handler
+    // itself makes via context.db — the app db pool defaults to postgres-js's
+    // max=10 (db/connection.ts, no DATABASE_POOL_MAX override) and is shared
+    // with HTTP request handlers, so turns can't be allowed to claim it all.
+    // A pool no larger than this limit can deadlock: every connection held by
+    // an open turn TX while each handler waits for one more.
+    const MAX_CONCURRENT_CONSUMER_TURNS = 4;
+    let activeConsumerTurns = 0;
+    const consumerTurnWaiters = [];
+    async function acquireConsumerTurnSlot() {
+        if (activeConsumerTurns >= MAX_CONCURRENT_CONSUMER_TURNS) {
+            // The releasing turn hands its slot over directly (count unchanged),
+            // so a synchronous acquire in between cannot overshoot the limit.
+            await new Promise((resolve) => consumerTurnWaiters.push(resolve));
+        }
+        else {
+            activeConsumerTurns++;
+        }
+        let released = false;
+        return () => {
+            // skip: already released — finally-blocks calling this twice must not
+            // free the same slot twice
+            if (released)
+                return;
+            released = true;
+            const nextWaiter = consumerTurnWaiters.shift();
+            if (nextWaiter)
+                nextWaiter();
+            else
+                activeConsumerTurns--;
+        };
+    }
+    // Per-(consumer, instanceId) turn guard, so one slow consumer never holds
+    // back the others. Keyed like consumerBackoff: a consumer already running
+    // its turn hands back that same promise instead of starting a second one.
+    const inFlightTurns = new Map();
+    async function runConsumerTurn(consumer, effective, idleKeys) {
+        const key = `${consumer.name}:${consumerInstanceId(consumer, options.instanceId)}`;
+        const existing = inFlightTurns.get(key);
+        if (existing)
+            return existing;
+        // Feature-gate, backoff-gate and idle-gate are resolved synchronously,
+        // before any promise is registered — a gated consumer must resolve
+        // without ever occupying inFlightTurns, or it would look "still
+        // running" forever.
+        if (effective && consumer.featureName && !effective.has(consumer.featureName)) {
+            return { processed: 0, failed: 0 };
+        }
+        const backoff = consumerBackoff.get(key);
+        if (backoff && backoff.retryAtMs > Date.now()) {
+            return { processed: 0, failed: 0 };
+        }
+        // Idle-gate: doPass already proved this consumer has nothing to do
+        // (see selectIdleConsumerKeys) — skip the turn without ever opening a
+        // db.begin/FOR UPDATE SKIP LOCKED (see doPass for why that matters).
+        // Clear any prior backoff same as a successful pass would: the
+        // backoff-gate above already confirmed retryAtMs has elapsed, and
+        // "provably nothing to deliver" is itself proof the consumer isn't
+        // presently failing.
+        if (idleKeys.has(key)) {
+            consumerBackoff.delete(key);
+            return { processed: 0, failed: 0 };
+        }
+        // Registered before awaiting the slot — a turn queued behind the
+        // concurrency limit still counts as in-flight, so it isn't started twice.
+        const turn = (async () => {
+            const releaseSlot = await acquireConsumerTurnSlot();
+            try {
+                return await processConsumer(consumer);
+            }
+            finally {
+                releaseSlot();
+            }
+        })();
+        inFlightTurns.set(key, turn);
+        try {
+            return await turn;
+        }
+        finally {
+            if (inFlightTurns.get(key) === turn)
+                inFlightTurns.delete(key);
+        }
+    }
+    async function drainInFlightTurns() {
+        await Promise.all([...inFlightTurns.values()].map((turn) => turn.catch(() => {
+            // skip: errors already recorded per-consumer inside the pass
+        })));
+    }
+    // A pass still in doPass's idle pre-check (selectIdleConsumerKeys) hasn't
+    // registered any consumer turn in inFlightTurns yet — stop() must wait for
+    // it too, or it keeps querying the pool stop() is about to close.
+    const inFlightPasses = new Set();
+    async function drainInFlightPasses() {
+        await Promise.all([...inFlightPasses].map((pass) => pass.catch(() => {
+            // skip: errors already recorded per-consumer inside the pass
+        })));
+    }
+    async function runOnce() {
+        if (!preRegistered) {
+            throw new Error("EventDispatcher.runOnce() called before start() — consumer state rows are not registered. Call start() first (production) or ensureRegistered() (tests after truncating kumiko_event_consumers).");
+        }
+        const pass = doPass();
+        inFlightPasses.add(pass);
+        try {
+            return await pass;
+        }
+        finally {
+            inFlightPasses.delete(pass);
+        }
+    }
+    async function doPass() {
+        let totalProcessed = 0;
+        let totalFailed = 0;
+        const byConsumer = {};
+        // Feature-toggle snapshot taken once per pass (not per consumer): all
+        // consumers see the same disabled-set even if an operator flips a
+        // toggle mid-pass, so "this event batch" decisions stay consistent.
+        //
+        // Sprint-8a tier-composition: per-tenant resolver per-pass-konsultiert
+        // mit SYSTEM_TENANT_ID. Async-events sind tier-agnostic — wenn ein
+        // Tenant downgrade'd, sollen seine queued events trotzdem verarbeitet
+        // werden (events sind immutable, projection ist eventually-consistent).
+        // Tier-cuts wirken request-time im sync-dispatcher + lifecycle-pipeline,
+        // nicht im async-replay. App-level resolver entscheidet was er bei
+        // SYSTEM_TENANT_ID returnt (typisch: union-of-all-tier-features).
+        const effective = context.effectiveFeatures?.(SYSTEM_TENANT_ID);
+        // Lock-free pre-check, once per pass: which consumers are provably idle
+        // (no locking, no xid, no WAL — see selectIdleConsumerKeys). Read
+        // outside any transaction, before a single db.begin/FOR UPDATE SKIP
+        // LOCKED opens — an idle deployment (most consumers, most ticks)
+        // otherwise pays a row lock + WAL record + commit fsync per consumer
+        // per tick for nothing.
+        //
+        // A failure here (e.g. a transient DB blip) must not abort the whole
+        // pass — timer/LISTEN callers swallow doPass()'s rejection silently
+        // (see start()), which would mean no log, no persistConsumerPassFailure,
+        // and no backoff for every consumer this tick. Falling back to "nothing
+        // proven idle" instead routes every consumer through its normal
+        // acquireConsumerState path, whose own try/catch already covers this.
+        let idleKeys;
+        try {
+            idleKeys = await selectIdleConsumerKeys(db, consumers.map((consumer) => ({
+                name: consumer.name,
+                instanceId: consumerInstanceId(consumer, options.instanceId),
+            })));
+            if (idlePreCheckErrorLogged) {
+                idlePreCheckErrorLogged = false;
+                const recoveredMsg = "[event-dispatcher] idle pre-check recovered, database reachable again";
+                if (context.log) {
+                    context.log.info(recoveredMsg);
+                }
+                else {
+                    console.log(recoveredMsg);
+                }
+            }
+        }
+        catch (e) {
+            idleKeys = new Set();
+            // Log once per outage, not once per poll tick — see
+            // idlePreCheckErrorLogged above. Every consumer still gets delivered
+            // correctly this tick (fallback above), so this is a degradation
+            // signal for ops, not a delivery failure.
+            if (!idlePreCheckErrorLogged) {
+                idlePreCheckErrorLogged = true;
+                // The LISTEN peer talks to the same DB, so it is gone too; only a
+                // later onlisten may flip the gauge back to 1.
+                emitEventDispatcherListenConnected(meter, false);
+                const msg = e instanceof Error ? e.message : String(e);
+                const logMsg = `[event-dispatcher] idle pre-check failed, falling back to per-consumer locking every tick until it recovers: ${msg}`;
+                if (context.log) {
+                    context.log.error(logMsg);
+                }
+                else {
+                    console.error(logMsg);
+                }
+            }
+        }
+        // Every consumer runs its own turn concurrently (own TX, own cursor,
+        // own backoff/feature-gate/idle-gate) — a slow consumer no longer
+        // blocks the others' delivery until it commits, see runConsumerTurn.
+        const results = await Promise.all(consumers.map(async (consumer) => {
+            const perConsumer = await runConsumerTurn(consumer, effective, idleKeys);
+            return [consumer.name, perConsumer];
+        }));
+        for (const [name, perConsumer] of results) {
+            byConsumer[name] = perConsumer;
+            totalProcessed += perConsumer.processed;
+            totalFailed += perConsumer.failed;
+        }
+        return { processed: totalProcessed, failed: totalFailed, byConsumer };
+    }
+    async function processConsumer(consumer) {
+        let processed = 0;
+        let failed = 0;
+        const instanceId = consumerInstanceId(consumer, options.instanceId);
+        const span = tracer.startSpan("events.consumer.pass", {
+            attributes: {
+                "consumer.name": consumer.name,
+                "consumer.instance_id": instanceId,
+            },
+        });
+        try {
+            await db.begin(async (tx) => {
+                const acquired = await acquireConsumerState(tx, consumer.name, instanceId, rearmCooldownMs, maxRearmCount);
+                // skip: another instance holds the lock, or the consumer is
+                // disabled/dead. Nothing to deliver this pass.
+                if (acquired.skip !== null) {
+                    span.setAttribute("consumer.skip_reason", acquired.skip);
+                    if (acquired.skip === "dead") {
+                        const reportKey = `${consumer.name}:${instanceId}`;
+                        if (!reportedDeadConsumers.has(reportKey)) {
+                            reportedDeadConsumers.add(reportKey);
+                            emitEventConsumerRearmExhausted(meter, { consumer: consumer.name, instanceId });
+                        }
+                    }
+                    // skip: skip reason already recorded on the span above (and, for
+                    // "dead", already emitted as a metric) — nothing left to deliver.
+                    return;
+                }
+                // Acquired normally (including via a successful auto-rearm) — clear
+                // any prior dead-report so a future exhaustion re-emits instead of
+                // staying permanently suppressed by this process's Set.
+                reportedDeadConsumers.delete(`${consumer.name}:${instanceId}`);
+                const oldCursor = acquired.state.lastProcessedEventId;
+                const pendingGaps = acquired.state.pendingGaps;
+                // xmin BEFORE the fetch: a range is provably burnt only once this has
+                // passed the xmax recorded with it. Only needed when ranges exist.
+                const xminNow = pendingGaps.length > 0 ? await selectSnapshotXmin(tx) : "0";
+                const events = await fetchPendingEvents(tx, oldCursor, batchSize, toIdRanges(pendingGaps));
+                const fetchedIds = events.map((e) => e.id);
+                const truncated = events.length === batchSize;
+                // Burnt: no visible id this turn, the fetch is proven to have
+                // covered the range (LIMIT can otherwise cut it off early), and
+                // xmin passed its recorded xmax — every xact that could still
+                // produce a row has finished without one ever appearing.
+                const { burnt: burntGaps, surviving: survivingGaps } = partitionBurntGaps(pendingGaps, fetchedIds, truncated, xminNow);
+                // skip: nothing to deliver and no burnt gap to clean up. The
+                // doPass idle pre-check (selectIdleConsumerKeys) already keeps the
+                // common idle case out of this transaction entirely — reaching
+                // here with an empty fetch means the pre-check couldn't prove
+                // idleness (pending gaps forced a fetch, or a race window). By this
+                // point acquireConsumerState's FOR UPDATE has already taken the
+                // lock, so skipping markProcessing/persistConsumerOutcome only
+                // avoids extra writes, not that lock's own WAL record.
+                if (events.length === 0 && burntGaps.length === 0) {
+                    span.setAttribute("consumer.skip_reason", "no_pending_events");
+                    return;
+                }
+                await markProcessing(tx, consumer.name, instanceId);
+                const outcome = events.length > 0
+                    ? await deliverEvents(consumer, events, context, maxAttempts, acquired.state)
+                    : {
+                        cursor: oldCursor,
+                        attempts: acquired.state.attempts,
+                        lastError: acquired.state.lastError,
+                        deadLettered: false,
+                        processed: 0,
+                        failed: 0,
+                        resolvedPendingIds: [],
+                    };
+                processed = outcome.processed;
+                failed = outcome.failed;
+                // Carve delivered/skip-applied ids out of the surviving ranges —
+                // an id still fetched-but-unresolved (halt-on-poison stopped before
+                // it) stays put, since it's below the cursor and only pending_gaps
+                // will ever retry it.
+                const keptGaps = survivingGaps.flatMap((gap) => splitRangeExcludingIds(gap, outcome.resolvedPendingIds));
+                // New gaps as ranges between consecutive fetched ids, so a huge id
+                // jump (retention prune) costs one entry, not one per missing id.
+                // Their xmax is read after the fetch; any later read is only more
+                // conservative, so it is fetched lazily.
+                const newWindowIds = fetchedIds.filter((id) => id > oldCursor && id <= outcome.cursor);
+                const newGapBounds = [];
+                let prev = oldCursor;
+                for (const id of newWindowIds) {
+                    if (id > prev + 1n)
+                        newGapBounds.push([prev + 1n, id - 1n]);
+                    prev = id;
+                }
+                const xmaxNow = newGapBounds.length > 0 ? await selectSnapshotXmax(tx) : "";
+                const newGaps = newGapBounds.map(([from, to]) => ({
+                    from: from.toString(),
+                    to: to.toString(),
+                    xmax: xmaxNow,
+                }));
+                const persistedOutcome = {
+                    ...outcome,
+                    pendingGaps: [...keptGaps, ...newGaps],
+                };
+                await persistConsumerOutcome(tx, consumer.name, instanceId, persistedOutcome);
+                await emitLagFromTx(tx, consumer.name, instanceId, outcome.cursor, meter);
+            });
+            // Pass completed without throwing — clear any backoff from a prior
+            // failure so the next unrelated outage gets its own fresh delay
+            // instead of inheriting a partially-escalated one.
+            consumerBackoff.delete(`${consumer.name}:${instanceId}`);
+            emitEventConsumerPassOutcome(meter, { consumer: consumer.name, instanceId }, processed, failed);
+            span.setAttribute("consumer.processed", processed);
+            span.setAttribute("consumer.failed", failed);
+            span.setStatus(failed === 0 ? "ok" : "error");
+        }
+        catch (e) {
+            // Unexpected: a handler error is caught inside deliverEvents and
+            // surfaces via `failed`, so anything landing here is infrastructure
+            // (db connection lost, serialization, standard-metrics not registered
+            // on this meter). Don't let one consumer's outage stall the others,
+            // but do log — a silent rollback here looks like "at-most-once" to
+            // callers and at-least-once-with-duplicate-delivery on the next pass;
+            // neither is what we want, so ops needs to see it.
+            const msg = e instanceof Error ? e.message : String(e);
+            if (context.log) {
+                context.log.error(`[event-dispatcher] ${consumer.name} pass failed: ${msg}`);
+            }
+            else {
+                // No logger wired onto this context (e.g. createEventDispatcher used
+                // standalone) — console.error is the only way ops still sees this in
+                // pod logs instead of the failure vanishing with the rolled-back tx.
+                console.error(`[event-dispatcher] ${consumer.name} pass failed: ${msg}`);
+            }
+            span.setStatus("error", msg);
+            // The whole pass rolled back, so markProcessing/persistConsumerOutcome
+            // never committed — attempts/last_error are stuck at their pre-pass
+            // values. Record the failure in its OWN transaction so it survives.
+            // Best-effort: this write must never itself take down the dispatcher,
+            // so any failure here is swallowed after logging.
+            try {
+                await db.begin(async (tx) => {
+                    await persistConsumerPassFailure(tx, consumer.name, instanceId, msg);
+                });
+            }
+            catch (persistErr) {
+                const persistMsg = persistErr instanceof Error ? persistErr.message : String(persistErr);
+                const failureMsg = `[event-dispatcher] ${consumer.name} failed to persist pass-failure state: ${persistMsg}`;
+                if (context.log) {
+                    context.log.error(failureMsg);
+                }
+                else {
+                    console.error(failureMsg);
+                }
+            }
+            // Escalate the backoff so this consumer isn't retried on the very
+            // next tick — see PASS_FAILURE_BASE_BACKOFF_MS/MAX_MS above.
+            const key = `${consumer.name}:${instanceId}`;
+            const consecutiveFailures = (consumerBackoff.get(key)?.consecutiveFailures ?? 0) + 1;
+            const delayMs = Math.min(PASS_FAILURE_BASE_BACKOFF_MS * 2 ** (consecutiveFailures - 1), PASS_FAILURE_MAX_BACKOFF_MS);
+            consumerBackoff.set(key, { consecutiveFailures, retryAtMs: Date.now() + delayMs });
+        }
+        finally {
+            span.end();
+        }
+        return { processed, failed };
+    }
+    return {
+        consumers,
+        async start() {
+            // skip: already running, idempotent
+            if (running)
+                return;
+            running = true;
+            // Pre-register consumer state rows. Without this, a consumer first
+            // bootstraps lazily on its first runOnce — and if prune runs between
+            // "process came up" and "first pass landed", prune wouldn't see the
+            // consumer in the state table, would delete events past its (absent)
+            // cursor, and the consumer's first pass would silently skip them.
+            //
+            // Pre-registering turns every consumer into a row-with-cursor-0 the
+            // moment the dispatcher starts — so the retention guard
+            // (pruneEvents → ConsumerLagError) correctly refuses to prune past
+            // any consumer that exists, including freshly-deployed ones.
+            await preRegisterConsumers(db, consumers, options.instanceId);
+            preRegistered = true;
+            timer = setInterval(() => {
+                void runOnce().catch(() => {
+                    // skip: per-consumer errors already recorded in the state row
+                });
+            }, pollIntervalMs);
+            // NOTIFY-based wake-up: subscribe on the same channel that
+            // event-store.append fires on commit. Fires runOnce directly, no
+            // polling round-trip. The timer stays on as a belt-and-braces
+            // fallback (dropped subscriptions, missed commits under load).
+            //
+            // Observability: the gauge kumiko_event_dispatcher_listen_connected
+            // flips to 1 on initial subscribe and again on every reconnect that
+            // succeeds (onlisten callback). It drops to 0 on the initial connect
+            // failure, on stop() and when the idle pre-check detects a DB outage.
+            // An outage shorter than one poll tick is not detected; the gauge then
+            // stays at 1 until postgres.js's delayed re-LISTEN lands.
+            emitEventDispatcherListenConnected(meter, false);
+            if (options.pgClient) {
+                try {
+                    const sub = await options.pgClient.listen(EVENTS_PUBSUB_CHANNEL, () => {
+                        void runOnce().catch(() => {
+                            // skip: per-consumer errors already recorded in the state row
+                        });
+                    }, () => {
+                        // Fires on initial connect and again when postgres.js's own
+                        // re-LISTEN lands after a drop; during an outage that re-LISTEN
+                        // waits in its connection backoff (measured 10-30s after the DB
+                        // returns), polling covers delivery meanwhile. Never unlisten()
+                        // + listen() by hand: that leaves duplicate listeners.
+                        // ponytail: no own LISTEN reconnect, the backoff gap is polled;
+                        // add a dedicated LISTEN connection if prod wires a pgClient
+                        // and that latency matters.
+                        emitEventDispatcherListenConnected(meter, true);
+                    });
+                    pgUnlisten = sub.unlisten;
+                }
+                catch (e) {
+                    emitEventDispatcherListenConnected(meter, false);
+                    const msg = e instanceof Error ? e.message : String(e);
+                    context.log?.error(`[event-dispatcher] pg LISTEN failed: ${msg}`);
+                }
+            }
+        },
+        async stop() {
+            // skip: already stopped, idempotent
+            if (!running)
+                return;
+            running = false;
+            if (timer) {
+                clearInterval(timer);
+                timer = null;
+            }
+            if (pgUnlisten) {
+                await pgUnlisten().catch(() => {
+                    // skip: unlisten failure only matters during shutdown — the
+                    // subscription is being torn down anyway.
+                });
+                pgUnlisten = null;
+                emitEventDispatcherListenConnected(meter, false);
+            }
+            // Drain any in-flight pass so shutdown observes consistent state —
+            // including one still stuck in the idle pre-check above inFlightTurns.
+            await drainInFlightPasses();
+            await drainInFlightTurns();
+            // preRegistered stays true — the rows survive stop(). runOnce()
+            // after a stop() still works (tests stop the timer and then drain
+            // deterministically).
+        },
+        async ensureRegistered() {
+            await preRegisterConsumers(db, consumers, options.instanceId);
+            preRegistered = true;
+        },
+        drain: drainInFlightTurns,
+        runOnce,
+    };
+}
+export { disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher-admin.js";

package/dist/pipeline/event-retention.d.ts

@@ -0,0 +1,20 @@
+import type { DbConnection } from "../db/connection.js";
+export type PruneEventsOptions = {
+    readonly olderThan?: Temporal.Instant;
+    readonly olderThanDays?: number;
+    readonly aggregateTypes: readonly string[];
+    readonly dryRun?: boolean;
+};
+export type PruneEventsResult = {
+    readonly deletedCount: number;
+    readonly cutoff: Temporal.Instant;
+    readonly aggregateTypes: readonly string[];
+    readonly dryRun: boolean;
+};
+export declare class ConsumerLagError extends Error {
+    readonly laggingConsumer: string;
+    readonly consumerCursor: bigint;
+    readonly maxCandidateId: bigint;
+    constructor(laggingConsumer: string, consumerCursor: bigint, maxCandidateId: bigint);
+}
+export declare function pruneEvents(db: DbConnection, options: PruneEventsOptions): Promise<PruneEventsResult>;

package/dist/pipeline/event-retention.js

@@ -0,0 +1,78 @@
+import { lockEventConsumersShareMode } from "../db/queries/event-consumer.js";
+import { deleteMany, selectMany, transaction } from "../db/query.js";
+import { eventsTable } from "../event-store/index.js";
+import { eventConsumerStateTable } from "./event-consumer-state.js";
+export class ConsumerLagError extends Error {
+    laggingConsumer;
+    consumerCursor;
+    maxCandidateId;
+    constructor(laggingConsumer, consumerCursor, maxCandidateId) {
+        super(`Consumer "${laggingConsumer}" is behind the prune candidates ` +
+            `(cursor=${consumerCursor}, max candidate event id=${maxCandidateId}). ` +
+            `Pruning would silently drop deliveries. Catch up, disable, or skip the consumer first.`);
+        this.laggingConsumer = laggingConsumer;
+        this.consumerCursor = consumerCursor;
+        this.maxCandidateId = maxCandidateId;
+        this.name = "ConsumerLagError";
+    }
+}
+function resolveCutoff(opts) {
+    if (opts.olderThan)
+        return opts.olderThan;
+    const days = opts.olderThanDays;
+    if (days === undefined || days <= 0) {
+        throw new Error("pruneEvents: pass olderThan (Temporal.Instant) or olderThanDays (positive number).");
+    }
+    return Temporal.Now.instant().subtract({ hours: days * 24 });
+}
+export async function pruneEvents(db, options) {
+    const cutoff = resolveCutoff(options);
+    if (!options.aggregateTypes || options.aggregateTypes.length === 0) {
+        throw new Error("pruneEvents: aggregateTypes is required and must be non-empty. Pruning the event log is destructive — name the aggregate types to delete explicitly.");
+    }
+    const aggregateTypes = options.aggregateTypes;
+    const dryRun = options.dryRun === true;
+    return transaction(db, async (tx) => {
+        // Serialise against consumer-bootstrap INSERTs. Without this, the race
+        // is: prune reads consumers (snapshot misses a consumer bootstrapping
+        // in a parallel tx) → consumer commits its row with
+        // lastProcessedEventId=0 → prune deletes events below its new cursor
+        // → first pass of the new consumer silently skips the deleted events.
+        //
+        // SHARE is the lightest table-mode that conflicts with ROW EXCLUSIVE
+        // (the mode INSERT/UPDATE/DELETE take). Concurrent INSERTs on the
+        // consumer table queue until this TX commits; concurrent UPDATEs
+        // (cursor advances) do too, but prune is measured in milliseconds and
+        // pausing cursor advances for that window is cheap insurance against
+        // a silent data-loss bug.
+        await lockEventConsumersShareMode(tx);
+        // Step 1 — collect candidate event ids.
+        const candidates = await selectMany(tx, eventsTable, {
+            aggregateType: [...aggregateTypes],
+            createdAt: { lt: cutoff },
+        });
+        if (candidates.length === 0) {
+            return { deletedCount: 0, cutoff, aggregateTypes, dryRun };
+        }
+        const maxCandidateId = candidates.reduce((acc, row) => (row.id > acc ? row.id : acc), candidates[0]?.id ?? 0n);
+        // Step 2 — safety guard: check every ACTIVE consumer has moved past
+        // the candidate set. Disabled consumers are intentionally excluded —
+        // ops disables them precisely to park them while pruning happens.
+        // The SHARE lock above guarantees this SELECT sees a complete view:
+        // no new consumer can INSERT a fresh-cursor row between here and the
+        // DELETE below.
+        const activeConsumers = await selectMany(tx, eventConsumerStateTable, { status: { ne: "disabled" } });
+        for (const consumer of activeConsumers) {
+            if (consumer.lastProcessedEventId < maxCandidateId) {
+                throw new ConsumerLagError(consumer.name, consumer.lastProcessedEventId, maxCandidateId);
+            }
+        }
+        if (dryRun) {
+            return { deletedCount: candidates.length, cutoff, aggregateTypes, dryRun: true };
+        }
+        // Step 3 — actual delete, bounded to the candidate set.
+        const candidateIds = candidates.map((c) => c.id);
+        await deleteMany(tx, eventsTable, { id: candidateIds });
+        return { deletedCount: candidateIds.length, cutoff, aggregateTypes, dryRun: false };
+    });
+}

package/dist/pipeline/idempotency.d.ts

@@ -0,0 +1,19 @@
+import type { Redis } from "ioredis";
+export type IdempotencyCheckResult = {
+    readonly status: "cached";
+    readonly result: string;
+} | {
+    readonly status: "acquired";
+    readonly token: string;
+};
+export type IdempotencyGuard = {
+    check(tenantId: string, userId: string, requestId: string): Promise<IdempotencyCheckResult>;
+    store(tenantId: string, userId: string, requestId: string, token: string, result: unknown): Promise<void>;
+    release(tenantId: string, userId: string, requestId: string, token: string): Promise<void>;
+};
+export declare function createIdempotencyGuard(redis: Redis, options?: {
+    ttlSeconds?: number;
+    pendingTtlSeconds?: number;
+    waitTimeoutMs?: number;
+    pollIntervalMs?: number;
+}): IdempotencyGuard;