@cosmicdrift/kumiko-framework 0.353.0 → 0.354.1

package/dist/pipeline/dispatch-shared.js

@@ -1,7 +1,7 @@
 import { escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
 import { requestContext, runWithOrigin } from "../api/request-context.js";
 import { runInSavepoint, selectMany } from "../db/query.js";
-import { createTenantDb, createUncheckedSystemDb, hasTenantColumn, } from "../db/tenant-db.js";
+import { createTenantDb, createUncheckedSystemDb, hasTenantColumn, runInOwnTransaction, } from "../db/tenant-db.js";
 import { createDerivativesContext } from "../derivatives/derivatives-context.js";
 import { createSystemUser } from "../engine/system-user.js";
 import { isRateLimitDisabled, resolveAgentExposure } from "../engine/types/index.js";
@@ -40,6 +40,7 @@ export const TENANT_TIMEZONE_CONFIG_KEY = "tenant:config:timezone";
 // exercises both against the real config feature.
 export const CONFIG_WRITE_SET_TYPE = "config:write:set";
 export const CONFIG_WRITE_RESET_TYPE = "config:write:reset";
+export const DEFAULT_MEMBER_READ_TIMEOUT_MS = 10_000;
 // Narrowing-helper: AppContext.db ist DbConnection|TenantDb|undefined. Die
 // dispatch-Pfade brauchen DbConnection (oder DbTx aus Caller-Scope) für
 // appendEvent/projection-writes; TenantDb-Branch wird hier ausgeschlossen.
@@ -113,14 +114,57 @@ function denyingJobRunnerProxy() {
         },
     });
 }
+// Allowlist (not denylist): a method added to the target later stays denied
+// until someone decides it is read-only.
+function allowlistedReadOnlyProxy(target, allowedMethods) {
+    return new Proxy(target, {
+        get(inner, prop) {
+            if (typeof prop === "string" && allowedMethods.has(prop)) {
+                const method = Reflect.get(inner, prop, inner);
+                return typeof method === "function" ? method.bind(inner) : method;
+            }
+            // Symbol reads and "then" come from await/Promise.resolve/inspect, not from handler code.
+            if (typeof prop === "symbol" || prop === "then")
+                return undefined;
+            throw memberResolutionReadOnlyDenied();
+        },
+    });
+}
+const MEMBER_READ_SEARCH_METHODS = new Set(["search"]);
+const MEMBER_READ_ENTITY_CACHE_METHODS = new Set(["get", "mget"]);
+const MEMBER_READ_REDIS_METHODS = new Set([
+    "get",
+    "mget",
+    "exists",
+    "ttl",
+    "pttl",
+    "strlen",
+    "type",
+    "hget",
+    "hmget",
+    "hgetall",
+    "hexists",
+    "hlen",
+    "sismember",
+    "smembers",
+    "scard",
+    "llen",
+    "lrange",
+    "zscore",
+    "zcard",
+    "zrange",
+]);
 // Every write/side-effect surface throws, so "no writeAsMember" holds even
 // if a handler tries ctx.write directly on a resolved member principal.
+// Deliberately open: `db` (guarded by the READ ONLY transaction + memberReadOnly grant),
+// and read-only methods of searchAdapter / redis / entityCache (allowlisted).
 export function applyMemberResolutionReadOnly(handlerContext) {
     return {
         ...handlerContext,
         // `db` stays open — executeQuery runs the whole handler in a Postgres READ ONLY
         // transaction, and its `memberReadOnly` grant denies every raw-runner handout.
         dbOutsideTransaction: undefined,
+        outsideTransaction: undefined,
         write: denyMemberResolutionWrite,
         writeAs: denyMemberResolutionWrite,
         // A resolved member cannot switch identity either — the target would get a normal, writable context.
@@ -144,6 +188,15 @@ export function applyMemberResolutionReadOnly(handlerContext) {
         derivatives: undefined,
         ...(handlerContext.jobRunner && { jobRunner: denyingJobRunnerProxy() }),
         ...(handlerContext.notify && { notify: denyMemberResolutionWrite }),
+        ...(handlerContext.searchAdapter && {
+            searchAdapter: allowlistedReadOnlyProxy(handlerContext.searchAdapter, MEMBER_READ_SEARCH_METHODS),
+        }),
+        ...(handlerContext.redis && {
+            redis: allowlistedReadOnlyProxy(handlerContext.redis, MEMBER_READ_REDIS_METHODS),
+        }),
+        ...(handlerContext.entityCache && {
+            entityCache: allowlistedReadOnlyProxy(handlerContext.entityCache, MEMBER_READ_ENTITY_CACHE_METHODS),
+        }),
     };
 }
 export async function buildHandlerContext(ctx, type, user, origin, tx, afterCommitHooks, includeDeleted, isPostCommit) {
@@ -210,6 +263,11 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
         ? createSystemScopedDbGuard(type, "dbOutsideTransaction", "ctx.systemDb.outsideTransaction.assertTenantMatch(...) or " +
             "ctx.systemDb.outsideTransaction.acknowledgeCrossTenant(...)")
         : rawDbOutsideTransaction;
+    // Not offered to r.systemScope() handlers: the raw outside-tx db is unfiltered in
+    // system mode, so fn would hand them the same cross-tenant door the guards above close.
+    const outsideTransaction = !isSystem && rawDbOutsideTransaction
+        ? (fn) => runInOwnTransaction(rawDbOutsideTransaction, fn)
+        : undefined;
     const log = context.log?.child({
         handler: type,
         tenantId: user.tenantId,
@@ -255,7 +313,9 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
     // _fileProviderResolver when a file-provider plugin is mounted; falls back
     // to a statically-injected context.files (tests).
     const fileResolver = context._fileProviderResolver;
-    const files = fileResolver ? createFileContext(() => fileResolver(user.tenantId)) : context.files;
+    const files = fileResolver
+        ? createFileContext(() => fileResolver(user.tenantId), user.tenantId)
+        : context.files;
     // ctx.derivatives builds on ctx.files (needs a FileContext to read the
     // original + write the variant) plus db (to look up the FileRef row) — so
     // it can only be constructed exactly when files+db both resolved; falls
@@ -302,13 +362,13 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
         ungatedMemberReader ??= createMemberReaderFn(ctx, user.tenantId, tx);
         return ungatedMemberReader(userId, qn, payload);
     }, { ...identitySwitchAudit, tenantId: user.tenantId });
+    // queryProjection works against both single-stream and multi-stream
+    // projections. MSPs without a table cannot be queried — those are
+    // side-effect-only consumers (no state to read back).
+    const findProjectionTable = (qualifiedName) => registry.getAllProjections().get(qualifiedName)?.table ??
+        registry.getAllMultiStreamProjections().get(qualifiedName)?.table;
     const ungatedQueryProjection = async (qualifiedName, queryOptions) => {
-        // queryProjection works against both single-stream and multi-stream
-        // projections. MSPs without a table cannot be queried — those are
-        // side-effect-only consumers (no state to read back).
-        const singleProj = registry.getAllProjections().get(qualifiedName);
-        const mspProj = registry.getAllMultiStreamProjections().get(qualifiedName);
-        const projTable = singleProj?.table ?? mspProj?.table;
+        const projTable = findProjectionTable(qualifiedName);
         if (!projTable) {
             const singleNames = [...registry.getAllProjections().keys()];
             const mspNames = [...registry.getAllMultiStreamProjections().keys()].filter((n) => registry.getAllMultiStreamProjections().get(n)?.table);
@@ -330,7 +390,11 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
         const rows = await selectMany(dbSource, projTable, where);
         return rows; // @cast-boundary engine-payload
     };
-    const queryProjection = createGatedProjectionReader(`handler "${type}"`, hasIdentitySwitchGrant, ungatedQueryProjection, identitySwitchAudit);
+    const queryProjection = createGatedProjectionReader(`handler "${type}"`, hasIdentitySwitchGrant, ungatedQueryProjection, identitySwitchAudit, (qualifiedName) => {
+        const projTable = findProjectionTable(qualifiedName);
+        // Unknown projection: keep the grant check first, the ungated reader reports the real error.
+        return projTable === undefined || hasTenantColumn(projTable);
+    });
     const bridge = {
         query: (targetType, payload) => executeQuery(ctx, targetType, payload, user, origin, tx), // @wrapper-known semantic-alias
         queryAs: identitySwitch.queryAs,
@@ -624,6 +688,7 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
         registry,
         db: exposedDb,
         dbOutsideTransaction,
+        ...(outsideTransaction && { outsideTransaction }),
         ...(systemDb && { systemDb }),
         log,
         notify,

package/dist/pipeline/dispatch-stream.d.ts

@@ -1,3 +1,7 @@
 import type { SessionUser } from "../engine/types/index.js";
 import { type DispatchContext, type WriteOrigin } from "./dispatch-shared.js";
-export declare function executeStream(ctx: DispatchContext, type: string, payload: unknown, user: SessionUser, origin: WriteOrigin): AsyncGenerator<unknown>;
+export type StreamOptions = {
+    /** JWT exp (epoch seconds) the stream was opened with; the stream ends when it passes. */
+    readonly tokenExpiresAtSec?: number | undefined;
+};
+export declare function executeStream(ctx: DispatchContext, type: string, payload: unknown, user: SessionUser, origin: WriteOrigin, options?: StreamOptions): AsyncGenerator<unknown>;

package/dist/pipeline/dispatch-stream.js

@@ -1,4 +1,5 @@
 import { accessInvalidationCredentialFor } from "../api/sse-broker.js";
+import { scheduleTokenExpiry } from "../api/token-expiry-timer.js";
 import { hasAccess } from "../engine/access.js";
 import { AccessDeniedError, memberResolutionReadOnlyDenied, NotFoundError, validationErrorFromZod, } from "../errors/index.js";
 import { assertNoSecretLeak } from "../secrets/index.js";
@@ -10,10 +11,10 @@ import { handlerAccessError } from "./handler-access-error.js";
 // of returning a single response. streamHandler never entity-maps (unlike
 // write/queryHandler — see feature-entity-handlers.ts), so there's no
 // field-access filter or postQuery-hook stage to run here.
-export async function* executeStream(ctx, type, payload, user, origin) {
-    yield* runStreamInstrumented(ctx, type, user, () => executeStreamInner(ctx, type, payload, user, origin));
+export async function* executeStream(ctx, type, payload, user, origin, options) {
+    yield* runStreamInstrumented(ctx, type, user, () => executeStreamInner(ctx, type, payload, user, origin, options));
 }
-async function* executeStreamInner(ctx, type, payload, user, origin) {
+async function* executeStreamInner(ctx, type, payload, user, origin, options) {
     const { registry } = ctx;
     const handler = registry.getStreamHandler(type);
     if (!handler)
@@ -32,15 +33,18 @@ async function* executeStreamInner(ctx, type, payload, user, origin) {
     if (!parsed.success) {
         throw validationErrorFromZod(parsed.error);
     }
-    // Idle (heartbeat-only) streams must also cut on access revoke — race each
-    // pull against an invalidated Deferred instead of a post-chunk boolean.
-    let resolveInvalidated;
-    const invalidated = new Promise((resolve) => {
-        resolveInvalidated = resolve;
+    // Idle (heartbeat-only) streams must also cut on access revoke or token expiry — race each
+    // pull against an ended Deferred instead of a post-chunk boolean.
+    let resolveEnded;
+    const ended = new Promise((resolve) => {
+        resolveEnded = resolve;
     });
     const unsubscribeAccessInvalidation = ctx.sseBroker?.subscribeAccessInvalidation(user.id, () => {
-        resolveInvalidated?.();
+        resolveEnded?.("revoked");
     }, accessInvalidationCredentialFor(user));
+    const expiryTimer = scheduleTokenExpiry(options?.tokenExpiresAtSec, () => {
+        resolveEnded?.("expired");
+    });
     let iterator;
     // When access is revoked mid-pull, `iterator.next()` is still in flight.
     // Awaiting `iterator.return()` in that state deadlocks async generators in
@@ -55,14 +59,16 @@ async function* executeStreamInner(ctx, type, payload, user, origin) {
             const nextPull = iterator.next();
             const outcome = await Promise.race([
                 nextPull.then((result) => ({ kind: "chunk", result })),
-                invalidated.then(() => ({ kind: "invalidated" })),
+                ended.then((reason) => ({ kind: "ended", reason })),
             ]);
-            if (outcome.kind === "invalidated") {
+            if (outcome.kind === "ended") {
                 abandonedForInvalidation = true;
                 void nextPull.catch(() => { });
                 void iterator.return?.(undefined)?.then(undefined, () => { });
                 throw new AccessDeniedError({
-                    message: `access revoked mid-stream for ${type}`,
+                    message: outcome.reason === "expired"
+                        ? `token expired mid-stream for ${type}`
+                        : `access revoked mid-stream for ${type}`,
                     details: { handler: type },
                 });
             }
@@ -75,6 +81,7 @@ async function* executeStreamInner(ctx, type, payload, user, origin) {
     }
     finally {
         unsubscribeAccessInvalidation?.();
+        clearTimeout(expiryTimer);
         // Close the generator so cleanup runs; skip awaiting return() after
         // access-revoke abandonment — overlapping next()+return() can deadlock.
         if (iterator !== undefined && !abandonedForInvalidation) {

package/dist/pipeline/dispatcher.d.ts

@@ -4,6 +4,7 @@ import type { EffectiveFeaturesResolver } from "../engine/tier-resolver-extensio
 import type { TenantId } from "../engine/types/identifiers.js";
 import type { ActiveMembershipResult, AppContext, DispatchWriteRef, JobRunnerRef, MemberReader, Registry, SessionUser, WriteResult } from "../engine/types/index.js";
 import type { BatchCommand, BatchResult } from "./dispatch-shared.js";
+import { type StreamOptions } from "./dispatch-stream.js";
 import { type HandlerType } from "./dispatcher-utils.js";
 import type { IdempotencyGuard } from "./idempotency.js";
 import type { LifecycleHooks } from "./lifecycle-pipeline.js";
@@ -14,6 +15,7 @@ export type DispatcherOptions = {
     lifecycle?: LifecycleHooks;
     jobRunner?: JobRunnerRef;
     membershipQuery?: string;
+    memberReadTimeoutMs?: number;
     effectiveFeatures?: EffectiveFeaturesResolver;
     sseBroker?: SseBroker;
     cacheSync?: CacheSyncBus | null;
@@ -21,7 +23,7 @@ export type DispatcherOptions = {
 export type Dispatcher = {
     write(type: HandlerType, payload: unknown, user: SessionUser, requestId?: string): Promise<WriteResult>;
     query(type: HandlerType, payload: unknown, user: SessionUser): Promise<unknown>;
-    stream(type: HandlerType, payload: unknown, user: SessionUser): AsyncGenerator<unknown>;
+    stream(type: HandlerType, payload: unknown, user: SessionUser, options?: StreamOptions): AsyncGenerator<unknown>;
     command(type: HandlerType, payload: unknown, user: SessionUser): Promise<void>;
     batch(commands: readonly BatchCommand[], user: SessionUser, requestId?: string): Promise<BatchResult>;
     resolveAuthClaims(user: SessionUser): Promise<Record<string, unknown>>;

package/dist/pipeline/dispatcher.js

@@ -1,5 +1,4 @@
 import { runWithWriteOrigin } from "../api/request-context.js";
-import { DEPRECATED_CROSS_TENANT_SIGNAL, isDeprecatedCrossTenantHandler, } from "../engine/entity-handlers.js";
 import { TENANT_MEMBERSHIPS_QUERY } from "../engine/extension-names.js";
 import { InternalError, reraiseAsKumikoError } from "../errors/index.js";
 import { createEscapeHatchReportWindow, getFallbackMeter, getFallbackTracer, registerStandardMetrics, } from "../observability/index.js";
@@ -7,7 +6,7 @@ import { CACHE_SYNC_TOPICS, isTenantConfigSyncMessage } from "../redis/cache-syn
 import { INTERACTIVE_SIGN_IN_POLICY, resolveActiveMembershipFn } from "./active-membership.js";
 import { runBatch, unwrapSingle } from "./dispatch-batch.js";
 import { executeQuery } from "./dispatch-query.js";
-import { resolveAuthClaimsFn, TENANT_TIMEZONE_CONFIG_KEY } from "./dispatch-shared.js";
+import { DEFAULT_MEMBER_READ_TIMEOUT_MS, resolveAuthClaimsFn, TENANT_TIMEZONE_CONFIG_KEY, } from "./dispatch-shared.js";
 import { executeStream } from "./dispatch-stream.js";
 import { resolveType } from "./dispatcher-utils.js";
 import { createMemberReaderFn } from "./member-reader.js";
@@ -49,6 +48,12 @@ export function dispatcherToWriteRef(dispatcher) {
 export function createDispatcher(registry, context, options = {}) {
     const { idempotency, lifecycle, jobRunner, effectiveFeatures, sseBroker } = options;
     const membershipQuery = options.membershipQuery ?? TENANT_MEMBERSHIPS_QUERY;
+    const memberReadTimeoutMs = options.memberReadTimeoutMs ?? DEFAULT_MEMBER_READ_TIMEOUT_MS;
+    if (!Number.isSafeInteger(memberReadTimeoutMs) || memberReadTimeoutMs <= 0) {
+        throw new InternalError({
+            message: `DispatcherOptions.memberReadTimeoutMs must be a positive integer, got ${memberReadTimeoutMs}`,
+        });
+    }
     // Pre-build tables and transition maps for auto-guard (avoid per-request allocation)
     const tableCache = new Map();
     const transitionCache = new Map();
@@ -75,17 +80,6 @@ export function createDispatcher(registry, context, options = {}) {
     // Ensure standard metrics exist on whatever meter we ended up with.
     // Idempotent: buildServer may have registered them already.
     registerStandardMetrics(dispatcherMeter);
-    for (const def of [
-        ...registry.getAllWriteHandlers().values(),
-        ...registry.getAllQueryHandlers().values(),
-    ]) {
-        if (isDeprecatedCrossTenantHandler(def.handler)) {
-            context.log?.warn(DEPRECATED_CROSS_TENANT_SIGNAL, {
-                handler: def.name,
-                migration: 'replace crossTenant: true with escapeHatch: { reason: "<why this operator handler reads/writes every tenant>" } (bun node_modules/@cosmicdrift/kumiko-framework/src/scripts/codemod/migrate-cross-tenant.ts)',
-            });
-        }
-    }
     const ctx = {
         registry,
         appContext: context,
@@ -102,6 +96,7 @@ export function createDispatcher(registry, context, options = {}) {
         tracer: dispatcherTracer,
         meter: dispatcherMeter,
         membershipQuery,
+        memberReadTimeoutMs,
     };
     const dispatcher = {
         async write(typeOrRef, payload, user, requestId) {
@@ -116,9 +111,9 @@ export function createDispatcher(registry, context, options = {}) {
             const origin = rootWriteOrigin(registry, type, user);
             return runWithWriteOrigin(origin, () => executeQuery(ctx, type, payload, user, origin));
         },
-        stream: (typeOrRef, payload, user) => {
+        stream: (typeOrRef, payload, user, options) => {
             const type = resolveType(typeOrRef);
-            return executeStream(ctx, type, payload, user, rootWriteOrigin(registry, type, user));
+            return executeStream(ctx, type, payload, user, rootWriteOrigin(registry, type, user), options);
         },
         async command(typeOrRef, payload, user) {
             const type = resolveType(typeOrRef);

package/dist/pipeline/event-consumer-state.js

@@ -1,6 +1,5 @@
 import { bigint, index, instant, integer, jsonb, table as pgTable, primaryKey, sql, text, } from "../db/dialect.js";
 import { alterTableAddColumn } from "../db/queries/ddl.js";
-import { deleteOrphanedPerInstanceConsumerRows } from "../db/queries/event-consumer.js";
 import { tableExists } from "../db/schema-inspection.js";
 import { unsafePushTables } from "../stack/index.js";
 // Reserved sentinel used in the instance_id column for consumers whose
@@ -39,20 +38,6 @@ export const eventConsumerStateTable = pgTable("kumiko_event_consumers", {
     pk: primaryKey({ columns: [t.name, t.instanceId] }),
     statusIdx: index("kumiko_event_consumers_status_idx").on(t.status),
 }));
-// fw#2625: these three consumers moved from delivery: "per-instance" to
-// "shared" (see system-hooks.ts and bundled-features/feature-toggles/
-// feature.ts). Their old per-instance rows are now orphaned — no dispatcher
-// will ever advance them again — and pruneEvents stays pinned to whichever
-// one has the lowest cursor forever. Named here (not imported from
-// system-hooks.ts / feature.ts) to avoid a pipeline → system-hooks /
-// bundled-features dependency for three string literals; those modules'
-// own exported constants (or, for feature-toggles, the qualified consumer
-// name qualify() produces) are the source of truth these must match.
-const ORPHANED_PER_INSTANCE_CONSUMER_NAMES = [
-    "system:consumer:sse-broadcast",
-    "system:consumer:access-invalidation",
-    "feature-toggles:projection:toggle-cache-sync",
-];
 // Object-const form lets call sites write `ConsumerStatuses.disabled` instead
 // of the raw string, which keeps status checks refactor-safe. The runtime
 // value on each field is the same string the DB stores — no mapping needed.
@@ -84,19 +69,8 @@ export async function createEventConsumerStateTable(db) {
         /* ifNotExists */ true);
         await alterTableAddColumn(db, "kumiko_event_consumers", "last_failed_event_id", "bigint", "", "", 
         /* ifNotExists */ true);
-        // Runs on every boot, including mid-rolling-deploy: a new pod can delete
-        // an old pod's still-in-use per-instance row out from under it. That old
-        // pod's next acquireConsumer then finds no row, delivers no more SSE/
-        // access-invalidation events for the rest of its lifetime — a one-time,
-        // few-second gap that resolves itself as clients reconnect onto new pods
-        // during the same rollout. Not a replay risk: preRegisterConsumers only
-        // (re-)inserts a row at dispatcher startup, so the deleted pod won't
-        // silently get a fresh cursor-0 row and replay history mid-flight.
-        await deleteOrphanedPerInstanceConsumerRows(db, ORPHANED_PER_INSTANCE_CONSUMER_NAMES, SHARED_INSTANCE_SENTINEL);
         // skip: table (+ any missing column) is already up to date
         return;
     }
-    // A freshly created table has no rows to clean up — the DELETE only
-    // matters for a DB that already ran the old per-instance consumers.
     await unsafePushTables(db, { kumikoEventConsumers: eventConsumerStateTable });
 }

package/dist/pipeline/event-dispatcher-admin.js

@@ -3,7 +3,7 @@ import { selectNextEventIdAfter, selectSmallestVisibleIdInRanges, } from "../db/
 import { coerceRow, extractTableInfo, selectMany } from "../db/query.js";
 import { getEventsHighWaterMark } from "../event-store/index.js";
 import { eventConsumerStateTable, SHARED_INSTANCE_SENTINEL, } from "./event-consumer-state.js";
-import { rangeContainsId, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges.js";
+import { mergeContiguousGapRanges, rangeContainsId, splitRangeExcludingIds, toIdRanges, } from "./pending-gap-ranges.js";
 // --- Ops recovery surface ---
 //
 // These are intentionally verb-distinct; each maps to a CLI sub-command.
@@ -84,6 +84,19 @@ export async function enableConsumer(db, name, instanceId = SHARED_INSTANCE_SENT
     }
     return applyConsumerStatusTransition(db, name, instanceId, "idle");
 }
+// Delivery stops at the poison event, so ids between the cursor and it were
+// invisible during that pass (in-flight commits); keeping them as a pending gap
+// avoids losing them when the cursor jumps over.
+async function invisibleWindowGap(tx, cursor, poisonId) {
+    const windowStart = cursor + 1n;
+    if (poisonId <= windowStart)
+        return null;
+    return {
+        from: windowStart.toString(),
+        to: (poisonId - 1n).toString(),
+        xmax: await selectSnapshotXmax(tx),
+    };
+}
 // skipPoisonEvent skips the event whose handler failed last
 // (last_failed_event_id, set by the delivery's halt-on-poison branch): inside
 // pending_gaps it is split out of its range, otherwise the cursor moves onto
@@ -109,7 +122,7 @@ export async function skipPoisonEvent(db, name, instanceId = SHARED_INSTANCE_SEN
             const newPendingGaps = pendingGaps.flatMap((gap) => rangeContainsId(gap, pendingPoisonId)
                 ? splitRangeExcludingIds(gap, [pendingPoisonId])
                 : [gap]);
-            const raw = await removePendingGapReturning(tx, name, instanceId, newPendingGaps);
+            const raw = await removePendingGapReturning(tx, name, instanceId, mergeContiguousGapRanges(newPendingGaps));
             const updated = raw && coerceRow(raw, extractTableInfo(eventConsumerStateTable));
             if (!updated)
                 throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`);
@@ -130,15 +143,8 @@ export async function skipPoisonEvent(db, name, instanceId = SHARED_INSTANCE_SEN
         // Delivery stops at the poison event, so ids between the cursor and it were
         // invisible during that pass (in-flight commits). Jumping the cursor over them
         // would lose them; keep them as a pending gap like the dispatcher's own gaps.
-        const skippedWindowStart = before.lastProcessedEventId + 1n;
-        const gapToAdd = poisonId > skippedWindowStart
-            ? {
-                from: skippedWindowStart.toString(),
-                to: (poisonId - 1n).toString(),
-                xmax: await selectSnapshotXmax(tx),
-            }
-            : null;
-        const raw = await advanceConsumerPastEventReturning(tx, name, instanceId, poisonId, gapToAdd);
+        const gapToAdd = await invisibleWindowGap(tx, before.lastProcessedEventId, poisonId);
+        const raw = await advanceConsumerPastEventReturning(tx, name, instanceId, poisonId, mergeContiguousGapRanges(gapToAdd ? [...pendingGaps, gapToAdd] : pendingGaps));
         const updated = raw && coerceRow(raw, extractTableInfo(eventConsumerStateTable));
         if (!updated)
             throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`);

package/dist/pipeline/event-dispatcher-delivery.d.ts

@@ -68,4 +68,4 @@ export type PersistedConsumerOutcome = DeliveryOutcome & {
     readonly pendingGaps: readonly PendingGapEntry[];
 };
 export declare function persistConsumerOutcome(tx: DbTx, name: string, instanceId: string, outcome: PersistedConsumerOutcome): Promise<void>;
-export declare function emitLagFromTx(tx: DbTx, consumerName: string, instanceId: string, cursor: bigint, meter: Meter): Promise<void>;
+export declare function emitLagFromTx(tx: DbTx, consumerName: string, instanceId: string, cursor: bigint, meter: Meter, pendingGapRanges: number): Promise<void>;

package/dist/pipeline/event-dispatcher-delivery.js

@@ -6,7 +6,7 @@ import { selectEventsHeadId, selectPendingAndNewEventRows, } from "../db/queries
 import { coerceRow, extractTableInfo } from "../db/query.js";
 import { qnScope } from "../engine/qualified-name.js";
 import { eventsTable, toStoredEvent as rowToStoredEvent, } from "../event-store/index.js";
-import { emitDispatcherError, emitEventConsumerLag, getFallbackMeter, } from "../observability/index.js";
+import { emitDispatcherError, emitEventConsumerLag, emitEventConsumerPendingGaps, getFallbackMeter, } from "../observability/index.js";
 import { ConsumerStatuses, eventConsumerStateTable, SHARED_INSTANCE_SENTINEL, } from "./event-consumer-state.js";
 import { parseWriteOrigin } from "./write-origin.js";
 // Fails closed without throwing: a throw would poison the event for every consumer.
@@ -315,8 +315,9 @@ export async function persistConsumerOutcome(tx, name, instanceId, outcome) {
 // Emit the lag gauge inside the consumer pass's tx so ops sees a snapshot
 // consistent with the cursor we just advanced to. `MAX(id)` on the events
 // table is an O(1) reverse-index scan — cheap even under load.
-export async function emitLagFromTx(tx, consumerName, instanceId, cursor, meter) {
+export async function emitLagFromTx(tx, consumerName, instanceId, cursor, meter, pendingGapRanges) {
     const head = await selectEventsHeadId(tx);
     const lag = head > cursor ? Number(head - cursor) : 0;
     emitEventConsumerLag(meter, { consumer: consumerName, instanceId }, lag);
+    emitEventConsumerPendingGaps(meter, { consumer: consumerName, instanceId }, pendingGapRanges);
 }

package/dist/pipeline/event-dispatcher.js

@@ -1,15 +1,33 @@
+import { extractPgError } from "../db/pg-error.js";
 import { recordConsumerPassFailure, selectSnapshotXmax, selectSnapshotXmin, } from "../db/queries/event-consumer.js";
 import { SYSTEM_TENANT_ID } from "../engine/types/identifiers.js";
 import { EVENTS_PUBSUB_CHANNEL } from "../event-store/index.js";
 import { emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, getFallbackMeter, getFallbackTracer, } from "../observability/index.js";
 import { SHARED_INSTANCE_SENTINEL } from "./event-consumer-state.js";
 import { acquireConsumerState, consumerInstanceId, deliverEvents, emitLagFromTx, fetchPendingEvents, markProcessing, persistConsumerOutcome, preRegisterConsumers, selectIdleConsumerKeys, } from "./event-dispatcher-delivery.js";
-import { partitionBurntGaps, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges.js";
+import { mergeContiguousGapRanges, partitionBurntGaps, splitRangeExcludingIds, toIdRanges, } from "./pending-gap-ranges.js";
 const DEFAULT_BATCH_SIZE = 200;
 const DEFAULT_POLL_MS = 100;
 const DEFAULT_MAX_ATTEMPTS = 10;
 const DEFAULT_REARM_COOLDOWN_MS = 5 * 60_000;
 const DEFAULT_MAX_REARM_COUNT = 3;
+const PENDING_GAPS_WARN_THRESHOLD = 1000;
+// Only these say the database socket itself is gone; timeouts, schema or pool errors do not
+// imply the dedicated LISTEN connection dropped.
+const CONNECTION_LOSS_CODES = new Set([
+    "CONNECTION_CLOSED",
+    "CONNECTION_DESTROYED",
+    "CONNECTION_ENDED",
+    "CONNECT_TIMEOUT",
+    "ECONNREFUSED",
+    "ECONNRESET",
+    "ERR_POSTGRES_CONNECTION_CLOSED",
+    "57P01",
+]);
+function isConnectionLossError(e) {
+    const code = extractPgError(e)?.code;
+    return code !== undefined && CONNECTION_LOSS_CODES.has(code);
+}
 // Backoff for a consumer whose pass THREW (the processConsumer catch below)
 // — an infra-level failure (db.begin's callback rejected outside deliverEvents'
 // own per-event try/catch), not a poisoned event. Doubles per consecutive
@@ -52,6 +70,19 @@ export function createEventDispatcher(options) {
     // which is fine: a fresh process re-observing a still-dead consumer is
     // exactly the "still needs a human" signal ops wants.
     const reportedDeadConsumers = new Set();
+    // Same once-per-process dedup for the pending_gaps size warning.
+    const reportedLargeGapSets = new Set();
+    // A long-running writer transaction pins the cluster-wide xmin, so rolled-back appends
+    // (each burns a sequence id) accumulate as gaps and every turn ships them all to the fetch.
+    const warnOnceOnLargeGapSet = (consumerName, instance, count) => {
+        const key = `${consumerName}:${instance}`;
+        // skip: below the warn threshold, or already reported once for this consumer instance
+        if (count <= PENDING_GAPS_WARN_THRESHOLD || reportedLargeGapSets.has(key))
+            return;
+        reportedLargeGapSets.add(key);
+        logDispatcherError(`[event-dispatcher] ${consumerName} tracks ${count} pending gap ranges (> ${PENDING_GAPS_WARN_THRESHOLD}); ` +
+            `a long-running write transaction is probably holding back the snapshot xmin`);
+    };
     // Per-(consumer, instanceId) backoff after a thrown pass (see the catch in
     // processConsumer). Keyed the same way as reportedDeadConsumers.
     // consecutiveFailures drives the exponential delay; retryAtMs is the
@@ -67,6 +98,9 @@ export function createEventDispatcher(options) {
     // unrelated outage gets its own fresh log line instead of staying
     // permanently suppressed by this process's flag.
     let idlePreCheckErrorLogged = false;
+    // Mirrors the last gauge value derived from the LISTEN connection (set by onlisten), so the
+    // pre-check recovery can restore it after a connection-loss error flipped it to 0.
+    let listenSubscribed = false;
     let running = false;
     // Separate from `running` on purpose: pre-registration of consumer state
     // rows is a one-time boot action, while running/timer/LISTEN is a
@@ -241,6 +275,7 @@ export function createEventDispatcher(options) {
             })));
             if (idlePreCheckErrorLogged) {
                 idlePreCheckErrorLogged = false;
+                emitEventDispatcherListenConnected(meter, listenSubscribed);
                 const recoveredMsg = "[event-dispatcher] idle pre-check recovered, database reachable again";
                 if (context.log) {
                     context.log.info(recoveredMsg);
@@ -258,9 +293,10 @@ export function createEventDispatcher(options) {
             // signal for ops, not a delivery failure.
             if (!idlePreCheckErrorLogged) {
                 idlePreCheckErrorLogged = true;
-                // The LISTEN peer talks to the same DB, so it is gone too; only a
-                // later onlisten may flip the gauge back to 1.
-                emitEventDispatcherListenConnected(meter, false);
+                // A lost DB socket takes the LISTEN peer down with it; other pre-check failures
+                // (statement_timeout, pool exhaustion, schema errors) say nothing about LISTEN.
+                if (isConnectionLossError(e))
+                    emitEventDispatcherListenConnected(meter, false);
                 const msg = e instanceof Error ? e.message : String(e);
                 const logMsg = `[event-dispatcher] idle pre-check failed, falling back to per-consumer locking every tick until it recovers: ${msg}`;
                 logDispatcherError(logMsg);
@@ -290,6 +326,7 @@ export function createEventDispatcher(options) {
                 "consumer.instance_id": instanceId,
             },
         });
+        let cursorBeforePass;
         try {
             await db.begin(async (tx) => {
                 const acquired = await acquireConsumerState(tx, consumer.name, instanceId, rearmCooldownMs, maxRearmCount);
@@ -313,6 +350,7 @@ export function createEventDispatcher(options) {
                 // staying permanently suppressed by this process's Set.
                 reportedDeadConsumers.delete(`${consumer.name}:${instanceId}`);
                 const oldCursor = acquired.state.lastProcessedEventId;
+                cursorBeforePass = oldCursor;
                 const pendingGaps = acquired.state.pendingGaps;
                 // xmin BEFORE the fetch: a range is provably burnt only once this has
                 // passed the xmax recorded with it. Only needed when ranges exist.
@@ -377,10 +415,11 @@ export function createEventDispatcher(options) {
                 }));
                 const persistedOutcome = {
                     ...outcome,
-                    pendingGaps: [...keptGaps, ...newGaps],
+                    pendingGaps: mergeContiguousGapRanges([...keptGaps, ...newGaps]),
                 };
+                warnOnceOnLargeGapSet(consumer.name, instanceId, persistedOutcome.pendingGaps.length);
                 await persistConsumerOutcome(tx, consumer.name, instanceId, persistedOutcome);
-                await emitLagFromTx(tx, consumer.name, instanceId, outcome.cursor, meter);
+                await emitLagFromTx(tx, consumer.name, instanceId, outcome.cursor, meter, persistedOutcome.pendingGaps.length);
             });
             // Pass completed without throwing — clear any backoff from a prior
             // failure so the next unrelated outage gets its own fresh delay
@@ -409,7 +448,7 @@ export function createEventDispatcher(options) {
             // so any failure here is swallowed after logging.
             try {
                 await db.begin(async (tx) => {
-                    await recordConsumerPassFailure(tx, consumer.name, instanceId, msg);
+                    await recordConsumerPassFailure(tx, consumer.name, instanceId, msg, cursorBeforePass);
                 });
             }
             catch (persistErr) {
@@ -472,6 +511,7 @@ export function createEventDispatcher(options) {
                         // ponytail: no own LISTEN reconnect, the backoff gap is polled;
                         // add a dedicated LISTEN connection if prod wires a pgClient
                         // and that latency matters.
+                        listenSubscribed = true;
                         emitEventDispatcherListenConnected(meter, true);
                     });
                     pgUnlisten = sub.unlisten;
@@ -498,6 +538,7 @@ export function createEventDispatcher(options) {
                     // subscription is being torn down anyway.
                 });
                 pgUnlisten = null;
+                listenSubscribed = false;
                 emitEventDispatcherListenConnected(meter, false);
             }
             // Drain any in-flight pass so shutdown observes consistent state —

package/dist/pipeline/index.d.ts

@@ -1,5 +1,7 @@
 export type { EscapeHatchReportWindow } from "../observability/escape-hatch-report.js";
 export { createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "../observability/escape-hatch-report.js";
+export type { AppendDomainEventCoreDeps } from "./append-event-core.js";
+export { appendDomainEventCore } from "./append-event-core.js";
 export type { ResolveAuthClaimsArgs } from "./auth-claims-resolver.js";
 export { resolveAuthClaims } from "./auth-claims-resolver.js";
 export { createCascadeDeleteHook } from "./cascade-handler.js";

package/dist/pipeline/index.js

@@ -1,4 +1,5 @@
 export { createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "../observability/escape-hatch-report.js";
+export { appendDomainEventCore } from "./append-event-core.js";
 export { resolveAuthClaims } from "./auth-claims-resolver.js";
 export { createCascadeDeleteHook } from "./cascade-handler.js";
 export { createDispatcher, dispatcherToWriteRef } from "./dispatcher.js";