@cosmicdrift/kumiko-framework 0.352.0 → 0.354.0

package/dist/pipeline/event-dispatcher-admin.js

@@ -3,7 +3,7 @@ import { selectNextEventIdAfter, selectSmallestVisibleIdInRanges, } from "../db/
 import { coerceRow, extractTableInfo, selectMany } from "../db/query.js";
 import { getEventsHighWaterMark } from "../event-store/index.js";
 import { eventConsumerStateTable, SHARED_INSTANCE_SENTINEL, } from "./event-consumer-state.js";
-import { rangeContainsId, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges.js";
+import { mergeContiguousGapRanges, rangeContainsId, splitRangeExcludingIds, toIdRanges, } from "./pending-gap-ranges.js";
 // --- Ops recovery surface ---
 //
 // These are intentionally verb-distinct; each maps to a CLI sub-command.
@@ -84,6 +84,19 @@ export async function enableConsumer(db, name, instanceId = SHARED_INSTANCE_SENT
     }
     return applyConsumerStatusTransition(db, name, instanceId, "idle");
 }
+// Delivery stops at the poison event, so ids between the cursor and it were
+// invisible during that pass (in-flight commits); keeping them as a pending gap
+// avoids losing them when the cursor jumps over.
+async function invisibleWindowGap(tx, cursor, poisonId) {
+    const windowStart = cursor + 1n;
+    if (poisonId <= windowStart)
+        return null;
+    return {
+        from: windowStart.toString(),
+        to: (poisonId - 1n).toString(),
+        xmax: await selectSnapshotXmax(tx),
+    };
+}
 // skipPoisonEvent skips the event whose handler failed last
 // (last_failed_event_id, set by the delivery's halt-on-poison branch): inside
 // pending_gaps it is split out of its range, otherwise the cursor moves onto
@@ -109,7 +122,7 @@ export async function skipPoisonEvent(db, name, instanceId = SHARED_INSTANCE_SEN
             const newPendingGaps = pendingGaps.flatMap((gap) => rangeContainsId(gap, pendingPoisonId)
                 ? splitRangeExcludingIds(gap, [pendingPoisonId])
                 : [gap]);
-            const raw = await removePendingGapReturning(tx, name, instanceId, newPendingGaps);
+            const raw = await removePendingGapReturning(tx, name, instanceId, mergeContiguousGapRanges(newPendingGaps));
             const updated = raw && coerceRow(raw, extractTableInfo(eventConsumerStateTable));
             if (!updated)
                 throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`);
@@ -130,15 +143,8 @@ export async function skipPoisonEvent(db, name, instanceId = SHARED_INSTANCE_SEN
         // Delivery stops at the poison event, so ids between the cursor and it were
         // invisible during that pass (in-flight commits). Jumping the cursor over them
         // would lose them; keep them as a pending gap like the dispatcher's own gaps.
-        const skippedWindowStart = before.lastProcessedEventId + 1n;
-        const gapToAdd = poisonId > skippedWindowStart
-            ? {
-                from: skippedWindowStart.toString(),
-                to: (poisonId - 1n).toString(),
-                xmax: await selectSnapshotXmax(tx),
-            }
-            : null;
-        const raw = await advanceConsumerPastEventReturning(tx, name, instanceId, poisonId, gapToAdd);
+        const gapToAdd = await invisibleWindowGap(tx, before.lastProcessedEventId, poisonId);
+        const raw = await advanceConsumerPastEventReturning(tx, name, instanceId, poisonId, mergeContiguousGapRanges(gapToAdd ? [...pendingGaps, gapToAdd] : pendingGaps));
         const updated = raw && coerceRow(raw, extractTableInfo(eventConsumerStateTable));
         if (!updated)
             throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`);

package/dist/pipeline/event-dispatcher-delivery.d.ts

@@ -68,4 +68,4 @@ export type PersistedConsumerOutcome = DeliveryOutcome & {
     readonly pendingGaps: readonly PendingGapEntry[];
 };
 export declare function persistConsumerOutcome(tx: DbTx, name: string, instanceId: string, outcome: PersistedConsumerOutcome): Promise<void>;
-export declare function emitLagFromTx(tx: DbTx, consumerName: string, instanceId: string, cursor: bigint, meter: Meter): Promise<void>;
+export declare function emitLagFromTx(tx: DbTx, consumerName: string, instanceId: string, cursor: bigint, meter: Meter, pendingGapRanges: number): Promise<void>;

package/dist/pipeline/event-dispatcher-delivery.js

@@ -6,7 +6,7 @@ import { selectEventsHeadId, selectPendingAndNewEventRows, } from "../db/queries
 import { coerceRow, extractTableInfo } from "../db/query.js";
 import { qnScope } from "../engine/qualified-name.js";
 import { eventsTable, toStoredEvent as rowToStoredEvent, } from "../event-store/index.js";
-import { emitDispatcherError, emitEventConsumerLag, getFallbackMeter, } from "../observability/index.js";
+import { emitDispatcherError, emitEventConsumerLag, emitEventConsumerPendingGaps, getFallbackMeter, } from "../observability/index.js";
 import { ConsumerStatuses, eventConsumerStateTable, SHARED_INSTANCE_SENTINEL, } from "./event-consumer-state.js";
 import { parseWriteOrigin } from "./write-origin.js";
 // Fails closed without throwing: a throw would poison the event for every consumer.
@@ -315,8 +315,9 @@ export async function persistConsumerOutcome(tx, name, instanceId, outcome) {
 // Emit the lag gauge inside the consumer pass's tx so ops sees a snapshot
 // consistent with the cursor we just advanced to. `MAX(id)` on the events
 // table is an O(1) reverse-index scan — cheap even under load.
-export async function emitLagFromTx(tx, consumerName, instanceId, cursor, meter) {
+export async function emitLagFromTx(tx, consumerName, instanceId, cursor, meter, pendingGapRanges) {
     const head = await selectEventsHeadId(tx);
     const lag = head > cursor ? Number(head - cursor) : 0;
     emitEventConsumerLag(meter, { consumer: consumerName, instanceId }, lag);
+    emitEventConsumerPendingGaps(meter, { consumer: consumerName, instanceId }, pendingGapRanges);
 }

package/dist/pipeline/event-dispatcher.js

@@ -1,15 +1,33 @@
+import { extractPgError } from "../db/pg-error.js";
 import { recordConsumerPassFailure, selectSnapshotXmax, selectSnapshotXmin, } from "../db/queries/event-consumer.js";
 import { SYSTEM_TENANT_ID } from "../engine/types/identifiers.js";
 import { EVENTS_PUBSUB_CHANNEL } from "../event-store/index.js";
 import { emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, getFallbackMeter, getFallbackTracer, } from "../observability/index.js";
 import { SHARED_INSTANCE_SENTINEL } from "./event-consumer-state.js";
 import { acquireConsumerState, consumerInstanceId, deliverEvents, emitLagFromTx, fetchPendingEvents, markProcessing, persistConsumerOutcome, preRegisterConsumers, selectIdleConsumerKeys, } from "./event-dispatcher-delivery.js";
-import { partitionBurntGaps, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges.js";
+import { mergeContiguousGapRanges, partitionBurntGaps, splitRangeExcludingIds, toIdRanges, } from "./pending-gap-ranges.js";
 const DEFAULT_BATCH_SIZE = 200;
 const DEFAULT_POLL_MS = 100;
 const DEFAULT_MAX_ATTEMPTS = 10;
 const DEFAULT_REARM_COOLDOWN_MS = 5 * 60_000;
 const DEFAULT_MAX_REARM_COUNT = 3;
+const PENDING_GAPS_WARN_THRESHOLD = 1000;
+// Only these say the database socket itself is gone; timeouts, schema or pool errors do not
+// imply the dedicated LISTEN connection dropped.
+const CONNECTION_LOSS_CODES = new Set([
+    "CONNECTION_CLOSED",
+    "CONNECTION_DESTROYED",
+    "CONNECTION_ENDED",
+    "CONNECT_TIMEOUT",
+    "ECONNREFUSED",
+    "ECONNRESET",
+    "ERR_POSTGRES_CONNECTION_CLOSED",
+    "57P01",
+]);
+function isConnectionLossError(e) {
+    const code = extractPgError(e)?.code;
+    return code !== undefined && CONNECTION_LOSS_CODES.has(code);
+}
 // Backoff for a consumer whose pass THREW (the processConsumer catch below)
 // — an infra-level failure (db.begin's callback rejected outside deliverEvents'
 // own per-event try/catch), not a poisoned event. Doubles per consecutive
@@ -52,6 +70,19 @@ export function createEventDispatcher(options) {
     // which is fine: a fresh process re-observing a still-dead consumer is
     // exactly the "still needs a human" signal ops wants.
     const reportedDeadConsumers = new Set();
+    // Same once-per-process dedup for the pending_gaps size warning.
+    const reportedLargeGapSets = new Set();
+    // A long-running writer transaction pins the cluster-wide xmin, so rolled-back appends
+    // (each burns a sequence id) accumulate as gaps and every turn ships them all to the fetch.
+    const warnOnceOnLargeGapSet = (consumerName, instance, count) => {
+        const key = `${consumerName}:${instance}`;
+        // skip: below the warn threshold, or already reported once for this consumer instance
+        if (count <= PENDING_GAPS_WARN_THRESHOLD || reportedLargeGapSets.has(key))
+            return;
+        reportedLargeGapSets.add(key);
+        logDispatcherError(`[event-dispatcher] ${consumerName} tracks ${count} pending gap ranges (> ${PENDING_GAPS_WARN_THRESHOLD}); ` +
+            `a long-running write transaction is probably holding back the snapshot xmin`);
+    };
     // Per-(consumer, instanceId) backoff after a thrown pass (see the catch in
     // processConsumer). Keyed the same way as reportedDeadConsumers.
     // consecutiveFailures drives the exponential delay; retryAtMs is the
@@ -67,6 +98,9 @@ export function createEventDispatcher(options) {
     // unrelated outage gets its own fresh log line instead of staying
     // permanently suppressed by this process's flag.
     let idlePreCheckErrorLogged = false;
+    // Mirrors the last gauge value derived from the LISTEN connection (set by onlisten), so the
+    // pre-check recovery can restore it after a connection-loss error flipped it to 0.
+    let listenSubscribed = false;
     let running = false;
     // Separate from `running` on purpose: pre-registration of consumer state
     // rows is a one-time boot action, while running/timer/LISTEN is a
@@ -241,6 +275,7 @@ export function createEventDispatcher(options) {
             })));
             if (idlePreCheckErrorLogged) {
                 idlePreCheckErrorLogged = false;
+                emitEventDispatcherListenConnected(meter, listenSubscribed);
                 const recoveredMsg = "[event-dispatcher] idle pre-check recovered, database reachable again";
                 if (context.log) {
                     context.log.info(recoveredMsg);
@@ -258,9 +293,10 @@ export function createEventDispatcher(options) {
             // signal for ops, not a delivery failure.
             if (!idlePreCheckErrorLogged) {
                 idlePreCheckErrorLogged = true;
-                // The LISTEN peer talks to the same DB, so it is gone too; only a
-                // later onlisten may flip the gauge back to 1.
-                emitEventDispatcherListenConnected(meter, false);
+                // A lost DB socket takes the LISTEN peer down with it; other pre-check failures
+                // (statement_timeout, pool exhaustion, schema errors) say nothing about LISTEN.
+                if (isConnectionLossError(e))
+                    emitEventDispatcherListenConnected(meter, false);
                 const msg = e instanceof Error ? e.message : String(e);
                 const logMsg = `[event-dispatcher] idle pre-check failed, falling back to per-consumer locking every tick until it recovers: ${msg}`;
                 logDispatcherError(logMsg);
@@ -290,6 +326,7 @@ export function createEventDispatcher(options) {
                 "consumer.instance_id": instanceId,
             },
         });
+        let cursorBeforePass;
         try {
             await db.begin(async (tx) => {
                 const acquired = await acquireConsumerState(tx, consumer.name, instanceId, rearmCooldownMs, maxRearmCount);
@@ -313,6 +350,7 @@ export function createEventDispatcher(options) {
                 // staying permanently suppressed by this process's Set.
                 reportedDeadConsumers.delete(`${consumer.name}:${instanceId}`);
                 const oldCursor = acquired.state.lastProcessedEventId;
+                cursorBeforePass = oldCursor;
                 const pendingGaps = acquired.state.pendingGaps;
                 // xmin BEFORE the fetch: a range is provably burnt only once this has
                 // passed the xmax recorded with it. Only needed when ranges exist.
@@ -377,10 +415,11 @@ export function createEventDispatcher(options) {
                 }));
                 const persistedOutcome = {
                     ...outcome,
-                    pendingGaps: [...keptGaps, ...newGaps],
+                    pendingGaps: mergeContiguousGapRanges([...keptGaps, ...newGaps]),
                 };
+                warnOnceOnLargeGapSet(consumer.name, instanceId, persistedOutcome.pendingGaps.length);
                 await persistConsumerOutcome(tx, consumer.name, instanceId, persistedOutcome);
-                await emitLagFromTx(tx, consumer.name, instanceId, outcome.cursor, meter);
+                await emitLagFromTx(tx, consumer.name, instanceId, outcome.cursor, meter, persistedOutcome.pendingGaps.length);
             });
             // Pass completed without throwing — clear any backoff from a prior
             // failure so the next unrelated outage gets its own fresh delay
@@ -409,7 +448,7 @@ export function createEventDispatcher(options) {
             // so any failure here is swallowed after logging.
             try {
                 await db.begin(async (tx) => {
-                    await recordConsumerPassFailure(tx, consumer.name, instanceId, msg);
+                    await recordConsumerPassFailure(tx, consumer.name, instanceId, msg, cursorBeforePass);
                 });
             }
             catch (persistErr) {
@@ -472,6 +511,7 @@ export function createEventDispatcher(options) {
                         // ponytail: no own LISTEN reconnect, the backoff gap is polled;
                         // add a dedicated LISTEN connection if prod wires a pgClient
                         // and that latency matters.
+                        listenSubscribed = true;
                         emitEventDispatcherListenConnected(meter, true);
                     });
                     pgUnlisten = sub.unlisten;
@@ -498,6 +538,7 @@ export function createEventDispatcher(options) {
                     // subscription is being torn down anyway.
                 });
                 pgUnlisten = null;
+                listenSubscribed = false;
                 emitEventDispatcherListenConnected(meter, false);
             }
             // Drain any in-flight pass so shutdown observes consistent state —

package/dist/pipeline/index.d.ts

@@ -1,5 +1,7 @@
 export type { EscapeHatchReportWindow } from "../observability/escape-hatch-report.js";
 export { createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "../observability/escape-hatch-report.js";
+export type { AppendDomainEventCoreDeps } from "./append-event-core.js";
+export { appendDomainEventCore } from "./append-event-core.js";
 export type { ResolveAuthClaimsArgs } from "./auth-claims-resolver.js";
 export { resolveAuthClaims } from "./auth-claims-resolver.js";
 export { createCascadeDeleteHook } from "./cascade-handler.js";

package/dist/pipeline/index.js

@@ -1,4 +1,5 @@
 export { createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "../observability/escape-hatch-report.js";
+export { appendDomainEventCore } from "./append-event-core.js";
 export { resolveAuthClaims } from "./auth-claims-resolver.js";
 export { createCascadeDeleteHook } from "./cascade-handler.js";
 export { createDispatcher, dispatcherToWriteRef } from "./dispatcher.js";

package/dist/pipeline/member-read-only-transaction.js

@@ -1,8 +1,9 @@
 import { extractPgError } from "../db/pg-error.js";
 import { asRawClient, runInSavepoint, transaction } from "../db/query.js";
-import { AccessDeniedError, InternalError, memberResolutionReadOnlyDenied, } from "../errors/index.js";
+import { AccessDeniedError, InternalError, MemberReadTimeoutError, memberResolutionReadOnlyDenied, } from "../errors/index.js";
 import { resolveDbSource } from "./dispatch-shared.js";
 const PG_READ_ONLY_SQLSTATE = "25006";
+const PG_QUERY_CANCELED_SQLSTATE = "57014";
 function isReadOnlyTransactionViolation(e) {
     return extractPgError(e)?.code === PG_READ_ONLY_SQLSTATE;
 }
@@ -12,12 +13,27 @@ class SavepointDiscard extends Error {
         this.name = "SavepointDiscard";
     }
 }
-async function runInDiscardedReadOnlySavepoint(tx, fn) {
+function isStatementTimeout(e) {
+    return extractPgError(e)?.code === PG_QUERY_CANCELED_SQLSTATE;
+}
+// SET cannot take bind parameters, so the value is interpolated — only after it is proven a plain integer.
+function memberReadSessionSettingsSql(timeoutMs, bindIdleTimeout) {
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) {
+        throw new InternalError({
+            message: `member read timeout must be a positive integer, got ${timeoutMs}`,
+        });
+    }
+    const idleTimeout = bindIdleTimeout
+        ? `; SET LOCAL idle_in_transaction_session_timeout = ${timeoutMs}`
+        : "";
+    return `SET TRANSACTION READ ONLY; SET LOCAL statement_timeout = ${timeoutMs}${idleTimeout}`;
+}
+async function runInDiscardedReadOnlySavepoint(tx, timeoutMs, bindIdleTimeout, fn) {
     const settled = {};
     try {
         await runInSavepoint(tx, async (sp) => {
             // kumiko-lint-ignore raw-sql transaction characteristic, no query helper
-            await asRawClient(sp).unsafe("SET TRANSACTION READ ONLY");
+            await asRawClient(sp).unsafe(memberReadSessionSettingsSql(timeoutMs, bindIdleTimeout));
             // @cast-boundary driver savepoint handle — structurally a DbTx, same shape runInSavepoint's caller relies on
             settled.outcome = { value: await fn(sp) };
             // Always discarded: RELEASE SAVEPOINT would leave the outer tx read-only too.
@@ -36,9 +52,10 @@ async function runInDiscardedReadOnlySavepoint(tx, fn) {
     return settled.outcome.value;
 }
 export async function runInMemberReadOnlyTransaction(ctx, tx, fn) {
+    const timeoutMs = ctx.memberReadTimeoutMs;
     try {
         if (tx)
-            return await runInDiscardedReadOnlySavepoint(tx, fn);
+            return await runInDiscardedReadOnlySavepoint(tx, timeoutMs, false, fn);
         const pool = resolveDbSource(ctx, undefined);
         if (!pool) {
             throw new InternalError({
@@ -48,13 +65,16 @@ export async function runInMemberReadOnlyTransaction(ctx, tx, fn) {
         // Savepoint even on the pool path: Postgres rejects a READ WRITE reset (SQLSTATE 25001) inside a subtransaction.
         return await transaction(pool, (outer) => 
         // @cast-boundary driver begin() handle — structurally a DbTx, same shape runInSavepoint's caller relies on
-        runInDiscardedReadOnlySavepoint(outer, fn));
+        runInDiscardedReadOnlySavepoint(outer, timeoutMs, true, fn));
     }
     catch (e) {
         if (e instanceof AccessDeniedError)
             throw e;
         if (isReadOnlyTransactionViolation(e))
             throw memberResolutionReadOnlyDenied(e);
+        if (isStatementTimeout(e)) {
+            throw new MemberReadTimeoutError({ timeoutMs, ...(e instanceof Error && { cause: e }) });
+        }
         throw e;
     }
 }

package/dist/pipeline/member-reader.js

@@ -67,6 +67,7 @@ export function createMemberReaderFn(ctx, tenantId, tx) {
     }
     // Only successful resolutions are cached — a rejected/failed resolution
     // deletes its own entry so a transient error can't poison the rest of the run.
+    // No TTL: a membership revoked mid-run is only noticed by the next reader.
     const cache = new Map();
     function resolve(userId) {
         const cached = cache.get(userId);

package/dist/pipeline/msp-rebuild.js

@@ -8,7 +8,7 @@ import { createRebuildDeadLetterTable, recordRebuildDeadLetters, } from "../even
 import { upcastStoredEvents } from "../event-store/upcaster.js";
 import { emitProjectionRebuild } from "../observability/standard-metrics.js";
 import { SHARED_INSTANCE_SENTINEL } from "./event-consumer-state.js";
-import { capPendingGapsBelowCursor, subtractSortedIdsFromRanges } from "./pending-gap-ranges.js";
+import { capPendingGapsBelowCursor, mergeContiguousGapRanges, subtractSortedIdsFromRanges, } from "./pending-gap-ranges.js";
 function createRebuildCtx(registry, db, tenantId) {
     // Both surfaces throw — rebuild MUST NOT emit. We share one impl.
     const refuseAppend = async (args) => {
@@ -128,9 +128,9 @@ export async function rebuildMultiStreamProjection(mspName, deps) {
             const survivingGaps = subtractSortedIdsFromRanges(seededGaps, replayedIds);
             const cappedGaps = capPendingGapsBelowCursor(survivingGaps, lastProcessedEventId);
             const postReplayXmax = cappedGaps.length > 0 ? await selectSnapshotXmax(tx) : null;
-            const pendingGaps = postReplayXmax === null
+            const pendingGaps = mergeContiguousGapRanges(postReplayXmax === null
                 ? cappedGaps
-                : cappedGaps.map((gap) => ({ ...gap, xmax: postReplayXmax }));
+                : cappedGaps.map((gap) => ({ ...gap, xmax: postReplayXmax })));
             await updateConsumerRebuildCursor(tx, mspName, SHARED_INSTANCE_SENTINEL, lastProcessedEventId, pendingGaps);
             await swapShadowIntoLive(tx, meta.tableName);
         });

package/dist/pipeline/pending-gap-ranges.d.ts

@@ -10,3 +10,4 @@ export declare function partitionBurntGaps(gaps: readonly PendingGapEntry[], fet
 export declare function splitRangeExcludingIds(range: PendingGapEntry, excludeIds: readonly bigint[]): PendingGapEntry[];
 export declare function subtractSortedIdsFromRanges(ranges: readonly PendingGapEntry[], sortedIds: readonly bigint[]): PendingGapEntry[];
 export declare function capPendingGapsBelowCursor(gaps: readonly PendingGapEntry[], cursor: bigint): PendingGapEntry[];
+export declare function mergeContiguousGapRanges(gaps: readonly PendingGapEntry[]): PendingGapEntry[];

package/dist/pipeline/pending-gap-ranges.js

@@ -97,3 +97,25 @@ export function capPendingGapsBelowCursor(gaps, cursor) {
     }
     return capped;
 }
+// Merges ranges that touch exactly (end + 1 === next start). Keeps the row small
+// when many adjacent id holes appear one pass after another. The merged xmax is
+// the larger one: finality may only be claimed once the later-recorded bound has
+// passed, so merging can delay it but never make it premature.
+export function mergeContiguousGapRanges(gaps) {
+    const sorted = [...gaps].sort((a, b) => {
+        const diff = BigInt(a.from) - BigInt(b.from);
+        return diff < 0n ? -1 : diff > 0n ? 1 : 0;
+    });
+    const merged = [];
+    for (const gap of sorted) {
+        const last = merged.at(-1);
+        if (last && BigInt(last.to) + 1n === BigInt(gap.from)) {
+            const xmax = BigInt(gap.xmax) > BigInt(last.xmax) ? gap.xmax : last.xmax;
+            merged[merged.length - 1] = { from: last.from, to: gap.to, xmax };
+        }
+        else {
+            merged.push(gap);
+        }
+    }
+    return merged;
+}

package/dist/pipeline/system-hooks.js

@@ -400,7 +400,7 @@ export function createSseBroadcastEventConsumer(sseBroker) {
         // exist yet has no audience to replay it to.
         startFrom: "now",
         handler: async (event) => {
-            sseBroker.pushToChannel(tenantChannel(event.tenantId), {
+            await sseBroker.pushToChannel(tenantChannel(event.tenantId), {
                 type: event.type,
                 data: {
                     id: event.aggregateId,
@@ -571,7 +571,7 @@ export function createAccessInvalidationEventConsumer(sseBroker) {
                 // every user behind one bad row).
                 if (typeof userId !== "string" || userId.length === 0)
                     return;
-                sseBroker.publishAccessInvalidation(userId, readSessionRevokedScope(event.payload));
+                await sseBroker.publishAccessInvalidation(userId, readSessionRevokedScope(event.payload));
             }
             if (event.type === PAT_REVOKED_EVENT_TYPE) {
                 const userId = event.payload["userId"];
@@ -579,7 +579,7 @@ export function createAccessInvalidationEventConsumer(sseBroker) {
                 // session-revoked above.
                 if (typeof userId !== "string" || userId.length === 0)
                     return;
-                sseBroker.publishAccessInvalidation(userId, readPatRevokedScope(event.payload));
+                await sseBroker.publishAccessInvalidation(userId, readPatRevokedScope(event.payload));
             }
             if (event.type === TENANT_MEMBERSHIP_UPDATED_EVENT_TYPE ||
                 event.type === TENANT_MEMBERSHIP_DELETED_EVENT_TYPE) {
@@ -588,7 +588,7 @@ export function createAccessInvalidationEventConsumer(sseBroker) {
                 // reasoning as above.
                 if (userId === undefined)
                     return;
-                sseBroker.publishAccessInvalidation(userId);
+                await sseBroker.publishAccessInvalidation(userId);
             }
         },
     };

package/dist/pipeline/system-identity-switch.d.ts

@@ -24,7 +24,7 @@ type MemberReaderAudit = IdentitySwitchAudit & {
 };
 export declare function createGatedIdentitySwitch(callerLabel: string, caller: SessionUser | undefined, hasGrant: boolean, ungated: IdentitySwitch, audit?: IdentitySwitchAudit): IdentitySwitch;
 export declare function createGatedMemberReader(callerLabel: string, allowSystemIdentity: boolean, ungated: MemberReader, audit?: MemberReaderAudit): MemberReader;
-export declare function createGatedProjectionReader(callerLabel: string, hasGrant: boolean, ungated: ProjectionReader, audit?: IdentitySwitchAudit): ProjectionReader;
+export declare function createGatedProjectionReader(callerLabel: string, hasGrant: boolean, ungated: ProjectionReader, audit?: IdentitySwitchAudit, projectionHasTenantColumn?: (qualifiedName: string) => boolean): ProjectionReader;
 export declare function withHookEscapeHatchGrant<TContext extends object>(context: TContext, callerLabel: string, escapeHatch: EscapeHatchDeclaration | undefined): TContext;
 export declare function bindHookEscapeHatchGrant(fn: LifecycleHookFn, label: string, escapeHatch: EscapeHatchDeclaration | undefined): LifecycleHookFn;
 export {};

package/dist/pipeline/system-identity-switch.js

@@ -104,11 +104,12 @@ export function createGatedMemberReader(callerLabel, allowSystemIdentity, ungate
 }
 // Same idea, for ctx.queryProjection — a single function rather than a pair.
 const ungatedProjectionReaderByGated = new WeakMap();
-// ctx.queryProjection's gate: only { unsafeAllTenants: true } is grant-checked —
-// a plain call always passes through, tenant-scoped by the projection's own table.
-export function createGatedProjectionReader(callerLabel, hasGrant, ungated, audit) {
+// ctx.queryProjection's gate: only { unsafeAllTenants: true } on a projection with a
+// tenant column is grant-checked — a plain call always passes through, tenant-scoped
+// by the projection's own table, and the flag is a no-op on a tenant-less one.
+export function createGatedProjectionReader(callerLabel, hasGrant, ungated, audit, projectionHasTenantColumn = () => true) {
     const gated = async (qualifiedName, options) => {
-        if (options?.unsafeAllTenants === true) {
+        if (options?.unsafeAllTenants === true && projectionHasTenantColumn(qualifiedName)) {
             if (!hasGrant)
                 throw unsafeAllTenantsDenied(callerLabel);
             if (audit?.reason !== undefined) {

package/dist/pipeline/tenant-timezone-cache.js

@@ -4,9 +4,11 @@
 // SELECT (sometimes two, cascade + fallback) per request, often inside an
 // open write transaction. A tenant-keyed TTL cache removes the steady-state
 // cost; dispatch-write.ts invalidates entries synchronously on
-// config:write:set/reset for this key, so the TTL below only covers writes
-// that bypass that path (migrations, seeds, direct DB edits).
-const DEFAULT_TTL_MS = 5 * 60 * 1000;
+// config:write:set/reset for this key, but only in the process that handled
+// the write. The TTL below therefore bounds how long OTHER replicas (and
+// writes that bypass that path: migrations, seeds, direct DB edits) keep
+// serving the old value, hence the short default.
+const DEFAULT_TTL_MS = 30 * 1000;
 const DEFAULT_MAX_ENTRIES = 1000;
 export function createTenantTimezoneCache(opts = {}) {
     const ttlMs = opts.ttlMs ?? DEFAULT_TTL_MS;

package/dist/redis/cache-sync-topics.d.ts

@@ -2,6 +2,7 @@ import type { TenantId } from "../engine/types/identifiers.js";
 export declare const CACHE_SYNC_TOPICS: {
     readonly tenantConfig: "tenant-config";
     readonly tierAssignment: "tier-assignment";
+    readonly featureToggle: "feature-toggle";
 };
 export type TenantConfigSyncMessage = {
     readonly tenantId: TenantId;

package/dist/redis/cache-sync-topics.js

@@ -1,6 +1,7 @@
 export const CACHE_SYNC_TOPICS = {
     tenantConfig: "tenant-config",
     tierAssignment: "tier-assignment",
+    featureToggle: "feature-toggle",
 };
 function hasOptionalStringKey(value) {
     return !("key" in value) || value.key === undefined || typeof value.key === "string";

package/dist/redis/client.d.ts

@@ -6,4 +6,5 @@ export type RedisClientOptions = {
     readonly extra?: Readonly<Omit<RedisOptions, "connectTimeout" | "commandTimeout" | "maxRetriesPerRequest">>;
 };
 export declare function createRedisClient(url: string, options?: RedisClientOptions): Redis;
+export declare function redisChannelPrefixFromEnv(env?: Readonly<Record<string, string | undefined>>): string;
 export declare function redisClientOptionsFromEnv(env?: Readonly<Record<string, string | undefined>>): RedisClientOptions;

package/dist/redis/client.js

@@ -26,6 +26,13 @@ export function createRedisClient(url, options = {}) {
         maxRetriesPerRequest,
     });
 }
+// Redis Pub/Sub ignores the DB index and ioredis keyPrefix, so apps or environments sharing one
+// Redis instance would see each other's SSE and toggle-sync traffic. An optional prefix
+// namespaces those channels; unset keeps the unprefixed channel names.
+export function redisChannelPrefixFromEnv(env = process.env) {
+    const raw = env["KUMIKO_REDIS_CHANNEL_PREFIX"]?.trim();
+    return raw ? `${raw}:` : "";
+}
 // Env-var reader — mirrors dbConnectionOptionsFromEnv's contract. Strict:
 // malformed values throw at boot rather than silently falling back to
 // defaults.

package/dist/redis/index.d.ts

@@ -3,6 +3,6 @@ export { createCacheSyncBusOverSignal, createDefaultCacheSyncBus, createLocalCac
 export type { TenantConfigSyncMessage, TierAssignmentSyncMessage } from "./cache-sync-topics.js";
 export { CACHE_SYNC_TOPICS, isTenantConfigSyncMessage, isTierAssignmentSyncMessage, } from "./cache-sync-topics.js";
 export type { RedisClientOptions } from "./client.js";
-export { createRedisClient, redisClientOptionsFromEnv } from "./client.js";
+export { createRedisClient, redisChannelPrefixFromEnv, redisClientOptionsFromEnv, } from "./client.js";
 export type { PubSubSignal, PubSubSignalOptions } from "./pubsub-signal.js";
 export { createRedisPubSubSignal } from "./pubsub-signal.js";

package/dist/redis/index.js

@@ -1,4 +1,4 @@
 export { createCacheSyncBusOverSignal, createDefaultCacheSyncBus, createLocalCacheSyncBus, createRedisCacheSyncBus, resolveCacheSyncBus, } from "./cache-sync-bus.js";
 export { CACHE_SYNC_TOPICS, isTenantConfigSyncMessage, isTierAssignmentSyncMessage, } from "./cache-sync-topics.js";
-export { createRedisClient, redisClientOptionsFromEnv } from "./client.js";
+export { createRedisClient, redisChannelPrefixFromEnv, redisClientOptionsFromEnv, } from "./client.js";
 export { createRedisPubSubSignal } from "./pubsub-signal.js";

package/dist/redis/pubsub-signal.d.ts

@@ -1,6 +1,7 @@
 import { type RedisClientOptions } from "./client.js";
 export type PubSubSignal = {
     publish(channel: string, payload: unknown): void;
+    publishConfirmed(channel: string, payload: unknown): Promise<void>;
     onMessage(listener: (channel: string, payload: unknown) => void): void;
     onReconnect(listener: () => void): void;
     close(): Promise<void>;

package/dist/redis/pubsub-signal.js

@@ -92,6 +92,9 @@ export function createRedisPubSubSignal(opts) {
                 console.error(`[kumiko:${opts.label}] publish failed:`, err);
             });
         },
+        async publishConfirmed(channel, payload) {
+            await publisher.publish(channel, JSON.stringify(payload));
+        },
         onMessage(listener) {
             listeners.add(listener);
         },

package/dist/search/meilisearch-adapter.js

@@ -20,6 +20,14 @@ async function awaitSucceededTask(enqueued, toleratedErrorCodes = []) {
 // would fail forever.
 const REMOVE_TOLERATED_ERROR_CODES = [ErrorStatusCode.INDEX_NOT_FOUND];
 const INDEX_LIST_PAGE_SIZE = 100;
+// What an unconfigured Meilisearch index reports as searchableAttributes.
+const MEILI_ALL_ATTRIBUTES = "*";
+function isIndexNotFound(err) {
+    const cause = typeof err === "object" && err !== null ? err.cause : null;
+    return (typeof cause === "object" &&
+        cause !== null &&
+        cause.code === ErrorStatusCode.INDEX_NOT_FOUND);
+}
 // Exported for unit tests (index-name / primary-key shape) without a live Meili.
 export function meilisearchTenantIndex(prefix, tenantId) {
     return `${prefix}t${tenantId}`;
@@ -58,6 +66,24 @@ export function createMeilisearchAdapter(options) {
     // found". A failed configure is never memoized — remove the map entry (iff
     // it's still the same in-flight promise) and rethrow, so a transient
     // Meili outage doesn't permanently poison the tenant for the process.
+    // The configured-tenants map is process memory only: after a restart a stored explicit
+    // configure() must not be overwritten by the registry default, so the default is applied only
+    // to an index that is missing or still carries Meilisearch's factory settings.
+    async function indexNeedsDefaultConfig(tenantId) {
+        try {
+            const settings = await client.index(meilisearchTenantIndex(prefix, tenantId)).getSettings();
+            const searchable = settings.searchableAttributes ?? [MEILI_ALL_ATTRIBUTES];
+            const filterable = settings.filterableAttributes ?? [];
+            return (searchable.length === 1 &&
+                searchable[0] === MEILI_ALL_ATTRIBUTES &&
+                !filterable.includes("_type"));
+        }
+        catch (err) {
+            if (isIndexNotFound(err))
+                return true;
+            throw err;
+        }
+    }
     async function ensureConfigured(tenantId) {
         // skip: no registry-derived default — callers configure tenants explicitly.
         if (!defaultConfig)
@@ -68,10 +94,14 @@ export function createMeilisearchAdapter(options) {
             // skip: configured (or in flight) already for this tenant.
             return;
         }
-        await trackConfigure(tenantId, defaultConfig);
+        const config = defaultConfig;
+        await trackConfigure(tenantId, async () => {
+            if (await indexNeedsDefaultConfig(tenantId))
+                await applyConfig(tenantId, config);
+        });
     }
-    async function trackConfigure(tenantId, config) {
-        const pending = applyConfig(tenantId, config);
+    async function trackConfigure(tenantId, run) {
+        const pending = run();
         configuredTenants.set(tenantId, pending);
         try {
             await pending;
@@ -88,7 +118,7 @@ export function createMeilisearchAdapter(options) {
             defaultConfig = config;
         },
         async configure(tenantId, config) {
-            await trackConfigure(tenantId, config);
+            await trackConfigure(tenantId, () => applyConfig(tenantId, config));
         },
         async index(tenantId, doc) {
             await ensureConfigured(tenantId);

package/dist/stack/test-stack.d.ts

@@ -146,7 +146,7 @@ export type TestStackOptions = {
         runLogger?: (deps: {
             registry: Registry;
             db: import("../db/index.js").DbConnection;
-        }) => Pick<JobRunnerOptions, "onJobStart" | "onJobComplete" | "onJobFailed"> | undefined;
+        }) => Pick<JobRunnerOptions, "onJobStart" | "onJobComplete" | "onJobFailed" | "onJobQueued" | "onJobDropped"> | undefined;
     };
     /** Override the event dispatcher's polling-timer interval. Default 50ms.
      *  Tests that assert LISTEN/NOTIFY wake-up latency need this pushed far
@@ -157,6 +157,9 @@ export type TestStackOptions = {
      *  them themselves. Default true — false pushes only projection/MSP/
      *  storeTable sources, same as before fw#3102. */
     entityTables?: boolean;
+    /** Time source of the stack's rate-limit resolver, so tests can freeze the
+     *  clock instead of racing the window. Default: system time. */
+    rateLimitNowMs?: () => number;
     /** Forwarded to buildServer's top-level `ServerOptions.trustedProxyHops`
      *  — see there. Default 0. */
     trustedProxyHops?: number;