@cosmicdrift/kumiko-framework 0.364.0 → 0.365.0

package/dist/jobs/job-runner.js

@@ -12,7 +12,7 @@ import { InternalError } from "../errors/index.js";
 import { isKumikoError } from "../errors/kumiko-error.js";
 import { createFileContext } from "../files/file-handle.js";
 import { createFallbackLogger } from "../logging/index.js";
-import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
+import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
 import { RedisKeys } from "../pipeline/redis-keys.js";
 import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js";
@@ -82,6 +82,18 @@ export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
 function schedulerSeenKey(queue) {
     return queue.toKey("kumiko-scheduler-seen");
 }
+// "Seen" has to mean a live runner holds the scheduler, so runners re-stamp
+// their own ids periodically; a single stamp at start() would let a runner
+// restarting during a mixed-version window longer than the grace period delete
+// a scheduler a healthy newer pod still owns.
+async function markSchedulersSeen(queue, schedulerIds) {
+    // skip: no cron scheduler on this lane, nothing to stamp
+    if (schedulerIds.size === 0)
+        return;
+    const stamp = String(Date.now());
+    const entries = Object.fromEntries([...schedulerIds].map((id) => [id, stamp]));
+    await (await queue.client).hset(schedulerSeenKey(queue), entries);
+}
 // A cron job dropped from the app (or an older scheduler-id format) leaves its
 // scheduler in Redis, where it keeps firing jobs the worker no longer knows
 // ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
@@ -364,6 +376,7 @@ export function createJobRunner(options) {
     const { registry, context, redisUrl, consumerLane } = options;
     const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;
     const bootRedisTimeoutMs = options.bootRedisTimeoutMs ?? BOOT_REDIS_TIMEOUT_MS;
+    const schedulerSeenRefreshMs = options.schedulerSeenRefreshMs ?? ORPHAN_SCHEDULER_GRACE_MS / 4;
     const redisOpts = parseRedisOpts(redisUrl);
     // Use the context's tracer when present (observability-provider injected at
     // boot); otherwise noop so dispatch/handleJob stay zero-cost without config.
@@ -524,6 +537,11 @@ export function createJobRunner(options) {
     }
     let worker = null;
     let queueDepthTimer = null;
+    let schedulerSeenTimer = null;
+    let schedulerStampPending = false;
+    // A timed-out gate is abandoned, not aborted (the job context carries no
+    // AbortSignal); stop() waits for it so it cannot write after the shutdown.
+    const abandonedBootGates = new Set();
     // Forward reference to the runner's own API, exposed on the job-handler ctx
     // so a handler can dispatch a follow-up job (job→job chaining, e.g.
     // delivery.render → delivery.send). Assigned just before return; reads happen
@@ -943,6 +961,7 @@ export function createJobRunner(options) {
             attempt,
             finalAttempt,
             log: createJobLogger(logs),
+            metricsFor: createMetricsForFactory(context.meter),
             ...(triggerName !== undefined && { triggerName }),
             ...(triggerEvent !== undefined && { triggerEvent }),
             write: (qn, payload) => {
@@ -1078,17 +1097,18 @@ export function createJobRunner(options) {
                     });
                 }, BOOT_GATE_SLOW_WARN_MS)
                 : undefined;
+            const gateRun = handleJob({
+                id: bootJobIdForJobName(name),
+                name,
+                data: {},
+                attemptsMade: 0,
+                retryable: false,
+            });
+            const gateSettled = gateRun.then(() => undefined, () => undefined);
+            abandonedBootGates.add(gateSettled);
+            void gateSettled.then(() => abandonedBootGates.delete(gateSettled));
             try {
-                await Promise.race([
-                    handleJob({
-                        id: bootJobIdForJobName(name),
-                        name,
-                        data: {},
-                        attemptsMade: 0,
-                        retryable: false,
-                    }),
-                    ...(gateTimeout ? [gateTimeout.promise] : []),
-                ]);
+                await Promise.race([gateRun, ...(gateTimeout ? [gateTimeout.promise] : [])]);
             }
             catch (err) {
                 // Leaving the worker's Redis connections open would keep the event loop alive and turn the aborted boot into
@@ -1104,6 +1124,74 @@ export function createJobRunner(options) {
             }
         }
     }
+    function cronSchedulerIdsForLane(lane) {
+        const ids = new Set();
+        for (const [name, jobDef] of allJobs) {
+            if (laneForJob(jobDef) === lane && "cron" in jobDef.trigger) {
+                ids.add(schedulerIdForJobName(name));
+            }
+        }
+        return ids;
+    }
+    async function upsertCronSchedulers(queue, lane) {
+        for (const [name, jobDef] of allJobs) {
+            if (laneForJob(jobDef) !== lane)
+                continue;
+            if ("cron" in jobDef.trigger) {
+                await queue.upsertJobScheduler(schedulerIdForJobName(name), cronRepeatOptions(jobDef.trigger), {
+                    name: jobDef.perTenant ? `_perTenant:${name}` : name,
+                    data: {},
+                    // Queue-level defaultJobOptions (jobRetentionOpts) covers
+                    // retention; a count here would sweep queue-wide again and
+                    // evict perTenant children and boot jobs.
+                    opts: {
+                        ...buildRetryBullOpts(jobDef),
+                    },
+                });
+            }
+        }
+    }
+    // After a failed initial stamp an older runner may have removed our
+    // schedulers as unseen, so the next tick stamps and re-upserts before the
+    // flag clears.
+    async function refreshSchedulerSeen(queue, lane, schedulerIds) {
+        await markSchedulersSeen(queue, schedulerIds);
+        if (schedulerStampPending) {
+            await upsertCronSchedulers(queue, lane);
+            schedulerStampPending = false;
+        }
+    }
+    function startSchedulerSeenRefresh(queue, lane, schedulerIds) {
+        if (schedulerIds.size === 0)
+            return null;
+        const timer = setInterval(() => {
+            refreshSchedulerSeen(queue, lane, schedulerIds).catch((err) => {
+                errorLogger.warn("scheduler seen refresh failed", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            });
+        }, schedulerSeenRefreshMs);
+        timer.unref();
+        return timer;
+    }
+    // Bounded: a gate that never returns must not block shutdown as well.
+    async function settleAbandonedBootGates(maxWaitMs) {
+        // skip: no timed-out gate left behind
+        if (abandonedBootGates.size === 0)
+            return;
+        const wait = timeoutReject(maxWaitMs, "job-runner: abandoned boot gate still running at stop()");
+        try {
+            await Promise.race([Promise.all(abandonedBootGates), wait.promise]);
+        }
+        catch (err) {
+            errorLogger.warn("abandoned boot gate did not finish before stop()", {
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+        finally {
+            wait.cancel();
+        }
+    }
     const runnerApi = {
         async start() {
             // skip: enqueuer-only runner — no BullMQ worker, no cron schedules,
@@ -1152,28 +1240,18 @@ export function createJobRunner(options) {
             // Only schedule cron + boot for jobs that belong to this lane. Jobs
             // assigned to the other lane get their cron/boot wiring from the
             // runner running on that lane. Running both here would double-fire.
-            const expectedSchedulerIds = new Set();
-            for (const [name, jobDef] of allJobs) {
-                if (laneForJob(jobDef) !== consumerLane)
-                    continue;
-                if ("cron" in jobDef.trigger) {
-                    const schedulerId = schedulerIdForJobName(name);
-                    expectedSchedulerIds.add(schedulerId);
-                    await consumerQueue.upsertJobScheduler(schedulerId, cronRepeatOptions(jobDef.trigger), {
-                        name: jobDef.perTenant ? `_perTenant:${name}` : name,
-                        data: {},
-                        // Queue-level defaultJobOptions (jobRetentionOpts) covers
-                        // retention; a count here would sweep queue-wide again and
-                        // evict perTenant children and boot jobs.
-                        opts: {
-                            ...buildRetryBullOpts(jobDef),
-                        },
-                    });
-                    await (await consumerQueue.client).hset(schedulerSeenKey(consumerQueue), {
-                        [schedulerId]: String(Date.now()),
-                    });
-                }
-            }
+            const expectedSchedulerIds = cronSchedulerIdsForLane(consumerLane);
+            // Before the upserts: a seen entry without a scheduler is harmless, a
+            // scheduler a concurrent older runner finds without one gets deleted.
+            // Best-effort like the periodic refresh: a failed stamp must not stop the start.
+            await markSchedulersSeen(consumerQueue, expectedSchedulerIds).catch((err) => {
+                schedulerStampPending = true;
+                errorLogger.warn("scheduler seen stamp failed", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            });
+            await upsertCronSchedulers(consumerQueue, consumerLane);
+            schedulerSeenTimer = startSchedulerSeenRefresh(consumerQueue, consumerLane, expectedSchedulerIds);
             await removeOrphanJobSchedulers(consumerQueue, expectedSchedulerIds, consumerLane, {
                 info: (msg, data) => context.log ? context.log.info(msg, data) : errorLogger.debug(msg, data),
                 warn: errorLogger.warn,
@@ -1221,6 +1299,10 @@ export function createJobRunner(options) {
                 clearInterval(queueDepthTimer);
                 queueDepthTimer = null;
             }
+            if (schedulerSeenTimer) {
+                clearInterval(schedulerSeenTimer);
+                schedulerSeenTimer = null;
+            }
             if (worker) {
                 await worker.close();
                 worker = null;
@@ -1230,6 +1312,7 @@ export function createJobRunner(options) {
             // without listeners -> unhandled rejection (fw#1805). Wait for readiness
             // first; a failed/timed-out wait must not block the close itself.
             const stopReadyTimeoutMs = Math.min(bootRedisTimeoutMs, STOP_QUEUE_READY_TIMEOUT_MS);
+            await settleAbandonedBootGates(stopReadyTimeoutMs);
             const readinessTimeout = timeoutReject(stopReadyTimeoutMs, `job-runner: Queue not ready within ${stopReadyTimeoutMs}ms during stop()`);
             try {
                 await Promise.race([

package/dist/observability/escape-hatch-report.d.ts

@@ -15,6 +15,7 @@ export type EscapeHatchProcessDedup = {
 };
 export declare function createEscapeHatchProcessDedup(kinds: readonly EscapeHatchKind[]): EscapeHatchProcessDedup;
 export declare function withEscapeHatchAuditScope<T>(run: () => Promise<T>): Promise<T>;
+export declare function withEscapeHatchAuditScopeStream<T>(run: () => AsyncGenerator<T>): AsyncGenerator<T>;
 export declare function flushEscapeHatchAudits(): Promise<void>;
 export declare function reportEscapeHatchUse(event: EscapeHatchUseEvent, deps: {
     readonly sink?: EscapeHatchAuditSink;

package/dist/observability/escape-hatch-report.js

@@ -40,9 +40,14 @@ const processPendingAudits = new Set();
 // The sink writes of the dispatch or job run currently executing.
 const scopePendingAudits = new AsyncLocalStorage();
 function trackPendingAudit(write) {
+    const scope = scopePendingAudits.getStore();
     processPendingAudits.add(write);
-    scopePendingAudits.getStore()?.add(write);
-    void write.finally(() => processPendingAudits.delete(write));
+    scope?.add(write);
+    // Settled writes leave the scope so a long-lived stream does not accumulate them.
+    void write.finally(() => {
+        processPendingAudits.delete(write);
+        scope?.delete(write);
+    });
 }
 // Awaits the audit writes started inside `run` once it has returned, so callers place it around
 // the transaction and the write is durable when the dispatch or run ends. Nested scopes belong
@@ -59,6 +64,30 @@ export async function withEscapeHatchAuditScope(run) {
         await Promise.allSettled([...pending]);
     }
 }
+// Stream variant: the scope has to be entered on every pull because an async generator body
+// runs in the caller's async context, and the wait happens once the stream ends or is closed.
+export async function* withEscapeHatchAuditScopeStream(run) {
+    if (scopePendingAudits.getStore() !== undefined) {
+        yield* run();
+        // skip: an enclosing scope already waits for the pending audits
+        return;
+    }
+    const pending = new Set();
+    const source = run();
+    try {
+        while (true) {
+            const next = await scopePendingAudits.run(pending, () => source.next());
+            // skip: the source is exhausted
+            if (next.done)
+                return;
+            yield next.value;
+        }
+    }
+    finally {
+        await scopePendingAudits.run(pending, () => source.return(undefined));
+        await Promise.allSettled([...pending]);
+    }
+}
 export async function flushEscapeHatchAudits() {
     while (processPendingAudits.size > 0) {
         await Promise.allSettled([...processPendingAudits]);

package/dist/observability/index.d.ts

@@ -4,7 +4,7 @@ export { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeH
 export { getFallbackMeter, getFallbackProvider, getFallbackTracer } from "./fallback.js";
 export { generateSpanId, generateTraceId } from "./ids.js";
 export { buildMetricName, validateLabelKey, validateMetricName, } from "./metric-validator.js";
-export { createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
+export { createMetricsForFactory, createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
 export { type ObservabilityWiring, prometheusMetricsEnvSchema, resolveObservabilityWiring, } from "./metrics-wiring.js";
 export { createNoopProvider } from "./noop-provider.js";
 export { createPrometheusMeter, type PrometheusMeter, type PrometheusMeterSnapshot, serializeOpenMetrics, } from "./prometheus-meter.js";

package/dist/observability/index.js

@@ -5,7 +5,7 @@ export { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeH
 export { getFallbackMeter, getFallbackProvider, getFallbackTracer } from "./fallback.js";
 export { generateSpanId, generateTraceId } from "./ids.js";
 export { buildMetricName, validateLabelKey, validateMetricName, } from "./metric-validator.js";
-export { createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
+export { createMetricsForFactory, createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
 export { prometheusMetricsEnvSchema, resolveObservabilityWiring, } from "./metrics-wiring.js";
 export { createNoopProvider } from "./noop-provider.js";
 export { createPrometheusMeter, serializeOpenMetrics, } from "./prometheus-meter.js";

package/dist/observability/metrics-handle.d.ts

@@ -3,3 +3,4 @@ export declare function createMetricsHandle(meter: Meter, featureName: string):
 export declare function createSafeMetricsHandle(meter: Meter, featureName: string): MetricsHandle;
 export declare function createUnboundMetricsHandle(meter: Meter): MetricsHandle;
 export declare function createNoopMetricsHandle(): MetricsHandle;
+export declare function createMetricsForFactory(meter: Meter | undefined): (featureName: string) => MetricsHandle;

package/dist/observability/metrics-handle.js

@@ -100,3 +100,10 @@ export function createNoopMetricsHandle() {
         set(_name, _value, _labels) { },
     };
 }
+// Shared factory behind ctx.metricsFor for handler and job contexts, so both
+// keep the same no-op-on-unregistered contract (framework#1844).
+export function createMetricsForFactory(meter) {
+    return meter
+        ? (featureName) => createSafeMetricsHandle(meter, featureName)
+        : () => createNoopMetricsHandle();
+}

package/dist/observability/standard-metrics.js

@@ -62,15 +62,15 @@ export const STANDARD_METRIC_DEFS = [
     // Prometheus collapses per-instance shards into last-writer-wins and
     // per-instance lag is invisible to alerting.
     {
-        name: "kumiko_event_consumer_pending_gaps",
+        name: "kumiko_event_consumer_lag_events",
         type: "gauge",
-        description: "Number of unresolved id ranges below the consumer's cursor (ids not yet visible or proven rolled back).",
+        description: "Number of events between the consumer's cursor and the events head.",
         labels: ["consumer", "instance_id"],
     },
     {
-        name: "kumiko_event_consumer_lag_events",
+        name: "kumiko_event_consumer_pending_gaps",
         type: "gauge",
-        description: "Number of events between the consumer's cursor and the events head.",
+        description: "Number of unresolved id ranges below the consumer's cursor (ids not yet visible or proven rolled back).",
         labels: ["consumer", "instance_id"],
     },
     {

package/dist/pipeline/dispatch-query.d.ts

@@ -1,4 +1,5 @@
 import type { DbTx } from "../db/connection.js";
 import type { SessionUser } from "../engine/types/index.js";
 import { type DispatchContext, type WriteOrigin } from "./dispatch-shared.js";
-export declare function executeQuery(ctx: DispatchContext, type: string, payload: unknown, user: SessionUser, origin: WriteOrigin, tx?: DbTx): Promise<unknown>;
+import type { AfterCommitHook } from "./dispatcher-utils.js";
+export declare function executeQuery(ctx: DispatchContext, type: string, payload: unknown, user: SessionUser, origin: WriteOrigin, tx?: DbTx, outerNotifySink?: AfterCommitHook[]): Promise<unknown>;

package/dist/pipeline/dispatch-query.js

@@ -10,10 +10,10 @@ import { runInMemberReadOnlyTransaction } from "./member-read-only-transaction.j
 // Standalone query execution — used by the public dispatcher.query() and
 // by ctx.query/ctx.queryAs inside handlers. Runs the handler, applies
 // field-level read filters for the given user, logs the event.
-export async function executeQuery(ctx, type, payload, user, origin, tx) {
-    return runHandlerInstrumented(ctx, type, "query", user, () => executeQueryInner(ctx, type, payload, user, origin, tx));
+export async function executeQuery(ctx, type, payload, user, origin, tx, outerNotifySink) {
+    return runHandlerInstrumented(ctx, type, "query", user, () => executeQueryInner(ctx, type, payload, user, origin, tx, outerNotifySink));
 }
-async function executeQueryInner(ctx, type, payload, user, origin, tx) {
+async function executeQueryInner(ctx, type, payload, user, origin, tx, outerNotifySink) {
     const { registry } = ctx;
     const handler = registry.getQueryHandler(type);
     if (!handler)
@@ -54,11 +54,11 @@ async function executeQueryInner(ctx, type, payload, user, origin, tx) {
     // A resolved member (ctx.queryAsMember) runs in a Postgres READ ONLY transaction, not just the ctx surface below.
     return isMemberResolutionPrincipal(user)
         ? runInMemberReadOnlyTransaction(ctx, tx, (readOnlyTx) => runQueryHandler(ctx, type, handler, parsed.data, includeDeleted, user, origin, readOnlyTx))
-        : runQueryHandler(ctx, type, handler, parsed.data, includeDeleted, user, origin, tx);
+        : runQueryHandler(ctx, type, handler, parsed.data, includeDeleted, user, origin, tx, outerNotifySink);
 }
-async function runQueryHandler(ctx, type, handler, payload, includeDeleted, user, origin, tx) {
+async function runQueryHandler(ctx, type, handler, payload, includeDeleted, user, origin, tx, outerNotifySink) {
     const { registry } = ctx;
-    const handlerContext = await buildHandlerContext(ctx, type, user, origin, tx, undefined, includeDeleted);
+    const handlerContext = await buildHandlerContext(ctx, type, user, origin, tx, undefined, includeDeleted, undefined, outerNotifySink);
     let result = await handler.handler({ type, payload, user }, handlerContext);
     // postQuery-Hooks: fire BEFORE field-access-filter so hooks see raw data
     // and can merge custom-fields/computed-counts/tags/etc. Each hook is

package/dist/pipeline/dispatch-shared.d.ts

@@ -57,7 +57,7 @@ export declare function perCommandNestedDenied(targetType: string): InternalErro
 export declare function denyTransactionBoundSurfaces(handlerContext: HandlerContext): HandlerContext;
 export declare function applyBeforeTransactionReadOnly(handlerContext: HandlerContext): HandlerContext;
 export type HandlerTransactionScope = "inTransaction" | "withoutTransaction" | "postCommit";
-export declare function buildHandlerContext(ctx: DispatchContext, type: string, user: SessionUser, origin: WriteOrigin, tx?: DbTx, afterCommitHooks?: AfterCommitHook[], includeDeleted?: boolean, transactionScope?: HandlerTransactionScope): Promise<HandlerContext>;
+export declare function buildHandlerContext(ctx: DispatchContext, type: string, user: SessionUser, origin: WriteOrigin, tx?: DbTx, afterCommitHooks?: AfterCommitHook[], includeDeleted?: boolean, transactionScope?: HandlerTransactionScope, outerNotifySink?: AfterCommitHook[], readOnlyForTenant?: boolean): Promise<HandlerContext>;
 export declare function createHandlerBoundDigest(digest: (value: string) => string, handlerName: string): (value: string) => string;
 export declare function runHandlerInstrumented<T>(ctx: DispatchContext, type: string, operation: "query" | "write" | "stream", user: SessionUser, inner: () => Promise<T>): Promise<T>;
 export declare function runStreamInstrumented<T>(ctx: DispatchContext, type: string, user: SessionUser, inner: () => AsyncGenerator<T>): AsyncGenerator<T>;

package/dist/pipeline/dispatch-shared.js

@@ -9,13 +9,13 @@ import { isRateLimitDisabled, resolveAgentExposure } from "../engine/types/index
 import { AccessDeniedError, FeatureDisabledError, FrameworkReasons, InternalError, memberResolutionReadOnlyDenied, RateLimitError, RateLimitUnavailableError, VersionConflictError, writeFailure, } from "../errors/index.js";
 import { archiveStream as archiveStreamHelper, isStreamArchived, restoreStream as restoreStreamHelper, } from "../event-store/archive.js";
 import { IdempotentAppendConflictError as EventStoreIdempotentAppendConflictError, VersionConflictError as EventStoreVersionConflictError, } from "../event-store/errors.js";
-import { getStreamVersion, loadAggregate, loadAggregateAsOf, } from "../event-store/event-store.js";
+import { loadAggregate, loadAggregateAsOf, loadAggregateForWriting, } from "../event-store/event-store.js";
 import { effectiveSnapshotGeneration, loadAggregateWithSnapshot, saveSnapshot, } from "../event-store/snapshot.js";
 import { upcastStoredEvent, upcastStoredEvents } from "../event-store/upcaster.js";
 import { createFileContext } from "../files/file-handle.js";
 import { DEFAULT_LOCALE, isValidLocaleTag } from "../i18n/request-locale.js";
 import { createFallbackLogger } from "../logging/utils.js";
-import { createEscapeHatchReporter, createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, emitDispatcherError, emitDispatcherHandler, getFallbackTracer, observabilityContext, } from "../observability/index.js";
+import { createEscapeHatchReporter, createMetricsForFactory, createMetricsHandle, createNoopMetricsHandle, emitDispatcherError, emitDispatcherHandler, getFallbackTracer, observabilityContext, } from "../observability/index.js";
 import { buildBucketKey, buildPayloadBucketKey, normalizePayloadBucketValue, } from "../rate-limit/index.js";
 import { createTzContext, isValidIanaTimeZone } from "../time/index.js";
 import { isPlainObject } from "../utils/is-plain-object.js";
@@ -257,10 +257,14 @@ export function applyBeforeTransactionReadOnly(handlerContext) {
         ...denyTransactionBoundSurfaces(handlerContext),
         dbOutsideTransaction: undefined,
         outsideTransaction: undefined,
-        forTenant: undefined,
     };
 }
-export async function buildHandlerContext(ctx, type, user, origin, tx, afterCommitHooks, includeDeleted, transactionScope = tx ? "inTransaction" : "withoutTransaction") {
+export async function buildHandlerContext(ctx, type, user, origin, tx, afterCommitHooks, includeDeleted, transactionScope = tx ? "inTransaction" : "withoutTransaction", 
+// A query bridge has no commit sink of its own, but ctx.notify inside it must still wait for
+// the outer write's commit: the outer sink is lent to notify only, not to scheduleAfterCommit.
+outerNotifySink, 
+// beforeTransaction phase: ctx.forTenant stays available but every target db denies writes.
+readOnlyForTenant = false) {
     const { registry, appContext: context, effectiveFeatures, jobRunner, lifecycle } = ctx;
     const isSystem = registry.isHandlerSystemScoped(type);
     // The outer dispatcher receives a DbConnection from the server/stack;
@@ -358,23 +362,30 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
     const notifyInTransactionFactory = notifyFactory && tx !== undefined
         ? (notifyUser, notifyTenantId) => notifyFactory(notifyUser, notifyTenantId, notifyJobDispatcher, outsideTxSource, context._fileProviderResolver)
         : undefined;
+    const notifyAfterCommitFactory = notifyFactory
+        ? (notifyUser, notifyTenantId) => notifyFactory(notifyUser, notifyTenantId, notifyJobDispatcher, undefined, context._fileProviderResolver)
+        : undefined;
     const notify = buildNotify();
     function buildNotify() {
-        if (!notifyFactory)
+        if (!notifyAfterCommitFactory)
             return undefined;
-        const notifyAfterCommit = notifyFactory(user, user.tenantId, notifyJobDispatcher, undefined, context._fileProviderResolver);
+        const notifyAfterCommit = notifyAfterCommitFactory(user, user.tenantId);
         if (!notifyInTransactionFactory)
             return notifyAfterCommit;
         const notifyInTransaction = notifyInTransactionFactory(user, user.tenantId);
-        // A rolled-back write must leave no notification behind.
-        if (!hasLiveCommitSink)
+        // A rolled-back write must leave no notification behind. Without any commit to wait for (a
+        // post-commit context) delivery stays immediate.
+        const notifyCommitSink = hasLiveCommitSink ? bridgeSink : outerNotifySink;
+        if (notifyCommitSink === undefined)
             return notifyInTransaction;
         return createDeferringNotify({
             registry,
             tenantId: user.tenantId,
             notifyAfterCommit,
             notifyNow: notifyInTransaction,
-            scheduleAfterCommit,
+            scheduleAfterCommit: (hook) => {
+                notifyCommitSink.push(hook);
+            },
         });
     }
     // Mirror notify: only built when the config feature wired its factory.
@@ -417,7 +428,7 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
             if (targetTenantId === user.tenantId && exposedDb) {
                 // Only perCommand handlers lose forTenant writes; reservations keep their pre-transaction contract.
                 const ownTenantDb = denyDbWrites && !isPerCommand
-                    ? buildTenantScopedDb(dbSource, reqCtx?.signal)
+                    ? buildTenantScopedDb(dbSource, reqCtx?.signal, undefined, readOnlyForTenant)
                     : exposedDb;
                 return { db: ownTenantDb, streamTenantId: undefined };
             }
@@ -427,7 +438,7 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
                 });
             }
             return {
-                db: buildTenantScopedDb(dbSource, reqCtx?.signal, targetTenantId, denyDbWrites && isPerCommand),
+                db: buildTenantScopedDb(dbSource, reqCtx?.signal, targetTenantId, denyDbWrites && (isPerCommand || readOnlyForTenant)),
                 streamTenantId: targetTenantId,
             };
         }
@@ -455,16 +466,13 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
     const metrics = meter && featureName ? createMetricsHandle(meter, featureName) : createNoopMetricsHandle();
     // ctx.metricsFor(featureName) — shared/library code binds to a feature
     // name of its own choosing instead of the dispatching handler's
-    // (framework#1844). Unregistered names no-op rather than throw, see
-    // createSafeMetricsHandle.
-    const metricsFor = meter
-        ? (targetFeatureName) => createSafeMetricsHandle(meter, targetFeatureName)
-        : () => createNoopMetricsHandle();
+    // (framework#1844). Unregistered names no-op rather than throw.
+    const metricsFor = createMetricsForFactory(meter);
     // A failed inner write has already appended its event and projections when a later
     // step (e.g. an inTransaction hook) fails, and the outer handler may swallow the
     // failure result. The savepoint makes the failure undo exactly the inner write; the
     // inner afterCommit hooks only reach the outer sink if it succeeded.
-    const executeNestedWriteInSavepoint = async (asUser, targetType, payload) => {
+    const runNestedWriteInSavepoint = async (asUser, targetType, payload) => {
         if (registry.getWriteHandler(targetType)?.transaction === "perCommand") {
             return writeFailure(perCommandNestedDenied(targetType));
         }
@@ -488,6 +496,15 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
             throw e;
         }
     };
+    // Savepoints on one connection are not serialized by the driver: two parallel nested writes would
+    // interleave their statements and a ROLLBACK TO of the first would also discard the second's rows.
+    // Writes nested one level deeper run on their own savepoint handle and therefore their own chain.
+    let nestedWriteChain = Promise.resolve();
+    const executeNestedWriteInSavepoint = (asUser, targetType, payload) => {
+        const run = nestedWriteChain.then(() => runNestedWriteInSavepoint(asUser, targetType, payload));
+        nestedWriteChain = run.catch(() => undefined);
+        return run;
+    };
     // Without an open transaction (post-commit hooks, stream/query handlers, perCommand handlers) a bare
     // executeWrite would auto-commit event append, projections and inTransaction hooks separately and its
     // afterCommit hooks would land in a sink nobody flushes. A batch-of-one gives the write its own
@@ -498,7 +515,7 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
     const identitySwitchAudit = { reason: identitySwitchGrantReason, report: reportEscapeHatch };
     const identitySwitch = createGatedIdentitySwitch(`handler "${type}"`, user, hasIdentitySwitchGrant, {
         // Inherits the caller's origin, so switching to SYSTEM cannot shed an anonymous root.
-        queryAs: (asUser, targetType, payload) => executeQuery(ctx, targetType, payload, asUser, origin, tx), // @wrapper-known semantic-alias
+        queryAs: (asUser, targetType, payload) => executeQuery(ctx, targetType, payload, asUser, origin, tx, hasLiveCommitSink ? bridgeSink : outerNotifySink), // @wrapper-known semantic-alias
         writeAs: writeAsUser,
     }, identitySwitchAudit);
     // Lazy — creates the reader (which fails closed on SYSTEM_TENANT_ID) only
@@ -542,7 +559,7 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
         return projTable === undefined || hasTenantColumn(projTable);
     });
     const bridge = {
-        query: (targetType, payload) => executeQuery(ctx, targetType, payload, user, origin, tx), // @wrapper-known semantic-alias
+        query: (targetType, payload) => executeQuery(ctx, targetType, payload, user, origin, tx, hasLiveCommitSink ? bridgeSink : outerNotifySink), // @wrapper-known semantic-alias
         queryAs: identitySwitch.queryAs,
         write: (targetType, payload) => writeAsUser(user, targetType, payload),
         writeAs: identitySwitch.writeAs,
@@ -597,16 +614,9 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
                     message: `ctx.fetchForWriting("${args.aggregateId}") requires a database connection — none is configured.`,
                 });
             }
-            // Stream-version authoritative (same policy as CRUD executor + Block 0).
-            // A single SELECT MAX(version) is cheaper than loading the full stream
-            // when the caller just wants to append — but most callers also want
-            // the events (business-rule checks), so fetch both in parallel.
-            const [storedEvents, fetchedVersion] = await Promise.all([
-                loadAggregate(dbSource, args.aggregateId, user.tenantId, {
-                    aggregateType: args.aggregateType,
-                }),
-                getStreamVersion(dbSource, args.aggregateId, user.tenantId),
-            ]);
+            // Stream-version authoritative (same policy as CRUD executor + Block 0), read together
+            // with the events so the pinned version covers exactly what the handler's rules saw.
+            const { events: storedEvents, streamVersion: fetchedVersion } = await loadAggregateForWriting(dbSource, args.aggregateId, user.tenantId, { aggregateType: args.aggregateType });
             const events = await upcastStoredEvents(storedEvents, registry.getEventUpcasters(), {
                 db: dbSource,
                 tenantId: user.tenantId,
@@ -831,11 +841,12 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
     // able to compute bucket keys.
     const { _rateLimitPayloadDigest: payloadDigest, 
     // Bound to this call's tx: an outer context's factory must not leak into a tx-less one.
-    _notifyInTransactionFactory: _omitOuterNotifyInTransactionFactory, ...contextForHandler } = context;
+    _notifyInTransactionFactory: _omitOuterNotifyInTransactionFactory, _notifyAfterCommitFactory: _omitOuterNotifyAfterCommitFactory, ...contextForHandler } = context;
     const handlerContext = {
         ...contextForHandler,
         ...(payloadDigest && { rateLimitDigest: createHandlerBoundDigest(payloadDigest, type) }),
         ...(notifyInTransactionFactory && { _notifyInTransactionFactory: notifyInTransactionFactory }),
+        ...(notifyAfterCommitFactory && { _notifyAfterCommitFactory: notifyAfterCommitFactory }),
         registry,
         db: exposedDb,
         dbOutsideTransaction,

package/dist/pipeline/dispatch-stream.js

@@ -2,7 +2,7 @@ import { accessInvalidationCredentialFor } from "../api/sse-broker.js";
 import { scheduleTokenExpiry } from "../api/token-expiry-timer.js";
 import { hasAccess } from "../engine/access.js";
 import { tenantOverrideDenied } from "../engine/cross-tenant.js";
-import { AccessDeniedError, memberResolutionReadOnlyDenied, NotFoundError, validationErrorFromZod, } from "../errors/index.js";
+import { AccessDeniedError, memberResolutionReadOnlyDenied, NotFoundError, UnauthenticatedError, validationErrorFromZod, } from "../errors/index.js";
 import { assertNoSecretLeak } from "../secrets/index.js";
 import { buildHandlerContext, enforceRateLimit, ensureFeatureEnabled, isMemberResolutionPrincipal, runStreamInstrumented, } from "./dispatch-shared.js";
 import { handlerAccessError } from "./handler-access-error.js";
@@ -69,10 +69,16 @@ async function* executeStreamInner(ctx, type, payload, user, origin, options) {
                 abandonedForInvalidation = true;
                 void nextPull.catch(() => { });
                 void iterator.return?.(undefined)?.then(undefined, () => { });
+                // An expired token is a lapsed session (401, so clients show the
+                // session-end hint); revocation is a permission change (403).
+                if (outcome.reason === "expired") {
+                    throw new UnauthenticatedError({
+                        message: `token expired mid-stream for ${type}`,
+                        details: { handler: type, reason: outcome.reason },
+                    });
+                }
                 throw new AccessDeniedError({
-                    message: outcome.reason === "expired"
-                        ? `token expired mid-stream for ${type}`
-                        : `access revoked mid-stream for ${type}`,
+                    message: `access revoked mid-stream for ${type}`,
                     details: { handler: type },
                 });
             }

package/dist/pipeline/dispatcher.js

@@ -1,6 +1,7 @@
 import { runWithWriteOrigin } from "../api/request-context.js";
 import { TENANT_MEMBERSHIPS_QUERY } from "../engine/extension-names.js";
 import { InternalError, reraiseAsKumikoError } from "../errors/index.js";
+import { withEscapeHatchAuditScopeStream } from "../observability/escape-hatch-report.js";
 import { createEscapeHatchReportWindow, getFallbackMeter, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
 import { CACHE_SYNC_TOPICS, isTenantConfigSyncMessage } from "../redis/cache-sync-topics.js";
 import { INTERACTIVE_SIGN_IN_POLICY, resolveActiveMembershipFn } from "./active-membership.js";
@@ -114,7 +115,8 @@ export function createDispatcher(registry, context, options = {}) {
         },
         stream: (typeOrRef, payload, user, options) => {
             const type = resolveType(typeOrRef);
-            return executeStream(ctx, type, payload, user, rootWriteOrigin(registry, type, user), options);
+            const origin = rootWriteOrigin(registry, type, user);
+            return withEscapeHatchAuditScopeStream(() => executeStream(ctx, type, payload, user, origin, options));
         },
         async command(typeOrRef, payload, user) {
             const type = resolveType(typeOrRef);

package/dist/pipeline/event-dispatcher.d.ts

@@ -1,4 +1,4 @@
-import type { PgClient } from "../db/connection.js";
+import type { PgListenClient } from "../db/connection.js";
 import type { AppContext } from "../engine/types/index.js";
 import { type StoredEvent } from "../event-store/index.js";
 import { type Meter, type Tracer } from "../observability/index.js";
@@ -46,7 +46,7 @@ export type EventDispatcherOptions = {
     readonly tracer?: Tracer;
     readonly meter?: Meter;
     readonly instanceId?: string;
-    readonly pgClient?: PgClient;
+    readonly pgClient?: PgListenClient;
 };
 export declare function createEventDispatcher(options: EventDispatcherOptions): EventDispatcher;
 export type { ConsumerProgress, ConsumerRecoveryState } from "./event-dispatcher-admin.js";