@cosmicdrift/kumiko-framework 0.364.0 → 0.365.1

package/dist/jobs/job-runner.js

@@ -1,6 +1,6 @@
 import { escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
 import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
-import { Queue, UnrecoverableError, Worker } from "bullmq";
+import { DelayedError, Queue, UnrecoverableError, Worker } from "bullmq";
 import { Redis } from "ioredis";
 import { requestContext } from "../api/request-context.js";
 import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
@@ -12,13 +12,14 @@ import { InternalError } from "../errors/index.js";
 import { isKumikoError } from "../errors/kumiko-error.js";
 import { createFileContext } from "../files/file-handle.js";
 import { createFallbackLogger } from "../logging/index.js";
-import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
+import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
 import { RedisKeys } from "../pipeline/redis-keys.js";
 import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js";
 import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
 import { bridgeStub } from "../testing/handler-context.js";
 import { cronRepeatOptions } from "./cron-repeat-options.js";
+import { resolveMaxPerTenant } from "./max-per-tenant.js";
 // Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
 // (a security-relevant escape hatch that must fail the run closed), a
 // malformed `_triggerEvent` only costs an idempotency-key convenience — the
@@ -82,6 +83,18 @@ export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
 function schedulerSeenKey(queue) {
     return queue.toKey("kumiko-scheduler-seen");
 }
+// "Seen" has to mean a live runner holds the scheduler, so runners re-stamp
+// their own ids periodically; a single stamp at start() would let a runner
+// restarting during a mixed-version window longer than the grace period delete
+// a scheduler a healthy newer pod still owns.
+async function markSchedulersSeen(queue, schedulerIds) {
+    // skip: no cron scheduler on this lane, nothing to stamp
+    if (schedulerIds.size === 0)
+        return;
+    const stamp = String(Date.now());
+    const entries = Object.fromEntries([...schedulerIds].map((id) => [id, stamp]));
+    await (await queue.client).hset(schedulerSeenKey(queue), entries);
+}
 // A cron job dropped from the app (or an older scheduler-id format) leaves its
 // scheduler in Redis, where it keeps firing jobs the worker no longer knows
 // ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
@@ -236,6 +249,9 @@ function tenantFailureMessageKey(err, declaration) {
 // Leading underscore matches the existing internal-meta convention
 // (_triggeredById, _tenantId, _payload).
 const TRACE_CONTEXT_KEY = "_traceContext";
+const DEFER_COUNT_KEY = "_deferCount";
+const DEFER_DELAY_CAP_MS = 30_000;
+const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
 function readTraceContext(data) {
     const raw = data[TRACE_CONTEXT_KEY];
     if (!raw || typeof raw !== "object")
@@ -364,6 +380,7 @@ export function createJobRunner(options) {
     const { registry, context, redisUrl, consumerLane } = options;
     const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;
     const bootRedisTimeoutMs = options.bootRedisTimeoutMs ?? BOOT_REDIS_TIMEOUT_MS;
+    const schedulerSeenRefreshMs = options.schedulerSeenRefreshMs ?? ORPHAN_SCHEDULER_GRACE_MS / 4;
     const redisOpts = parseRedisOpts(redisUrl);
     // Use the context's tracer when present (observability-provider injected at
     // boot); otherwise noop so dispatch/handleJob stay zero-cost without config.
@@ -524,6 +541,11 @@ export function createJobRunner(options) {
     }
     let worker = null;
     let queueDepthTimer = null;
+    let schedulerSeenTimer = null;
+    let schedulerStampPending = false;
+    // A timed-out gate is abandoned, not aborted (the job context carries no
+    // AbortSignal); stop() waits for it so it cannot write after the shutdown.
+    const abandonedBootGates = new Set();
     // Forward reference to the runner's own API, exposed on the job-handler ctx
     // so a handler can dispatch a follow-up job (job→job chaining, e.g.
     // delivery.render → delivery.send). Assigned just before return; reads happen
@@ -537,18 +559,23 @@ export function createJobRunner(options) {
     // lane (jobDef.runIn is static), but walking both is cheap and avoids
     // a subtle bug if someone ever reassigns a job to a different lane
     // between deploys while old queue contents are still draining.
-    async function isOverPerTenantLimit(jobName, tenantId, max) {
+    // `activeOnly` + `self` serve the defer check, which runs inside the job being
+    // counted: it must not see waiting jobs or itself, and only counts jobs that
+    // became active before it. Without that order, workers that fetch a burst of
+    // jobs together all see each other as active and all defer, again and again.
+    async function isOverPerTenantLimit(jobName, tenantId, max, scope = {}) {
         const results = await Promise.all([
             queues.api.getActive(),
-            queues.api.getWaiting(),
             queues.worker.getActive(),
-            queues.worker.getWaiting(),
+            ...(scope.activeOnly ? [] : [queues.api.getWaiting(), queues.worker.getWaiting()]),
         ]);
         let count = 0;
         for (const list of results) {
             for (const j of list) {
                 if (j.name !== jobName)
                     continue;
+                if (scope.self && !isActiveBefore(j, scope.self))
+                    continue;
                 const t = j.data?._tenantId; // @cast-boundary dynamic-key
                 if (t === tenantId) {
                     count += 1;
@@ -559,6 +586,22 @@ export function createJobRunner(options) {
         }
         return false;
     }
+    function isActiveBefore(other, self) {
+        if (other.id === self.id)
+            return false;
+        if (self.processedOn === undefined)
+            return true;
+        const otherProcessedOn = other.processedOn ?? 0;
+        if (otherProcessedOn !== self.processedOn)
+            return otherProcessedOn < self.processedOn;
+        return (other.id ?? "") < (self.id ?? "");
+    }
+    // Growing delay bounds Redis and worker load while one tenant has a large
+    // backlog: every deferred job would otherwise be re-checked every deferMs.
+    function deferDelayMs(deferMs, deferCount) {
+        const grown = Math.min(deferMs * 2 ** Math.min(deferCount, DEFER_BACKOFF_MAX_DOUBLINGS), DEFER_DELAY_CAP_MS);
+        return grown + Math.floor(Math.random() * deferMs);
+    }
     // Only err.message, never payload or the logs array: job payloads can carry PII.
     function logFailedAttempt(entry) {
         errorLogger.error("job failed", {
@@ -629,7 +672,7 @@ export function createJobRunner(options) {
             });
         }
     }
-    async function handleJob(bullJob) {
+    async function handleJob(bullJob, token) {
         const rawName = bullJob.name;
         // Handle perTenant dispatch jobs — fan out to one job per tenant. The
         // fan-out re-enqueues into the lane the actual job is assigned to;
@@ -711,6 +754,25 @@ export function createJobRunner(options) {
             await reportPreRunFailure(jobName, bullJob, err);
             throw err;
         }
+        // maxPerTenant defer: checked before anything is recorded for this run, so a
+        // deferred pickup leaves no span, run-log entry or failure trace and does not
+        // consume an attempt (moveToDelayed + DelayedError, not a retry).
+        const maxPerTenant = resolveMaxPerTenant(jobDef);
+        const deferTenantId = bullJob.data["_tenantId"];
+        if (maxPerTenant?.onLimit === "defer" &&
+            typeof deferTenantId === "string" &&
+            token !== undefined &&
+            bullJob.moveToDelayed &&
+            (await isOverPerTenantLimit(jobName, deferTenantId, maxPerTenant.limit, {
+                activeOnly: true,
+                self: { id: bullJob.id, processedOn: bullJob.processedOn },
+            }))) {
+            const priorDeferCount = bullJob.data[DEFER_COUNT_KEY];
+            const deferCount = typeof priorDeferCount === "number" ? priorDeferCount : 0;
+            await bullJob.updateData?.({ ...bullJob.data, [DEFER_COUNT_KEY]: deferCount + 1 });
+            await bullJob.moveToDelayed(Date.now() + deferDelayMs(maxPerTenant.deferMs, deferCount), token);
+            throw new DelayedError();
+        }
         // Sequential gate: try to claim the per-name lock. If another worker
         // (or this worker on a different bullJob) holds it, re-enqueue with a
         // small delay and exit *successfully* — using throw would burn the
@@ -943,6 +1005,7 @@ export function createJobRunner(options) {
             attempt,
             finalAttempt,
             log: createJobLogger(logs),
+            metricsFor: createMetricsForFactory(context.meter),
             ...(triggerName !== undefined && { triggerName }),
             ...(triggerEvent !== undefined && { triggerEvent }),
             write: (qn, payload) => {
@@ -1078,17 +1141,18 @@ export function createJobRunner(options) {
                     });
                 }, BOOT_GATE_SLOW_WARN_MS)
                 : undefined;
+            const gateRun = handleJob({
+                id: bootJobIdForJobName(name),
+                name,
+                data: {},
+                attemptsMade: 0,
+                retryable: false,
+            });
+            const gateSettled = gateRun.then(() => undefined, () => undefined);
+            abandonedBootGates.add(gateSettled);
+            void gateSettled.then(() => abandonedBootGates.delete(gateSettled));
             try {
-                await Promise.race([
-                    handleJob({
-                        id: bootJobIdForJobName(name),
-                        name,
-                        data: {},
-                        attemptsMade: 0,
-                        retryable: false,
-                    }),
-                    ...(gateTimeout ? [gateTimeout.promise] : []),
-                ]);
+                await Promise.race([gateRun, ...(gateTimeout ? [gateTimeout.promise] : [])]);
             }
             catch (err) {
                 // Leaving the worker's Redis connections open would keep the event loop alive and turn the aborted boot into
@@ -1104,6 +1168,74 @@ export function createJobRunner(options) {
             }
         }
     }
+    function cronSchedulerIdsForLane(lane) {
+        const ids = new Set();
+        for (const [name, jobDef] of allJobs) {
+            if (laneForJob(jobDef) === lane && "cron" in jobDef.trigger) {
+                ids.add(schedulerIdForJobName(name));
+            }
+        }
+        return ids;
+    }
+    async function upsertCronSchedulers(queue, lane) {
+        for (const [name, jobDef] of allJobs) {
+            if (laneForJob(jobDef) !== lane)
+                continue;
+            if ("cron" in jobDef.trigger) {
+                await queue.upsertJobScheduler(schedulerIdForJobName(name), cronRepeatOptions(jobDef.trigger), {
+                    name: jobDef.perTenant ? `_perTenant:${name}` : name,
+                    data: {},
+                    // Queue-level defaultJobOptions (jobRetentionOpts) covers
+                    // retention; a count here would sweep queue-wide again and
+                    // evict perTenant children and boot jobs.
+                    opts: {
+                        ...buildRetryBullOpts(jobDef),
+                    },
+                });
+            }
+        }
+    }
+    // After a failed initial stamp an older runner may have removed our
+    // schedulers as unseen, so the next tick stamps and re-upserts before the
+    // flag clears.
+    async function refreshSchedulerSeen(queue, lane, schedulerIds) {
+        await markSchedulersSeen(queue, schedulerIds);
+        if (schedulerStampPending) {
+            await upsertCronSchedulers(queue, lane);
+            schedulerStampPending = false;
+        }
+    }
+    function startSchedulerSeenRefresh(queue, lane, schedulerIds) {
+        if (schedulerIds.size === 0)
+            return null;
+        const timer = setInterval(() => {
+            refreshSchedulerSeen(queue, lane, schedulerIds).catch((err) => {
+                errorLogger.warn("scheduler seen refresh failed", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            });
+        }, schedulerSeenRefreshMs);
+        timer.unref();
+        return timer;
+    }
+    // Bounded: a gate that never returns must not block shutdown as well.
+    async function settleAbandonedBootGates(maxWaitMs) {
+        // skip: no timed-out gate left behind
+        if (abandonedBootGates.size === 0)
+            return;
+        const wait = timeoutReject(maxWaitMs, "job-runner: abandoned boot gate still running at stop()");
+        try {
+            await Promise.race([Promise.all(abandonedBootGates), wait.promise]);
+        }
+        catch (err) {
+            errorLogger.warn("abandoned boot gate did not finish before stop()", {
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+        finally {
+            wait.cancel();
+        }
+    }
     const runnerApi = {
         async start() {
             // skip: enqueuer-only runner — no BullMQ worker, no cron schedules,
@@ -1114,7 +1246,7 @@ export function createJobRunner(options) {
                 return;
             }
             const consumerQueue = queues[consumerLane];
-            worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), handleJob, {
+            worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), (job, token) => handleJob(job, token), {
                 connection: redisOpts,
                 concurrency: 5,
                 // Not consuming until the boot gates passed: a job already queued from a
@@ -1152,28 +1284,18 @@ export function createJobRunner(options) {
             // Only schedule cron + boot for jobs that belong to this lane. Jobs
             // assigned to the other lane get their cron/boot wiring from the
             // runner running on that lane. Running both here would double-fire.
-            const expectedSchedulerIds = new Set();
-            for (const [name, jobDef] of allJobs) {
-                if (laneForJob(jobDef) !== consumerLane)
-                    continue;
-                if ("cron" in jobDef.trigger) {
-                    const schedulerId = schedulerIdForJobName(name);
-                    expectedSchedulerIds.add(schedulerId);
-                    await consumerQueue.upsertJobScheduler(schedulerId, cronRepeatOptions(jobDef.trigger), {
-                        name: jobDef.perTenant ? `_perTenant:${name}` : name,
-                        data: {},
-                        // Queue-level defaultJobOptions (jobRetentionOpts) covers
-                        // retention; a count here would sweep queue-wide again and
-                        // evict perTenant children and boot jobs.
-                        opts: {
-                            ...buildRetryBullOpts(jobDef),
-                        },
-                    });
-                    await (await consumerQueue.client).hset(schedulerSeenKey(consumerQueue), {
-                        [schedulerId]: String(Date.now()),
-                    });
-                }
-            }
+            const expectedSchedulerIds = cronSchedulerIdsForLane(consumerLane);
+            // Before the upserts: a seen entry without a scheduler is harmless, a
+            // scheduler a concurrent older runner finds without one gets deleted.
+            // Best-effort like the periodic refresh: a failed stamp must not stop the start.
+            await markSchedulersSeen(consumerQueue, expectedSchedulerIds).catch((err) => {
+                schedulerStampPending = true;
+                errorLogger.warn("scheduler seen stamp failed", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            });
+            await upsertCronSchedulers(consumerQueue, consumerLane);
+            schedulerSeenTimer = startSchedulerSeenRefresh(consumerQueue, consumerLane, expectedSchedulerIds);
             await removeOrphanJobSchedulers(consumerQueue, expectedSchedulerIds, consumerLane, {
                 info: (msg, data) => context.log ? context.log.info(msg, data) : errorLogger.debug(msg, data),
                 warn: errorLogger.warn,
@@ -1221,6 +1343,10 @@ export function createJobRunner(options) {
                 clearInterval(queueDepthTimer);
                 queueDepthTimer = null;
             }
+            if (schedulerSeenTimer) {
+                clearInterval(schedulerSeenTimer);
+                schedulerSeenTimer = null;
+            }
             if (worker) {
                 await worker.close();
                 worker = null;
@@ -1230,6 +1356,7 @@ export function createJobRunner(options) {
             // without listeners -> unhandled rejection (fw#1805). Wait for readiness
             // first; a failed/timed-out wait must not block the close itself.
             const stopReadyTimeoutMs = Math.min(bootRedisTimeoutMs, STOP_QUEUE_READY_TIMEOUT_MS);
+            await settleAbandonedBootGates(stopReadyTimeoutMs);
             const readinessTimeout = timeoutReject(stopReadyTimeoutMs, `job-runner: Queue not ready within ${stopReadyTimeoutMs}ms during stop()`);
             try {
                 await Promise.race([
@@ -1278,10 +1405,11 @@ export function createJobRunner(options) {
             // against; without it the guard is inactive (system jobs, ambient
             // dispatch). Fan-out children of perTenant jobs land here on their
             // recursive queue.add and DO carry _tenantId.
-            if (jobDef.maxPerTenant !== undefined) {
+            const maxPerTenant = resolveMaxPerTenant(jobDef);
+            if (maxPerTenant?.onLimit === "skip") {
                 const tenantId = payload?._tenantId; // @cast-boundary dynamic-key
                 if (tenantId !== undefined &&
-                    (await isOverPerTenantLimit(jobName, tenantId, jobDef.maxPerTenant))) {
+                    (await isOverPerTenantLimit(jobName, tenantId, maxPerTenant.limit))) {
                     return "skipped:max-per-tenant";
                 }
             }
@@ -1323,6 +1451,8 @@ export function createJobRunner(options) {
                 bullOpts["timeout"] = jobDef.timeout;
             if (meta?.priority !== undefined)
                 bullOpts["priority"] = meta.priority;
+            if (meta?.jobId !== undefined)
+                bullOpts["jobId"] = meta.jobId;
             // Pack meta into job data with _ prefix
             const data = { ...payload };
             if (meta?.triggeredById !== undefined)
@@ -1384,8 +1514,9 @@ export function createJobRunner(options) {
                     data["_correlationId"] = reqCtx.correlationId;
                 // Same maxPerTenant guard as dispatch — events that fan into many
                 // jobs must respect the per-tenant cap or the limit is one-sided.
-                if (jobDef.maxPerTenant !== undefined && user?.tenantId !== undefined) {
-                    if (await isOverPerTenantLimit(name, String(user.tenantId), jobDef.maxPerTenant)) {
+                const maxPerTenant = resolveMaxPerTenant(jobDef);
+                if (maxPerTenant?.onLimit === "skip" && user?.tenantId !== undefined) {
+                    if (await isOverPerTenantLimit(name, String(user.tenantId), maxPerTenant.limit)) {
                         continue;
                     }
                 }
@@ -1402,17 +1533,21 @@ export function createJobRunner(options) {
         async countPendingJobs() {
             let total = 0;
             for (const queue of Object.values(queues)) {
-                const counts = await queue.getJobCounts("waiting", "active", "prioritized", "waiting-children");
+                // One getJobCounts call is a single atomic snapshot. Reading delayed in a
+                // second call misses a deferred job that cycles delayed -> active between
+                // the reads, so delayed is counted here and only repeat schedulers (which
+                // stay delayed between cron fires) are subtracted afterwards.
+                const counts = await queue.getJobCounts("waiting", "active", "prioritized", "waiting-children", "delayed");
                 total +=
                     (counts["waiting"] ?? 0) +
                         (counts["active"] ?? 0) +
                         (counts["prioritized"] ?? 0) +
-                        (counts["waiting-children"] ?? 0);
-                // getJobCounts has no id-prefix filter for "delayed" — getJobs(["delayed"])
-                // loads the full Job objects (payload included) just to read .id; fine for
+                        (counts["waiting-children"] ?? 0) +
+                        (counts["delayed"] ?? 0);
+                // getJobs(["delayed"]) loads full Job objects just to read .id; fine for
                 // test-draining's low volume, not a pattern for a prod-path hot loop.
                 const delayed = await queue.getJobs(["delayed"]);
-                total += delayed.filter((job) => !(job.id ?? "").startsWith("repeat:")).length;
+                total -= delayed.filter((job) => (job.id ?? "").startsWith("repeat:")).length;
             }
             return total;
         },

package/dist/jobs/max-per-tenant.d.ts

@@ -0,0 +1,8 @@
+import type { JobDefinition } from "../engine/types/index.js";
+export declare const DEFAULT_MAX_PER_TENANT_DEFER_MS = 1000;
+export type ResolvedMaxPerTenant = {
+    readonly limit: number;
+    readonly onLimit: "skip" | "defer";
+    readonly deferMs: number;
+};
+export declare function resolveMaxPerTenant(jobDef: Pick<JobDefinition, "maxPerTenant">): ResolvedMaxPerTenant | undefined;

package/dist/jobs/max-per-tenant.js

@@ -0,0 +1,14 @@
+export const DEFAULT_MAX_PER_TENANT_DEFER_MS = 1_000;
+export function resolveMaxPerTenant(jobDef) {
+    const raw = jobDef.maxPerTenant;
+    if (raw === undefined)
+        return undefined;
+    if (typeof raw === "number") {
+        return { limit: raw, onLimit: "skip", deferMs: DEFAULT_MAX_PER_TENANT_DEFER_MS };
+    }
+    return {
+        limit: raw.limit,
+        onLimit: raw.onLimit,
+        deferMs: raw.deferMs ?? DEFAULT_MAX_PER_TENANT_DEFER_MS,
+    };
+}

package/dist/observability/escape-hatch-report.d.ts

@@ -15,6 +15,7 @@ export type EscapeHatchProcessDedup = {
 };
 export declare function createEscapeHatchProcessDedup(kinds: readonly EscapeHatchKind[]): EscapeHatchProcessDedup;
 export declare function withEscapeHatchAuditScope<T>(run: () => Promise<T>): Promise<T>;
+export declare function withEscapeHatchAuditScopeStream<T>(run: () => AsyncGenerator<T>): AsyncGenerator<T>;
 export declare function flushEscapeHatchAudits(): Promise<void>;
 export declare function reportEscapeHatchUse(event: EscapeHatchUseEvent, deps: {
     readonly sink?: EscapeHatchAuditSink;

package/dist/observability/escape-hatch-report.js

@@ -40,9 +40,14 @@ const processPendingAudits = new Set();
 // The sink writes of the dispatch or job run currently executing.
 const scopePendingAudits = new AsyncLocalStorage();
 function trackPendingAudit(write) {
+    const scope = scopePendingAudits.getStore();
     processPendingAudits.add(write);
-    scopePendingAudits.getStore()?.add(write);
-    void write.finally(() => processPendingAudits.delete(write));
+    scope?.add(write);
+    // Settled writes leave the scope so a long-lived stream does not accumulate them.
+    void write.finally(() => {
+        processPendingAudits.delete(write);
+        scope?.delete(write);
+    });
 }
 // Awaits the audit writes started inside `run` once it has returned, so callers place it around
 // the transaction and the write is durable when the dispatch or run ends. Nested scopes belong
@@ -59,6 +64,30 @@ export async function withEscapeHatchAuditScope(run) {
         await Promise.allSettled([...pending]);
     }
 }
+// Stream variant: the scope has to be entered on every pull because an async generator body
+// runs in the caller's async context, and the wait happens once the stream ends or is closed.
+export async function* withEscapeHatchAuditScopeStream(run) {
+    if (scopePendingAudits.getStore() !== undefined) {
+        yield* run();
+        // skip: an enclosing scope already waits for the pending audits
+        return;
+    }
+    const pending = new Set();
+    const source = run();
+    try {
+        while (true) {
+            const next = await scopePendingAudits.run(pending, () => source.next());
+            // skip: the source is exhausted
+            if (next.done)
+                return;
+            yield next.value;
+        }
+    }
+    finally {
+        await scopePendingAudits.run(pending, () => source.return(undefined));
+        await Promise.allSettled([...pending]);
+    }
+}
 export async function flushEscapeHatchAudits() {
     while (processPendingAudits.size > 0) {
         await Promise.allSettled([...processPendingAudits]);

package/dist/observability/index.d.ts

@@ -4,7 +4,7 @@ export { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeH
 export { getFallbackMeter, getFallbackProvider, getFallbackTracer } from "./fallback.js";
 export { generateSpanId, generateTraceId } from "./ids.js";
 export { buildMetricName, validateLabelKey, validateMetricName, } from "./metric-validator.js";
-export { createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
+export { createMetricsForFactory, createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
 export { type ObservabilityWiring, prometheusMetricsEnvSchema, resolveObservabilityWiring, } from "./metrics-wiring.js";
 export { createNoopProvider } from "./noop-provider.js";
 export { createPrometheusMeter, type PrometheusMeter, type PrometheusMeterSnapshot, serializeOpenMetrics, } from "./prometheus-meter.js";

package/dist/observability/index.js

@@ -5,7 +5,7 @@ export { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeH
 export { getFallbackMeter, getFallbackProvider, getFallbackTracer } from "./fallback.js";
 export { generateSpanId, generateTraceId } from "./ids.js";
 export { buildMetricName, validateLabelKey, validateMetricName, } from "./metric-validator.js";
-export { createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
+export { createMetricsForFactory, createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
 export { prometheusMetricsEnvSchema, resolveObservabilityWiring, } from "./metrics-wiring.js";
 export { createNoopProvider } from "./noop-provider.js";
 export { createPrometheusMeter, serializeOpenMetrics, } from "./prometheus-meter.js";

package/dist/observability/metrics-handle.d.ts

@@ -3,3 +3,4 @@ export declare function createMetricsHandle(meter: Meter, featureName: string):
 export declare function createSafeMetricsHandle(meter: Meter, featureName: string): MetricsHandle;
 export declare function createUnboundMetricsHandle(meter: Meter): MetricsHandle;
 export declare function createNoopMetricsHandle(): MetricsHandle;
+export declare function createMetricsForFactory(meter: Meter | undefined): (featureName: string) => MetricsHandle;

package/dist/observability/metrics-handle.js

@@ -100,3 +100,10 @@ export function createNoopMetricsHandle() {
         set(_name, _value, _labels) { },
     };
 }
+// Shared factory behind ctx.metricsFor for handler and job contexts, so both
+// keep the same no-op-on-unregistered contract (framework#1844).
+export function createMetricsForFactory(meter) {
+    return meter
+        ? (featureName) => createSafeMetricsHandle(meter, featureName)
+        : () => createNoopMetricsHandle();
+}

package/dist/observability/standard-metrics.js

@@ -62,15 +62,15 @@ export const STANDARD_METRIC_DEFS = [
     // Prometheus collapses per-instance shards into last-writer-wins and
     // per-instance lag is invisible to alerting.
     {
-        name: "kumiko_event_consumer_pending_gaps",
+        name: "kumiko_event_consumer_lag_events",
         type: "gauge",
-        description: "Number of unresolved id ranges below the consumer's cursor (ids not yet visible or proven rolled back).",
+        description: "Number of events between the consumer's cursor and the events head.",
         labels: ["consumer", "instance_id"],
     },
     {
-        name: "kumiko_event_consumer_lag_events",
+        name: "kumiko_event_consumer_pending_gaps",
         type: "gauge",
-        description: "Number of events between the consumer's cursor and the events head.",
+        description: "Number of unresolved id ranges below the consumer's cursor (ids not yet visible or proven rolled back).",
         labels: ["consumer", "instance_id"],
     },
     {

package/dist/pipeline/dispatch-query.d.ts

@@ -1,4 +1,5 @@
 import type { DbTx } from "../db/connection.js";
 import type { SessionUser } from "../engine/types/index.js";
 import { type DispatchContext, type WriteOrigin } from "./dispatch-shared.js";
-export declare function executeQuery(ctx: DispatchContext, type: string, payload: unknown, user: SessionUser, origin: WriteOrigin, tx?: DbTx): Promise<unknown>;
+import type { AfterCommitHook } from "./dispatcher-utils.js";
+export declare function executeQuery(ctx: DispatchContext, type: string, payload: unknown, user: SessionUser, origin: WriteOrigin, tx?: DbTx, outerNotifySink?: AfterCommitHook[]): Promise<unknown>;

package/dist/pipeline/dispatch-query.js

@@ -10,10 +10,10 @@ import { runInMemberReadOnlyTransaction } from "./member-read-only-transaction.j
 // Standalone query execution — used by the public dispatcher.query() and
 // by ctx.query/ctx.queryAs inside handlers. Runs the handler, applies
 // field-level read filters for the given user, logs the event.
-export async function executeQuery(ctx, type, payload, user, origin, tx) {
-    return runHandlerInstrumented(ctx, type, "query", user, () => executeQueryInner(ctx, type, payload, user, origin, tx));
+export async function executeQuery(ctx, type, payload, user, origin, tx, outerNotifySink) {
+    return runHandlerInstrumented(ctx, type, "query", user, () => executeQueryInner(ctx, type, payload, user, origin, tx, outerNotifySink));
 }
-async function executeQueryInner(ctx, type, payload, user, origin, tx) {
+async function executeQueryInner(ctx, type, payload, user, origin, tx, outerNotifySink) {
     const { registry } = ctx;
     const handler = registry.getQueryHandler(type);
     if (!handler)
@@ -54,11 +54,11 @@ async function executeQueryInner(ctx, type, payload, user, origin, tx) {
     // A resolved member (ctx.queryAsMember) runs in a Postgres READ ONLY transaction, not just the ctx surface below.
     return isMemberResolutionPrincipal(user)
         ? runInMemberReadOnlyTransaction(ctx, tx, (readOnlyTx) => runQueryHandler(ctx, type, handler, parsed.data, includeDeleted, user, origin, readOnlyTx))
-        : runQueryHandler(ctx, type, handler, parsed.data, includeDeleted, user, origin, tx);
+        : runQueryHandler(ctx, type, handler, parsed.data, includeDeleted, user, origin, tx, outerNotifySink);
 }
-async function runQueryHandler(ctx, type, handler, payload, includeDeleted, user, origin, tx) {
+async function runQueryHandler(ctx, type, handler, payload, includeDeleted, user, origin, tx, outerNotifySink) {
     const { registry } = ctx;
-    const handlerContext = await buildHandlerContext(ctx, type, user, origin, tx, undefined, includeDeleted);
+    const handlerContext = await buildHandlerContext(ctx, type, user, origin, tx, undefined, includeDeleted, undefined, outerNotifySink);
     let result = await handler.handler({ type, payload, user }, handlerContext);
     // postQuery-Hooks: fire BEFORE field-access-filter so hooks see raw data
     // and can merge custom-fields/computed-counts/tags/etc. Each hook is

package/dist/pipeline/dispatch-shared.d.ts

@@ -57,7 +57,7 @@ export declare function perCommandNestedDenied(targetType: string): InternalErro
 export declare function denyTransactionBoundSurfaces(handlerContext: HandlerContext): HandlerContext;
 export declare function applyBeforeTransactionReadOnly(handlerContext: HandlerContext): HandlerContext;
 export type HandlerTransactionScope = "inTransaction" | "withoutTransaction" | "postCommit";
-export declare function buildHandlerContext(ctx: DispatchContext, type: string, user: SessionUser, origin: WriteOrigin, tx?: DbTx, afterCommitHooks?: AfterCommitHook[], includeDeleted?: boolean, transactionScope?: HandlerTransactionScope): Promise<HandlerContext>;
+export declare function buildHandlerContext(ctx: DispatchContext, type: string, user: SessionUser, origin: WriteOrigin, tx?: DbTx, afterCommitHooks?: AfterCommitHook[], includeDeleted?: boolean, transactionScope?: HandlerTransactionScope, outerNotifySink?: AfterCommitHook[], readOnlyForTenant?: boolean): Promise<HandlerContext>;
 export declare function createHandlerBoundDigest(digest: (value: string) => string, handlerName: string): (value: string) => string;
 export declare function runHandlerInstrumented<T>(ctx: DispatchContext, type: string, operation: "query" | "write" | "stream", user: SessionUser, inner: () => Promise<T>): Promise<T>;
 export declare function runStreamInstrumented<T>(ctx: DispatchContext, type: string, user: SessionUser, inner: () => AsyncGenerator<T>): AsyncGenerator<T>;