@cosmicdrift/kumiko-framework 0.353.0 → 0.354.1

package/dist/jobs/job-runner.js

@@ -134,14 +134,14 @@ export function serializeJobSubject(subject) {
 // with the same subject values produce the same string and the later one
 // replaces the earlier record. Non-primitives would serialize into something
 // no caller can reconstruct for a lookup, so they fail the run loudly.
-function jobSubjectKey(jobName, payload, fields) {
+function jobSubjectKey(jobName, payload, fields, declaration) {
     if (fields === undefined || fields.length === 0)
         return null;
     const subject = {};
     for (const field of fields) {
         const value = payload[field] ?? null;
         if (value !== null && typeof value === "object") {
-            throw new Error(`Job "${jobName}": tenantVisibleFailure.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
+            throw new Error(`Job "${jobName}": ${declaration}.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
         }
         subject[field] = value;
     }
@@ -516,12 +516,72 @@ export function createJobRunner(options) {
         }
         return false;
     }
+    // Only err.message, never payload or the logs array: job payloads can carry PII.
+    function logFailedAttempt(entry) {
+        errorLogger.error("job failed", {
+            job: entry.job,
+            jobId: entry.jobId,
+            attempt: entry.attempt,
+            // BullMQ never retries an UnrecoverableError, whatever attempts remain.
+            final: entry.finalAttempt || entry.err instanceof UnrecoverableError,
+            error: entry.err instanceof Error ? entry.err.message : String(entry.err),
+        });
+    }
     async function reportPreRunFailure(jobName, bullJob, err) {
         const message = err instanceof Error ? err.message : String(err);
         await options.onJobFailed?.(jobName, bullJob.id ?? "unknown", message, [
             { level: "error", message, timestamp: Temporal.Now.instant() },
         ]);
     }
+    function handlerPayloadOf(data) {
+        const payload = {};
+        for (const [k, v] of Object.entries(data)) {
+            if (!k.startsWith("_"))
+                payload[k] = v;
+        }
+        return payload;
+    }
+    function tenantIdOfJobData(data) {
+        return (data["_tenantId"] ?? // @cast-boundary dynamic-key
+            data["tenantId"] ?? // @cast-boundary dynamic-key
+            SYSTEM_TENANT_ID);
+    }
+    // Tenant-visible queued state. A callback or subject problem must never fail
+    // the enqueue; a bad subject config surfaces as a failed run instead.
+    async function announceQueued(jobName, jobDef, bullJobId, data) {
+        const declaration = jobDef.tenantVisibleRun;
+        // skip: job did not opt in, no logger wired, or BullMQ returned no id
+        if (!declaration || !options.onJobQueued || bullJobId === undefined)
+            return;
+        const tenantId = tenantIdOfJobData(data);
+        // skip: tenant-less run — no tenant-scoped query could ever read it
+        if (tenantId === SYSTEM_TENANT_ID)
+            return;
+        try {
+            const subject = jobSubjectKey(jobName, handlerPayloadOf(data), declaration.subjectFields, "tenantVisibleRun");
+            await options.onJobQueued(jobName, bullJobId, { tenantId, subject });
+        }
+        catch (err) {
+            errorLogger.warn("tenant-visible queued state not recorded", {
+                job: jobName,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    async function announceDropped(jobName, jobDef, bullJobId) {
+        // skip: job did not opt in or no logger wired
+        if (!jobDef.tenantVisibleRun || !options.onJobDropped)
+            return;
+        try {
+            await options.onJobDropped(jobName, bullJobId);
+        }
+        catch (err) {
+            errorLogger.warn("tenant-visible queued state not cleared", {
+                job: jobName,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
     async function handleJob(bullJob) {
         const rawName = bullJob.name;
         // Handle perTenant dispatch jobs — fan out to one job per tenant. The
@@ -556,7 +616,8 @@ export function createJobRunner(options) {
                 // for that edge case instead: enqueue plainly, same as before.
                 const wrapperJobId = bullJob.id;
                 for (const tenantId of tenantIds) {
-                    await targetQueue.add(actualName, { ...bullJob.data, _tenantId: tenantId }, {
+                    const childData = { ...bullJob.data, _tenantId: tenantId };
+                    const child = await targetQueue.add(actualName, childData, {
                         ...buildRetryBullOpts(actualDef),
                         // Dedup over wrapper retries only holds as long as the children
                         // stay in Redis for the wrapper's whole retry window — see the
@@ -565,13 +626,22 @@ export function createJobRunner(options) {
                             ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
                             : {}),
                     });
+                    await announceQueued(actualName, actualDef, child.id, childData);
                 }
             }
             catch (err) {
                 // A retry may still succeed, and a succeeding wrapper reports no completion
                 // that could clear an earlier failure, so only the final attempt is reported.
                 const retries = allJobs.get(actualName)?.retries ?? 0;
-                if (bullJob.attemptsMade + 1 >= retries + 1) {
+                const finalAttempt = bullJob.attemptsMade + 1 >= retries + 1;
+                logFailedAttempt({
+                    job: rawName,
+                    jobId: bullJob.id ?? "unknown",
+                    attempt: bullJob.attemptsMade + 1,
+                    finalAttempt,
+                    err,
+                });
+                if (finalAttempt) {
                     await reportPreRunFailure(rawName, bullJob, err);
                 }
                 throw err;
@@ -583,6 +653,14 @@ export function createJobRunner(options) {
         const jobDef = allJobs.get(jobName);
         if (!jobDef) {
             const err = new Error(`Unknown job: ${jobName}`);
+            // Final regardless of attempts left: a retry cannot make the job exist.
+            logFailedAttempt({
+                job: jobName,
+                jobId: bullJob.id ?? "unknown",
+                attempt: bullJob.attemptsMade + 1,
+                finalAttempt: true,
+                err,
+            });
             await reportPreRunFailure(jobName, bullJob, err);
             throw err;
         }
@@ -605,10 +683,15 @@ export function createJobRunner(options) {
                 // Attempts already spent travel in _priorAttempts so a lock conflict
                 // on a retry cannot hand out a fresh budget and exceed `retries`.
                 const priorAttempts = priorAttemptsOf(bullJob.data) + bullJob.attemptsMade;
-                await queues[laneForJob(jobDef)].add(jobName, { ...bullJob.data, _priorAttempts: priorAttempts }, {
+                const requeuedData = { ...bullJob.data, _priorAttempts: priorAttempts };
+                const requeued = await queues[laneForJob(jobDef)].add(jobName, requeuedData, {
                     delay: SEQUENTIAL_RETRY_DELAY_MS,
                     ...buildRetryBullOpts(jobDef, priorAttempts),
                 });
+                if (bullJob.id !== undefined && bullJob.id !== requeued.id) {
+                    await announceDropped(jobName, jobDef, bullJob.id);
+                }
+                await announceQueued(jobName, jobDef, requeued.id, requeuedData);
                 // skip: lock taken, work re-enqueued with delay, current invocation done
                 return;
             }
@@ -629,27 +712,31 @@ export function createJobRunner(options) {
             attempt,
         };
         // Build handler payload (without internal meta fields)
-        const payload = {};
-        for (const [k, v] of Object.entries(rawData)) {
-            if (!k.startsWith("_"))
-                payload[k] = v;
-        }
+        const payload = handlerPayloadOf(rawData);
         // Determine tenantId and triggeredBy from meta
-        const tenantId = rawData["_tenantId"] ?? // @cast-boundary dynamic-key
-            payload["tenantId"] ?? // @cast-boundary dynamic-key
-            SYSTEM_TENANT_ID;
+        const tenantId = tenantIdOfJobData(rawData);
         const triggeredById = rawData["_triggeredById"] ?? null; // @cast-boundary dynamic-key
         // Tenant-visible failure record (JobDefinition.tenantVisibleFailure). The
         // subject is read from the handler payload, so it is resolved here where
         // the payload is built, not in the callbacks.
         const tenantVisibleDecl = jobDef.tenantVisibleFailure;
+        const tenantVisibleRunDecl = jobDef.tenantVisibleRun;
         // A bad subject config must surface as a failed run, so it is thrown from
         // inside runInSpan below instead of here, before any run record exists.
         let tenantVisibleSubject = null;
+        let tenantVisibleRunSubject = null;
         let subjectConfigError;
-        if (tenantVisibleDecl) {
+        if (tenantVisibleRunDecl) {
             try {
-                tenantVisibleSubject = jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields);
+                tenantVisibleRunSubject = jobSubjectKey(jobName, payload, tenantVisibleRunDecl.subjectFields, "tenantVisibleRun");
+            }
+            catch (err) {
+                subjectConfigError = err instanceof Error ? err : new Error(String(err));
+            }
+        }
+        if (tenantVisibleDecl && !subjectConfigError) {
+            try {
+                tenantVisibleSubject = jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields, "tenantVisibleFailure");
             }
             catch (err) {
                 subjectConfigError = err instanceof Error ? err : new Error(String(err));
@@ -658,13 +745,16 @@ export function createJobRunner(options) {
         // BullMQ stops retrying once attemptsMade reaches the configured attempts
         // (`retries + 1`), so this is the attempt whose failure is final.
         const finalAttempt = bullJob.retryable === false || attempt >= (jobDef.retries ?? 0) + 1;
-        const outcomeMeta = (messageKey) => ({
+        const outcomeMeta = (messageKey, unrecoverable = false) => ({
             tenantId,
-            finalAttempt,
+            // BullMQ never retries an UnrecoverableError, whatever attempts are left.
+            finalAttempt: finalAttempt || unrecoverable,
             ...(tenantVisibleDecl &&
                 !subjectConfigError && {
                 tenantVisible: { subject: tenantVisibleSubject, messageKey },
             }),
+            ...(tenantVisibleRunDecl &&
+                !subjectConfigError && { tenantVisibleRun: { subject: tenantVisibleRunSubject } }),
         });
         // Carry `_triggerName` from rawData when set — handleEvent injects it on
         // multi-trigger dispatch; exposed as jobContext.triggerName so handlers
@@ -693,7 +783,9 @@ export function createJobRunner(options) {
         // through the same _fileProviderResolver for jobs as for write-handlers,
         // otherwise event-triggered jobs silently get an unresolved ctx.files.
         const fileResolver = context._fileProviderResolver;
-        const files = fileResolver ? createFileContext(() => fileResolver(tenantId)) : context.files;
+        const files = fileResolver
+            ? createFileContext(() => fileResolver(tenantId), tenantId)
+            : context.files;
         const jobSystemUser = createSystemUser(tenantId);
         // Same buildHandlerContext parity as ctx.files above: ctx.notify/ctx.config
         // must resolve for jobs the same way they do for write-handlers, or an
@@ -813,6 +905,9 @@ export function createJobRunner(options) {
                 return memberReader(userId, qn, payload);
             },
         };
+        if (tenantVisibleRunDecl && !subjectConfigError) {
+            meta.tenantVisibleRun = { tenantId, subject: tenantVisibleRunSubject };
+        }
         await options.onJobStart?.(jobName, jobId, meta);
         // Cross-process trace continuation: if the enqueuing code captured a
         // parent span, start the job.execute span as its child. Works for event
@@ -865,7 +960,8 @@ export function createJobRunner(options) {
             catch (err) {
                 const errorMsg = err instanceof Error ? err.message : String(err);
                 logs.push({ level: "error", message: errorMsg, timestamp: Temporal.Now.instant() });
-                await options.onJobFailed?.(jobName, jobId, errorMsg, logs, outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl)));
+                logFailedAttempt({ job: jobName, jobId, attempt, finalAttempt, err });
+                await options.onJobFailed?.(jobName, jobId, errorMsg, logs, outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl), err instanceof UnrecoverableError));
                 throw err;
             }
         };
@@ -938,8 +1034,7 @@ export function createJobRunner(options) {
                 ]);
             }
             catch (err) {
-                // The worker is already consuming here; leaving its Redis connections
-                // open would keep the event loop alive and turn the aborted boot into
+                // Leaving the worker's Redis connections open would keep the event loop alive and turn the aborted boot into
                 // a hanging process instead of a non-zero exit.
                 await worker?.close();
                 worker = null;
@@ -965,6 +1060,9 @@ export function createJobRunner(options) {
             worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), handleJob, {
                 connection: redisOpts,
                 concurrency: 5,
+                // Not consuming until the boot gates passed: a job already queued from a
+                // previous run must not execute against an unmet boot precondition.
+                autorun: false,
             });
             worker.on("error", (err) => {
                 const log = stopping ? errorLogger.debug : errorLogger.error;
@@ -989,6 +1087,11 @@ export function createJobRunner(options) {
             // Ahead of cron/runOnBoot wiring on purpose: a failed gate must not
             // leave recurring schedulers behind.
             await runBootGates(consumerLane);
+            // Floating on purpose: run() resolves only when the worker closes.
+            void worker.run().catch((err) => {
+                const log = stopping ? errorLogger.debug : errorLogger.error;
+                log("worker run loop failed", { error: err instanceof Error ? err.message : String(err) });
+            });
             // Only schedule cron + boot for jobs that belong to this lane. Jobs
             // assigned to the other lane get their cron/boot wiring from the
             // runner running on that lane. Running both here would double-fire.
@@ -1142,6 +1245,7 @@ export function createJobRunner(options) {
                     for (const j of waiting) {
                         if (j.name === jobName && j.id) {
                             await j.remove();
+                            await announceDropped(jobName, jobDef, j.id);
                         }
                     }
                     break;
@@ -1182,6 +1286,7 @@ export function createJobRunner(options) {
                 data["_correlationId"] = reqCtx.correlationId;
             stampGatedWriteOrigin(data);
             const job = await targetQueue.add(jobName, data, bullOpts);
+            await announceQueued(jobName, jobDef, job.id, data);
             return job.id ?? "unknown";
         },
         async handleEvent(eventName, payload, user, triggerEvent) {
@@ -1230,7 +1335,8 @@ export function createJobRunner(options) {
                 stampGatedWriteOrigin(data);
                 // Route to the job's declared lane, not a fixed queue — that's
                 // the whole reason both queues are held.
-                await queues[laneForJob(jobDef)].add(name, data, buildRetryBullOpts(jobDef));
+                const job = await queues[laneForJob(jobDef)].add(name, data, buildRetryBullOpts(jobDef));
+                await announceQueued(name, jobDef, job.id, data);
             }
         },
         attachDispatcher(ref) {

package/dist/migrations/pii-event-backfill.js

@@ -14,6 +14,7 @@ import { bigint, instant, table as pgTable, sql, text } from "../db/dialect.js";
 import { extractTableName } from "../db/index.js";
 import { acquireNamespacedAdvisoryLock } from "../db/queries/advisory-lock.js";
 import { backfillEventPiiEncryptionBatch, } from "../db/queries/backfill-pii.js";
+import { alterTableAddColumn } from "../db/queries/ddl.js";
 import { persistPiiBackfillState, readPiiBackfillState, } from "../db/queries/pii-backfill-state.js";
 import { upsertOnConflict } from "../db/query.js";
 import { tableExists } from "../db/schema-inspection.js";
@@ -37,10 +38,12 @@ export const piiBackfillStateTable = pgTable("kumiko_pii_backfill_state", {
     updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql `now()`),
 });
 export async function createPiiBackfillStateTable(db) {
-    // skip: table already exists — boot runs on every replica
-    if (await tableExists(db, "public.kumiko_pii_backfill_state"))
-        return;
-    await unsafePushTables(db, { kumikoPiiBackfillState: piiBackfillStateTable });
+    if (!(await tableExists(db, "public.kumiko_pii_backfill_state"))) {
+        await unsafePushTables(db, { kumikoPiiBackfillState: piiBackfillStateTable });
+    }
+    // Added after the table's first deploy; IF NOT EXISTS keeps concurrent replica boots safe.
+    await alterTableAddColumn(db, "kumiko_pii_backfill_state", "failed_event_ids", "bigint[]", " DEFAULT '{}'::bigint[]", " NOT NULL", 
+    /* ifNotExists */ true);
 }
 // Replicas boot concurrently: two of them can both pass the tableExists gate and
 // one CREATE then fails. The table existing afterwards means the other won.
@@ -107,17 +110,19 @@ function initialScanCursor(state) {
 // One transaction per batch: the advisory lock serializes pods, and the payload
 // is re-read inside it so a value is never encrypted twice and two backfills
 // never mint the same DEK. Returns null when another pod already finished a full run.
-async function runOneBatch(ctx, progress) {
+async function runOneBatch(ctx, progress, retryEventIds) {
     return ctx.db.begin(async (tx) => {
         await acquireNamespacedAdvisoryLock(tx, PII_EVENT_BACKFILL_LOCK_NAMESPACE, PII_EVENT_BACKFILL_LOCK_KEY);
         const state = await readPiiBackfillState(tx, ctx.fingerprint);
-        if (ctx.mode === "full" && state?.completed === true)
+        if (retryEventIds === undefined && ctx.mode === "full" && state?.completed === true) {
             return null;
+        }
         const startCursor = ctx.mode === "full" && state
             ? maxBigInt(progress.scanCursor, BigInt(state.cursor))
             : progress.scanCursor;
         const batch = await backfillEventPiiEncryptionBatch(tx, ctx.registry, {
             afterEventId: startCursor,
+            ...(retryEventIds && { eventIds: retryEventIds }),
             batchSize: ctx.batchSize,
             resolveOwnerFromProjection: true,
             scanCache: progress.scanCache,
@@ -128,10 +133,15 @@ async function runOneBatch(ctx, progress) {
             : [];
         if (tables.length > 0)
             await queueRebuildTables(tx, tables, ctx.migrationId);
-        if (!progress.failureSeen) {
-            const completesRun = batch.scannedAll && batch.failures.length === 0;
-            await persistPiiBackfillState(tx, ctx.fingerprint, persistableCursor(batch, startCursor), completesRun);
-        }
+        // The cursor moves past failed events: they are recorded here and retried on their own,
+        // so one permanently failing event no longer forces a rescan from it on every boot.
+        const failedIds = batch.failures.map((failure) => failure.eventId);
+        await persistPiiBackfillState(tx, ctx.fingerprint, {
+            cursor: retryEventIds ? 0n : advancedCursor(batch, startCursor),
+            markCompleted: retryEventIds === undefined && batch.scannedAll,
+            addFailedEventIds: failedIds,
+            removeFailedEventIds: retryEventIds?.filter((id) => !failedIds.includes(id)) ?? [],
+        });
         return { batch, tables };
     });
 }
@@ -141,14 +151,25 @@ function recordBatch(progress, step) {
     progress.totals.updatedEvents += batch.updatedEvents;
     progress.totals.encryptedFields += batch.encryptedFields;
     progress.totals.erasedFields += batch.erasedFields;
-    progress.failures.push(...batch.failures);
+    for (const failure of batch.failures) {
+        // The catch-up overlap and the per-event retry can hit the same failed event twice in one run.
+        if (progress.failedIdsSeen.has(failure.eventId))
+            continue;
+        progress.failedIdsSeen.add(failure.eventId);
+        progress.failures.push(failure);
+    }
     for (const table of step.tables)
         progress.queuedTables.add(table);
-    if (batch.failures.length > 0)
-        progress.failureSeen = true;
 }
 // Returns true when the run was aborted between batches.
-async function scanAllBatches(ctx, progress, signal) {
+async function scanAllBatches(ctx, progress, signal, retryEventIds) {
+    for (let from = 0; from < retryEventIds.length; from += ctx.batchSize) {
+        if (signal?.aborted)
+            return true;
+        const step = await runOneBatch(ctx, progress, retryEventIds.slice(from, from + ctx.batchSize));
+        if (step !== null)
+            recordBatch(progress, step);
+    }
     for (;;) {
         if (signal?.aborted)
             return true;
@@ -163,7 +184,9 @@ async function scanAllBatches(ctx, progress, signal) {
 }
 function reportRunOutcome(log, progress) {
     if (progress.failures.length > 0) {
-        log.error(`${progress.failures.length} event(s) could not be PII-encrypted and will be retried on the next boot`, {
+        log.error(`${progress.failures.length} event(s) could not be PII-encrypted; the scan moved past them and ` +
+            "only these events are retried on each boot. Fix the cause, or erase subjects that stay unresolvable via " +
+            "backfillEventPiiEncryption(..., { eraseUnresolvableSubjects: true })", {
             failureCount: progress.failures.length,
             eventIds: progress.failures.slice(0, MAX_LOGGED_FAILURE_IDS).map((f) => f.eventId),
         });
@@ -180,24 +203,27 @@ export async function runPiiEventBackfill(db, registry, options = {}) {
     const fingerprint = piiAnnotationFingerprint(registry);
     await ensureInfraTable(db, "kumiko_pii_backfill_state", createPiiBackfillStateTable);
     await ensureInfraTable(db, "kumiko_pending_rebuilds", createPendingRebuildsTable);
-    const { mode, scanCursor } = initialScanCursor(await readPiiBackfillState(db, fingerprint));
+    const initialState = await readPiiBackfillState(db, fingerprint);
+    const { mode, scanCursor } = initialScanCursor(initialState);
     const ctx = {
         db,
         registry,
         fingerprint,
-        migrationId: `${PII_BACKFILL_MIGRATION_PREFIX}${fingerprint.slice(0, 12)}`,
+        // The per-run suffix keeps a peer replica's re-queue of the same table (new
+        // migration_id via the upsert) from being deleted by this run's clear.
+        migrationId: `${PII_BACKFILL_MIGRATION_PREFIX}${fingerprint.slice(0, 12)}:${crypto.randomUUID()}`,
         mode,
         batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE,
     };
     const progress = {
         scanCursor,
-        failureSeen: false,
         scanCache: {},
         totals: { scannedEvents: 0, updatedEvents: 0, encryptedFields: 0, erasedFields: 0 },
         failures: [],
         queuedTables: new Set(),
+        failedIdsSeen: new Set(),
     };
-    const aborted = await scanAllBatches(ctx, progress, options.signal);
+    const aborted = await scanAllBatches(ctx, progress, options.signal, initialState?.failedEventIds ?? []);
     reportRunOutcome(log, progress);
     const rebuild = aborted ? null : await rebuildQueuedTables(db, registry, options.signal, log);
     const queuedTables = [...progress.queuedTables];
@@ -218,12 +244,7 @@ export async function runPiiEventBackfill(db, registry, options = {}) {
         rebuild,
     };
 }
-// Never advance past a failed event so the next boot retries it.
-function persistableCursor(batch, startCursor) {
-    if (batch.firstFailedEventId !== null) {
-        const beforeFailure = BigInt(batch.firstFailedEventId) - 1n;
-        return beforeFailure > startCursor ? beforeFailure : startCursor;
-    }
+function advancedCursor(batch, startCursor) {
     return batch.lastEventId === null ? startCursor : BigInt(batch.lastEventId);
 }
 function maxBigInt(a, b) {

package/dist/observability/index.d.ts

@@ -12,5 +12,5 @@ export { type MetricEvent, type MetricEventHandler, RecordingMeter, } from "./re
 export { type RecordedSpan, RecordingTracer, type RecordingTracerOptions, serializeSpanContext, } from "./recording-tracer.js";
 export { wrapRedisClient } from "./redis-wrapper.js";
 export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
-export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
+export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerPendingGaps, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
 export type { Counter, Gauge, Histogram, Meter, MetricDefinition, MetricLabels, MetricsHandle, MetricType, ObservabilityOptions, ObservabilityProvider, SamplingConfig, SensitiveFilterConfig, SerializedTraceContext, Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatus, StartSpanOptions, Tracer, } from "./types/index.js";

package/dist/observability/index.js

@@ -13,4 +13,4 @@ export { RecordingMeter, } from "./recording-meter.js";
 export { RecordingTracer, serializeSpanContext, } from "./recording-tracer.js";
 export { wrapRedisClient } from "./redis-wrapper.js";
 export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
-export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
+export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerPendingGaps, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";

package/dist/observability/standard-metrics.d.ts

@@ -28,6 +28,10 @@ export declare function emitEventConsumerLag(meter: Meter, labels: {
     readonly consumer: string;
     readonly instanceId: string;
 }, lagEvents: number): void;
+export declare function emitEventConsumerPendingGaps(meter: Meter, labels: {
+    readonly consumer: string;
+    readonly instanceId: string;
+}, gapRanges: number): void;
 export declare function emitEventConsumerRearmExhausted(meter: Meter, labels: {
     readonly consumer: string;
     readonly instanceId: string;

package/dist/observability/standard-metrics.js

@@ -61,6 +61,12 @@ export const STANDARD_METRIC_DEFS = [
     // per-instance consumers (one cursor per dispatcher). Without this,
     // Prometheus collapses per-instance shards into last-writer-wins and
     // per-instance lag is invisible to alerting.
+    {
+        name: "kumiko_event_consumer_pending_gaps",
+        type: "gauge",
+        description: "Number of unresolved id ranges below the consumer's cursor (ids not yet visible or proven rolled back).",
+        labels: ["consumer", "instance_id"],
+    },
     {
         name: "kumiko_event_consumer_lag_events",
         type: "gauge",
@@ -212,6 +218,12 @@ export function emitEventConsumerLag(meter, labels, lagEvents) {
         instance_id: labels.instanceId,
     });
 }
+export function emitEventConsumerPendingGaps(meter, labels, gapRanges) {
+    meter.gauge("kumiko_event_consumer_pending_gaps").set(gapRanges, {
+        consumer: labels.consumer,
+        instance_id: labels.instanceId,
+    });
+}
 // Caller (event-dispatcher.ts) is responsible for calling this exactly
 // once per (consumer, instanceId) transition into exhausted-dead — not
 // once per poll pass while it stays dead.

package/dist/pipeline/dispatch-shared.d.ts

@@ -46,7 +46,9 @@ export type DispatchContext = {
     tracer: ReturnType<typeof getFallbackTracer>;
     meter: ReturnType<typeof getFallbackMeter>;
     membershipQuery: string;
+    memberReadTimeoutMs: number;
 };
+export declare const DEFAULT_MEMBER_READ_TIMEOUT_MS = 10000;
 export declare function resolveDbSource(ctx: DispatchContext, tx: DbTx | undefined): DbConnection | DbTx | undefined;
 export declare function isMemberResolutionPrincipal(user: SessionUser): boolean;
 export declare function applyMemberResolutionReadOnly(handlerContext: HandlerContext): HandlerContext;