@cosmicdrift/kumiko-framework 0.334.0 → 0.336.0

package/dist/jobs/job-runner.js

@@ -1,4 +1,4 @@
-import { Queue, Worker } from "bullmq";
+import { Queue, UnrecoverableError, Worker } from "bullmq";
 import { Redis } from "ioredis";
 import { requestContext } from "../api/request-context.js";
 import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
@@ -10,8 +10,7 @@ import { InternalError } from "../errors/index.js";
 import { isKumikoError } from "../errors/kumiko-error.js";
 import { createFileContext } from "../files/file-handle.js";
 import { createFallbackLogger } from "../logging/index.js";
-import { createEscapeHatchReporter } from "../observability/escape-hatch-report.js";
-import { emitJobLastSuccess, emitJobQueueDepth, getFallbackTracer, } from "../observability/index.js";
+import { createEscapeHatchReporter, emitJobLastSuccess, emitJobQueueDepth, getFallbackTracer, registerStandardMetrics, } from "../observability/index.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
 import { RedisKeys } from "../pipeline/redis-keys.js";
 import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
@@ -122,6 +121,14 @@ function createJobLogger(logs, baseContext = {}) {
     };
     return logger;
 }
+// Single source of the stored subject string: the writer (jobSubjectKey) and
+// the tenant failures query both go through it. Matching is exact — field names
+// and JSON types must equal what the writer stored (`"1"` is not `1`).
+export function serializeJobSubject(subject) {
+    return JSON.stringify(Object.keys(subject)
+        .sort()
+        .map((field) => [field, subject[field] ?? null]));
+}
 // Stable identity for the declared payload fields: sorted keys, so two runs
 // with the same subject values produce the same string and the later one
 // replaces the earlier record. Non-primitives would serialize into something
@@ -129,15 +136,31 @@ function createJobLogger(logs, baseContext = {}) {
 function jobSubjectKey(jobName, payload, fields) {
     if (fields === undefined || fields.length === 0)
         return null;
-    const entries = [];
-    for (const field of [...fields].sort()) {
+    const subject = {};
+    for (const field of fields) {
         const value = payload[field] ?? null;
         if (value !== null && typeof value === "object") {
             throw new Error(`Job "${jobName}": tenantVisibleFailure.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
         }
-        entries.push([field, value]);
+        subject[field] = value;
     }
-    return JSON.stringify(entries);
+    return serializeJobSubject(subject);
+}
+// Jobs that never touch ctx.db run fine on a runner built without one; a job
+// that does gets a named error instead of a bare TypeError on undefined.
+function missingDbProxy(jobName) {
+    // @cast-boundary db-operator — every access throws, so no TenantDb member is ever returned
+    return new Proxy({}, {
+        get() {
+            throw new InternalError({
+                message: `[jobs] job runner constructed without a db connection — ctx.db is unavailable for "${jobName}"`,
+            });
+        },
+    });
+}
+function priorAttemptsOf(data) {
+    const value = data["_priorAttempts"];
+    return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
 }
 // Only a translation key ever travels to the tenant: the thrown error's own
 // i18nKey when it carries one, otherwise the key declared at the job. The
@@ -202,6 +225,12 @@ function parseRedisOpts(url) {
 // redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
 // would otherwise hang start() forever with no health endpoint to notice.
 const BOOT_REDIS_TIMEOUT_MS = 10_000;
+// Shutdown must fit a k8s grace period even when Redis is down, so the
+// readiness wait in stop() is capped well below the boot timeout.
+const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
+// A gate without an explicit `timeout` may legitimately run long (migration-style
+// checks), so it is never rejected; a hang just must not stay silent.
+const BOOT_GATE_SLOW_WARN_MS = 60_000;
 function timeoutReject(ms, message) {
     let timer;
     const promise = new Promise((_, reject) => {
@@ -224,10 +253,10 @@ const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
 // perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
 // set must retry the same way regardless of how it got enqueued, or it
 // fails for good on the very first error on whichever path skips this.
-function buildRetryBullOpts(jobDef) {
+function buildRetryBullOpts(jobDef, priorAttempts = 0) {
     const opts = {};
     if (jobDef.retries !== undefined)
-        opts.attempts = jobDef.retries + 1;
+        opts.attempts = Math.max(1, jobDef.retries + 1 - priorAttempts);
     if (jobDef.backoff) {
         opts.backoff =
             typeof jobDef.backoff === "string"
@@ -282,6 +311,10 @@ export function createJobRunner(options) {
     // boot); otherwise noop so dispatch/handleJob stay zero-cost without config.
     const tracer = context.tracer ?? getFallbackTracer();
     const errorLogger = createFallbackLogger("job-runner", context.log);
+    // The last-success gauge throws if its metric was never registered; the runner
+    // must not depend on buildServer having done it first. Idempotent.
+    if (context.meter)
+        registerStandardMetrics(context.meter);
     // Set at the top of stop() — a graceful shutdown closes the redis/BullMQ
     // clients itself, which fires the exact same 'error' listeners below with
     // an expected "Connection is closed." Downgrading to debug once stopping
@@ -318,7 +351,12 @@ export function createJobRunner(options) {
                 registry,
                 ...bridgeStub(),
             });
-            // @cast-boundary engine-payload — generic query-handler return for typed convention
+            if (!Array.isArray(result) || result.some((id) => typeof id !== "string")) {
+                throw new InternalError({
+                    message: `${ACTIVE_TENANT_IDS_QUERY_NAME} must return string[] of tenant ids`,
+                });
+            }
+            // @cast-boundary engine-payload — checked to be string[] above; TenantId is a brand
             return result;
         };
     }
@@ -455,6 +493,12 @@ export function createJobRunner(options) {
         }
         return false;
     }
+    async function reportPreRunFailure(jobName, bullJob, err) {
+        const message = err instanceof Error ? err.message : String(err);
+        await options.onJobFailed?.(jobName, bullJob.id ?? "unknown", message, [
+            { level: "error", message, timestamp: Temporal.Now.instant() },
+        ]);
+    }
     async function handleJob(bullJob) {
         const rawName = bullJob.name;
         // Handle perTenant dispatch jobs — fan out to one job per tenant. The
@@ -463,34 +507,51 @@ export function createJobRunner(options) {
         // (it's picked up by this runner's own worker).
         if (rawName.startsWith("_perTenant:")) {
             const actualName = rawName.slice("_perTenant:".length);
-            if (!getActiveTenantIds) {
-                throw new Error(`perTenant job "${actualName}" requires either options.getActiveTenantIds or the ` +
-                    `tenant feature mounted (it registers "${ACTIVE_TENANT_IDS_QUERY_NAME}", which the ` +
-                    "framework uses to resolve active tenants on its own)");
-            }
-            const actualDef = allJobs.get(actualName);
-            if (!actualDef) {
-                throw new Error(`Unknown job: ${actualName}`);
+            // These throws happen before runInSpan, so its onJobFailed never sees them
+            // and a drained test stack would stay green over a fan-out that never ran.
+            try {
+                if (!getActiveTenantIds) {
+                    if (registry.getQueryHandler(ACTIVE_TENANT_IDS_QUERY_NAME)) {
+                        throw new Error(`perTenant job "${actualName}": the tenant feature is mounted but the job runner has no ` +
+                            "`context.db`: pass a DbConnection in the runner context or supply options.getActiveTenantIds");
+                    }
+                    throw new Error(`perTenant job "${actualName}" requires either options.getActiveTenantIds or the ` +
+                        `tenant feature mounted (it registers "${ACTIVE_TENANT_IDS_QUERY_NAME}", which the ` +
+                        "framework uses to resolve active tenants on its own)");
+                }
+                const actualDef = allJobs.get(actualName);
+                if (!actualDef) {
+                    throw new Error(`Unknown job: ${actualName}`);
+                }
+                const tenantIds = await getActiveTenantIds();
+                const targetQueue = queues[laneForJob(actualDef)];
+                // wrapperJobId is only absent for a Job BullMQ somehow never assigned
+                // an id to (not observed in practice) — falling back to a constant
+                // there would collide every such run's children onto the same ids
+                // and silently swallow later runs as "duplicates", which is worse
+                // than the duplicate this is meant to prevent. Skip id-based dedup
+                // for that edge case instead: enqueue plainly, same as before.
+                const wrapperJobId = bullJob.id;
+                for (const tenantId of tenantIds) {
+                    await targetQueue.add(actualName, { ...bullJob.data, _tenantId: tenantId }, {
+                        ...buildRetryBullOpts(actualDef),
+                        // Dedup over wrapper retries only holds as long as the children
+                        // stay in Redis for the wrapper's whole retry window — see the
+                        // COMPLETED_JOB_RETENTION_AGE_SEC invariant check above.
+                        ...(wrapperJobId !== undefined
+                            ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
+                            : {}),
+                    });
+                }
             }
-            const tenantIds = await getActiveTenantIds();
-            const targetQueue = queues[laneForJob(actualDef)];
-            // wrapperJobId is only absent for a Job BullMQ somehow never assigned
-            // an id to (not observed in practice) — falling back to a constant
-            // there would collide every such run's children onto the same ids
-            // and silently swallow later runs as "duplicates", which is worse
-            // than the duplicate this is meant to prevent. Skip id-based dedup
-            // for that edge case instead: enqueue plainly, same as before.
-            const wrapperJobId = bullJob.id;
-            for (const tenantId of tenantIds) {
-                await targetQueue.add(actualName, { ...bullJob.data, _tenantId: tenantId }, {
-                    ...buildRetryBullOpts(actualDef),
-                    // Dedup over wrapper retries only holds as long as the children
-                    // stay in Redis for the wrapper's whole retry window — see the
-                    // COMPLETED_JOB_RETENTION_AGE_SEC invariant check above.
-                    ...(wrapperJobId !== undefined
-                        ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
-                        : {}),
-                });
+            catch (err) {
+                // A retry may still succeed, and a succeeding wrapper reports no completion
+                // that could clear an earlier failure, so only the final attempt is reported.
+                const retries = allJobs.get(actualName)?.retries ?? 0;
+                if (bullJob.attemptsMade + 1 >= retries + 1) {
+                    await reportPreRunFailure(rawName, bullJob, err);
+                }
+                throw err;
             }
             // skip: fan-out dispatcher job, per-tenant children enqueued
             return;
@@ -498,7 +559,9 @@ export function createJobRunner(options) {
         const jobName = rawName;
         const jobDef = allJobs.get(jobName);
         if (!jobDef) {
-            throw new Error(`Unknown job: ${jobName}`);
+            const err = new Error(`Unknown job: ${jobName}`);
+            await reportPreRunFailure(jobName, bullJob, err);
+            throw err;
         }
         // Sequential gate: try to claim the per-name lock. If another worker
         // (or this worker on a different bullJob) holds it, re-enqueue with a
@@ -516,13 +579,12 @@ export function createJobRunner(options) {
                 // same queue the worker just picked from (since only the consuming
                 // lane runs handleJob at all), but route explicitly — no implicit
                 // coupling to "whichever queue the caller happened to be on".
-                // The re-enqueued job starts with a full retry budget; a remaining
-                // budget is deliberately not carried over, since finalAttempt/
-                // tenantVisibleFailure are computed from jobDef.retries against
-                // attemptsMade, not against some inherited remainder.
-                await queues[laneForJob(jobDef)].add(jobName, bullJob.data, {
+                // Attempts already spent travel in _priorAttempts so a lock conflict
+                // on a retry cannot hand out a fresh budget and exceed `retries`.
+                const priorAttempts = priorAttemptsOf(bullJob.data) + bullJob.attemptsMade;
+                await queues[laneForJob(jobDef)].add(jobName, { ...bullJob.data, _priorAttempts: priorAttempts }, {
                     delay: SEQUENTIAL_RETRY_DELAY_MS,
-                    ...buildRetryBullOpts(jobDef),
+                    ...buildRetryBullOpts(jobDef, priorAttempts),
                 });
                 // skip: lock taken, work re-enqueued with delay, current invocation done
                 return;
@@ -537,7 +599,7 @@ export function createJobRunner(options) {
         // retry number, so audit queries distinguish fresh from retry runs
         // without peeking at BullMQ internals.
         const rawData = bullJob.data;
-        const attempt = bullJob.attemptsMade + 1;
+        const attempt = bullJob.attemptsMade + 1 + priorAttemptsOf(rawData);
         const meta = {
             triggeredById: rawData["_triggeredById"], // @cast-boundary dynamic-key
             payload: rawData["_payload"], // @cast-boundary dynamic-key
@@ -558,16 +620,26 @@ export function createJobRunner(options) {
         // subject is read from the handler payload, so it is resolved here where
         // the payload is built, not in the callbacks.
         const tenantVisibleDecl = jobDef.tenantVisibleFailure;
-        const tenantVisibleSubject = tenantVisibleDecl
-            ? jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields)
-            : null;
+        // A bad subject config must surface as a failed run, so it is thrown from
+        // inside runInSpan below instead of here, before any run record exists.
+        let tenantVisibleSubject = null;
+        let subjectConfigError;
+        if (tenantVisibleDecl) {
+            try {
+                tenantVisibleSubject = jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields);
+            }
+            catch (err) {
+                subjectConfigError = err instanceof Error ? err : new Error(String(err));
+            }
+        }
         // BullMQ stops retrying once attemptsMade reaches the configured attempts
         // (`retries + 1`), so this is the attempt whose failure is final.
-        const finalAttempt = attempt >= (jobDef.retries ?? 0) + 1;
+        const finalAttempt = bullJob.retryable === false || attempt >= (jobDef.retries ?? 0) + 1;
         const outcomeMeta = (messageKey) => ({
             tenantId,
             finalAttempt,
-            ...(tenantVisibleDecl && {
+            ...(tenantVisibleDecl &&
+                !subjectConfigError && {
                 tenantVisible: { subject: tenantVisibleSubject, messageKey },
             }),
         });
@@ -658,8 +730,7 @@ export function createJobRunner(options) {
         let memberReader;
         const jobContext = {
             ...context,
-            // Undefined only for a runner built without a db; JobContext requires db non-optional.
-            db: jobDb, // @cast-boundary db-operator
+            db: jobDb ?? missingDbProxy(jobName),
             files,
             derivatives,
             ...(notify !== undefined && { notify }),
@@ -721,6 +792,10 @@ export function createJobRunner(options) {
         const jobCorrelationId = inheritedCorrelationId ?? jobRequestId;
         const runInSpan = async () => {
             try {
+                if (subjectConfigError) {
+                    // Retrying cannot fix a declaration error.
+                    throw new UnrecoverableError(subjectConfigError.message);
+                }
                 if (writeOriginInvalid) {
                     throw new InternalError({
                         message: `Job "${jobName}" received an unparseable _writeOrigin — refusing to run without ` +
@@ -732,13 +807,22 @@ export function createJobRunner(options) {
                     correlationId: jobCorrelationId,
                     // #3043 — events a job writes carry the job as their origin.
                     handler: jobName,
-                    feature: qnScope(jobName),
+                    feature: registry.getJobFeature(jobName) ?? qnScope(jobName),
                     writeOrigin: jobOrigin,
                 }, () => jobDef.handler(payload, jobContext));
                 // Stamped before the observer hook: a throwing onJobComplete must not
                 // make a run that actually succeeded look dead to the liveness alert.
-                if (context.meter)
-                    emitJobLastSuccess(context.meter, jobName);
+                if (context.meter) {
+                    try {
+                        emitJobLastSuccess(context.meter, jobName);
+                    }
+                    catch (err) {
+                        errorLogger.warn("job last-success metric failed", {
+                            job: jobName,
+                            error: err instanceof Error ? err.message : String(err),
+                        });
+                    }
+                }
                 const duration = Date.now() - startTime;
                 await options.onJobComplete?.(jobName, jobId, duration, logs, outcomeMeta(null));
             }
@@ -794,8 +878,28 @@ export function createJobRunner(options) {
                 continue;
             if (!jobDef.bootGate)
                 continue;
+            const gateTimeout = jobDef.timeout !== undefined
+                ? timeoutReject(jobDef.timeout, `job-runner: boot gate "${name}" did not finish within ${jobDef.timeout}ms`)
+                : undefined;
+            const slowWarnTimer = gateTimeout === undefined
+                ? setTimeout(() => {
+                    errorLogger.warn("boot gate still running, no timeout configured", {
+                        gate: name,
+                        afterMs: BOOT_GATE_SLOW_WARN_MS,
+                    });
+                }, BOOT_GATE_SLOW_WARN_MS)
+                : undefined;
             try {
-                await handleJob({ id: bootJobIdForJobName(name), name, data: {}, attemptsMade: 0 });
+                await Promise.race([
+                    handleJob({
+                        id: bootJobIdForJobName(name),
+                        name,
+                        data: {},
+                        attemptsMade: 0,
+                        retryable: false,
+                    }),
+                    ...(gateTimeout ? [gateTimeout.promise] : []),
+                ]);
             }
             catch (err) {
                 // The worker is already consuming here; leaving its Redis connections
@@ -805,6 +909,11 @@ export function createJobRunner(options) {
                 worker = null;
                 throw err;
             }
+            finally {
+                gateTimeout?.cancel();
+                if (slowWarnTimer !== undefined)
+                    clearTimeout(slowWarnTimer);
+            }
         }
     }
     const runnerApi = {
@@ -925,15 +1034,19 @@ export function createJobRunner(options) {
             // rejects the in-flight INFO with "Connection is closed." on an emitter
             // without listeners -> unhandled rejection (fw#1805). Wait for readiness
             // first; a failed/timed-out wait must not block the close itself.
-            const readinessTimeout = timeoutReject(bootRedisTimeoutMs, `job-runner: Queue not ready within ${bootRedisTimeoutMs}ms during stop()`);
+            const stopReadyTimeoutMs = Math.min(bootRedisTimeoutMs, STOP_QUEUE_READY_TIMEOUT_MS);
+            const readinessTimeout = timeoutReject(stopReadyTimeoutMs, `job-runner: Queue not ready within ${stopReadyTimeoutMs}ms during stop()`);
             try {
                 await Promise.race([
                     Promise.all([queues.api.waitUntilReady(), queues.worker.waitUntilReady()]),
                     readinessTimeout.promise,
                 ]);
             }
-            catch {
-                // close() below still has to run
+            catch (err) {
+                // close() below still has to run; stopping is already true, so debug matches the other shutdown logs
+                errorLogger.debug("queue readiness wait failed during stop()", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
             }
             finally {
                 readinessTimeout.cancel();

package/dist/migrations/index.d.ts

@@ -1,4 +1,4 @@
 export { assertKumikoSchemaCurrent, type ChecksumMismatch, detectKumikoDrift, formatKumikoDriftReport, type KumikoDriftReport, SchemaDriftError, } from "./kumiko-drift.js";
-export { createPendingRebuildsTable, type EnqueueProjectionRebuildDeps, type EnqueueProjectionRebuildResult, enqueueProjectionRebuild, listPendingRebuilds, type PendingRebuildRun, PROJECTION_REBUILD_JOB, pendingRebuildsTable, queueRebuildsFromMarkers, type RunPendingRebuildsOptions, rebuildProjectionOrMultiStream, runPendingRebuilds, } from "./pending-rebuilds.js";
+export { createPendingRebuildsTable, type EnqueueProjectionRebuildDeps, type EnqueueProjectionRebuildResult, enqueueProjectionRebuild, listPendingRebuilds, type PendingRebuildRun, PROJECTION_REBUILD_JOB, pendingRebuildsTable, queueRebuildsFromMarkers, type RunPendingRebuildsOptions, rebuildProjectionOrMultiStream, reportPendingRebuildRun, runPendingRebuilds, } from "./pending-rebuilds.js";
 export { createPiiBackfillStateTable, type PiiEventBackfillOptions, type PiiEventBackfillResult, type PiiRebuildRun, piiAnnotationFingerprint, piiBackfillStateTable, runPiiEventBackfill, } from "./pii-event-backfill.js";
 export { buildProjectionTableIndex } from "./projection-table-index.js";

package/dist/migrations/index.js

@@ -1,7 +1,7 @@
 // Drizzle-free gate (kumiko/migrations system) — the canonical boot-gate.
 export { assertKumikoSchemaCurrent, detectKumikoDrift, formatKumikoDriftReport, SchemaDriftError, } from "./kumiko-drift.js";
 // Persistente Pending-Rebuild-Queue (survives Rebuild-Failures + Crashes).
-export { createPendingRebuildsTable, enqueueProjectionRebuild, listPendingRebuilds, PROJECTION_REBUILD_JOB, pendingRebuildsTable, queueRebuildsFromMarkers, rebuildProjectionOrMultiStream, runPendingRebuilds, } from "./pending-rebuilds.js";
+export { createPendingRebuildsTable, enqueueProjectionRebuild, listPendingRebuilds, PROJECTION_REBUILD_JOB, pendingRebuildsTable, queueRebuildsFromMarkers, rebuildProjectionOrMultiStream, reportPendingRebuildRun, runPendingRebuilds, } from "./pending-rebuilds.js";
 // Boot-time backfill: re-encrypts plaintext events after a PII annotation was added.
 export { createPiiBackfillStateTable, piiAnnotationFingerprint, piiBackfillStateTable, runPiiEventBackfill, } from "./pii-event-backfill.js";
 // tableName → projection-name, für den app-seitigen Projection-Rebuild.

package/dist/migrations/pending-rebuilds.d.ts

@@ -57,6 +57,10 @@ export declare function rebuildProjectionOrMultiStream(name: string, deps: {
         readonly skipApplyErrors?: boolean;
     };
 }): Promise<RebuildResult>;
+/** Logs a finished queue drain and throws when any projection failed; the
+ *  failed tables stay queued. Shared by every `schema apply` entry point so the
+ *  report and the error text cannot drift apart. */
+export declare function reportPendingRebuildRun(run: PendingRebuildRun, log: (line: string) => void): void;
 /** Arbeitet die persistierte Queue ab: mappt Tabellen auf Projektionen,
  *  rebuildet jede betroffene Projektion und räumt ihre Tabellen erst nach
  *  ERFOLG aus der Queue. Fehlgeschlagene bleiben pending — der nächste

package/dist/migrations/pending-rebuilds.js

@@ -71,7 +71,12 @@ export async function clearPendingRebuilds(db, rows) {
 export async function rebuildProjectionOrMultiStream(name, deps) {
     const { db, registry, signal, errorPolicy } = deps;
     if (registry.getAllMultiStreamProjections().has(name)) {
-        return rebuildMultiStreamProjection(name, { db, registry, markDeadOnFailure: false });
+        return rebuildMultiStreamProjection(name, {
+            db,
+            registry,
+            markDeadOnFailure: false,
+            ...(signal && { signal }),
+        });
     }
     return rebuildProjection(name, {
         db,
@@ -80,6 +85,23 @@ export async function rebuildProjectionOrMultiStream(name, deps) {
         ...(errorPolicy && { errorPolicy }),
     });
 }
+/** Logs a finished queue drain and throws when any projection failed; the
+ *  failed tables stay queued. Shared by every `schema apply` entry point so the
+ *  report and the error text cannot drift apart. */
+export function reportPendingRebuildRun(run, log) {
+    if (run.rebuilt.length > 0) {
+        log(`  Rebuild ${run.rebuilt.length} Projection(s)…`);
+        for (const r of run.rebuilt) {
+            log(`    ↻ ${r.projection} (${r.eventsProcessed} events)`);
+        }
+        log("");
+    }
+    if (run.failed.length > 0) {
+        throw new Error(`Projection rebuild failed for: ${run.failed
+            .map((f) => `${f.projection} (${f.error})`)
+            .join("; ")}. Table(s) stay queued in kumiko_pending_rebuilds — retried on the next apply.`);
+    }
+}
 /** Arbeitet die persistierte Queue ab: mappt Tabellen auf Projektionen,
  *  rebuildet jede betroffene Projektion und räumt ihre Tabellen erst nach
  *  ERFOLG aus der Queue. Fehlgeschlagene bleiben pending — der nächste

package/dist/observability/escape-hatch-report.d.ts

@@ -20,4 +20,5 @@ export declare function createEscapeHatchReporter(opts: {
     readonly log?: Logger;
     readonly window?: EscapeHatchReportWindow;
 }): EscapeHatchReporter;
+export declare const UNATTRIBUTED_ACTOR = "system";
 export declare function fallbackEscapeHatchReporter(tenantId: TenantId): EscapeHatchReporter;

package/dist/observability/escape-hatch-report.js

@@ -83,6 +83,9 @@ export function createEscapeHatchReporter(opts) {
         reportEscapeHatchUse({ handler: opts.handler, kind, reason, tenantId: opts.tenantId, actor: opts.actor, target }, { sink: opts.sink, log: opts.log });
     };
 }
+// Actor for escape-hatch uses whose caller did not thread one through. Stays
+// "system" so audit events persisted before the constant existed remain valid.
+export const UNATTRIBUTED_ACTOR = "system";
 export function fallbackEscapeHatchReporter(tenantId) {
     return createEscapeHatchReporter({
         handler: "<unattributed>",

package/dist/observability/index.d.ts

@@ -1,5 +1,6 @@
 export { type ConsoleProviderOptions, createConsoleProvider } from "./console-provider.js";
 export { observabilityContext } from "./context.js";
+export { createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, type EscapeHatchReportWindow, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "./escape-hatch-report.js";
 export { getFallbackMeter, getFallbackProvider, getFallbackTracer } from "./fallback.js";
 export { generateSpanId, generateTraceId } from "./ids.js";
 export { buildMetricName, validateLabelKey, validateMetricName, } from "./metric-validator.js";
@@ -11,5 +12,5 @@ export { type MetricEvent, type MetricEventHandler, RecordingMeter, } from "./re
 export { type RecordedSpan, RecordingTracer, type RecordingTracerOptions, serializeSpanContext, } from "./recording-tracer.js";
 export { wrapRedisClient } from "./redis-wrapper.js";
 export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
-export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
+export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
 export type { Counter, Gauge, Histogram, Meter, MetricDefinition, MetricLabels, MetricsHandle, MetricType, ObservabilityOptions, ObservabilityProvider, SamplingConfig, SensitiveFilterConfig, SerializedTraceContext, Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatus, StartSpanOptions, Tracer, } from "./types/index.js";

package/dist/observability/index.js

@@ -1,6 +1,7 @@
 // Public surface of the observability module.
 export { createConsoleProvider } from "./console-provider.js";
 export { observabilityContext } from "./context.js";
+export { createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "./escape-hatch-report.js";
 export { getFallbackMeter, getFallbackProvider, getFallbackTracer } from "./fallback.js";
 export { generateSpanId, generateTraceId } from "./ids.js";
 export { buildMetricName, validateLabelKey, validateMetricName, } from "./metric-validator.js";
@@ -12,4 +13,4 @@ export { RecordingMeter, } from "./recording-meter.js";
 export { RecordingTracer, serializeSpanContext, } from "./recording-tracer.js";
 export { wrapRedisClient } from "./redis-wrapper.js";
 export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
-export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
+export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";

package/dist/observability/standard-metrics.d.ts

@@ -32,6 +32,10 @@ export declare function emitEventConsumerRearmExhausted(meter: Meter, labels: {
     readonly consumer: string;
     readonly instanceId: string;
 }): void;
+export declare function emitEventConsumerPassSkipped(meter: Meter, labels: {
+    readonly consumer: string;
+    readonly instanceId: string;
+}, reason: "idle"): void;
 export declare function emitEventDispatcherListenConnected(meter: Meter, connected: boolean): void;
 export declare function emitEventConsumerPassOutcome(meter: Meter, labels: {
     readonly consumer: string;

package/dist/observability/standard-metrics.js

@@ -93,6 +93,15 @@ export const STANDARD_METRIC_DEFS = [
         description: "Consumer transitioned to dead with its auto-rearm budget exhausted.",
         labels: ["consumer", "instance_id"],
     },
+    // An idle-gated consumer turn opens no events.consumer.pass span (the lock-free
+    // pre-check proves there is nothing to do), so this counter is the only trace of
+    // per-tick pass frequency for consumers that are idle or disabled.
+    {
+        name: "kumiko_event_consumer_pass_skipped_total",
+        type: "counter",
+        description: "Consumer turns skipped before opening a pass span because the idle pre-check proved nothing to deliver.",
+        labels: ["consumer", "instance_id", "reason"],
+    },
     // LISTEN-subscription health. 1 = active, 0 = dropped. Drops to 0 while
     // the dispatcher is running signal that delivery latency has regressed
     // from sub-millisecond (LISTEN) to pollIntervalMs (timer fallback); ops
@@ -203,6 +212,13 @@ export function emitEventConsumerRearmExhausted(meter, labels) {
         instance_id: labels.instanceId,
     });
 }
+export function emitEventConsumerPassSkipped(meter, labels, reason) {
+    meter.counter("kumiko_event_consumer_pass_skipped_total").inc(1, {
+        consumer: labels.consumer,
+        instance_id: labels.instanceId,
+        reason,
+    });
+}
 export function emitEventDispatcherListenConnected(meter, connected) {
     meter.gauge("kumiko_event_dispatcher_listen_connected").set(connected ? 1 : 0);
 }

package/dist/pipeline/dispatch-query.d.ts

@@ -1,4 +1,4 @@
 import type { DbTx } from "../db/connection.js";
 import type { SessionUser } from "../engine/types/index.js";
-import type { DispatchContext, WriteOrigin } from "./dispatch-shared.js";
+import { type DispatchContext, type WriteOrigin } from "./dispatch-shared.js";
 export declare function executeQuery(ctx: DispatchContext, type: string, payload: unknown, user: SessionUser, origin: WriteOrigin, tx?: DbTx): Promise<unknown>;

package/dist/pipeline/dispatch-query.js

@@ -2,7 +2,7 @@ import { hasAccess } from "../engine/access.js";
 import { filterReadFields } from "../engine/field-access.js";
 import { AccessDeniedError, NotFoundError, validationErrorFromZod } from "../errors/index.js";
 import { assertNoSecretLeak } from "../secrets/index.js";
-import { buildHandlerContext, enforceRateLimit, ensureFeatureEnabled, runHandlerInstrumented, } from "./dispatch-shared.js";
+import { buildHandlerContext, enforceRateLimit, ensureFeatureEnabled, isMemberResolutionPrincipal, runHandlerInstrumented, } from "./dispatch-shared.js";
 import { runInMemberReadOnlyTransaction } from "./member-read-only-transaction.js";
 // Standalone query execution — used by the public dispatcher.query() and
 // by ctx.query/ctx.queryAs inside handlers. Runs the handler, applies
@@ -51,7 +51,7 @@ async function executeQueryInner(ctx, type, payload, user, origin, tx) {
         parsed.data !== null &&
         parsed.data["includeDeleted"] === true; // @cast-boundary validated-payload
     // A resolved member (ctx.queryAsMember) runs in a Postgres READ ONLY transaction, not just the ctx surface below.
-    return user.origin === "member-resolution"
+    return isMemberResolutionPrincipal(user)
         ? runInMemberReadOnlyTransaction(ctx, tx, (readOnlyTx) => runQueryHandler(ctx, type, handler, parsed.data, includeDeleted, user, origin, readOnlyTx))
         : runQueryHandler(ctx, type, handler, parsed.data, includeDeleted, user, origin, tx);
 }

package/dist/pipeline/dispatch-shared.d.ts

@@ -6,8 +6,7 @@ import type { EffectiveFeaturesResolver } from "../engine/tier-resolver-extensio
 import type { TenantId } from "../engine/types/identifiers.js";
 import type { AppContext, HandlerContext, JobRunnerRef, RateLimitDeclaration, Registry, SessionUser, WriteResult } from "../engine/types/index.js";
 import { type WriteErrorInfo } from "../errors/index.js";
-import { type EscapeHatchReportWindow } from "../observability/escape-hatch-report.js";
-import { type getFallbackMeter, getFallbackTracer } from "../observability/index.js";
+import { type EscapeHatchReportWindow, type getFallbackMeter, getFallbackTracer } from "../observability/index.js";
 import { type AfterCommitHook } from "./dispatcher-utils.js";
 import type { IdempotencyGuard } from "./idempotency.js";
 import type { LifecycleHooks } from "./lifecycle-pipeline.js";
@@ -49,7 +48,7 @@ export type DispatchContext = {
 export declare function resolveDbSource(ctx: DispatchContext, tx: DbTx | undefined): DbConnection | DbTx | undefined;
 export declare function isMemberResolutionPrincipal(user: SessionUser): boolean;
 export declare function applyMemberResolutionReadOnly(handlerContext: HandlerContext): HandlerContext;
-export declare function buildHandlerContext(ctx: DispatchContext, type: string, user: SessionUser, origin: WriteOrigin, tx?: DbTx, afterCommitHooks?: AfterCommitHook[], includeDeleted?: boolean): Promise<HandlerContext>;
+export declare function buildHandlerContext(ctx: DispatchContext, type: string, user: SessionUser, origin: WriteOrigin, tx?: DbTx, afterCommitHooks?: AfterCommitHook[], includeDeleted?: boolean, isPostCommit?: boolean): Promise<HandlerContext>;
 export declare function runHandlerInstrumented<T>(ctx: DispatchContext, type: string, operation: "query" | "write" | "stream", user: SessionUser, inner: () => Promise<T>): Promise<T>;
 export declare function runStreamInstrumented<T>(ctx: DispatchContext, type: string, user: SessionUser, inner: () => AsyncGenerator<T>): AsyncGenerator<T>;
 export declare function checkFeatureEnabled(ctx: DispatchContext, qualifiedHandler: string, tenantId: TenantId): Promise<import("../errors/index.js").FeatureDisabledError | undefined>;