@cosmicdrift/kumiko-framework 0.333.0 → 0.335.0

package/dist/files/provider-resolver.js

@@ -50,8 +50,27 @@ export async function createFileProviderForTenant(ctx, tenantId, handlerName = "
         throw new Error(`${FEATURE_NAME}: provider "${provider}" registered without a build() — ` +
             `extension options must be a FileProviderPlugin.`);
     }
-    return usage.options.build(ctx, tenantId);
+    const built = await usage.options.build(ctx, tenantId);
+    // Extension payloads are never type-checked, so a plugin predating a
+    // required method (list) would otherwise surface as a misleading
+    // "missing s3:ListBucket permission" hint during erasure.
+    for (const method of REQUIRED_PROVIDER_METHODS) {
+        if (typeof built[method] !== "function") {
+            throw new Error(`${FEATURE_NAME}: provider "${provider}" built without the required method ${method}() — ` +
+                `update the file-provider plugin to the current FileStorageProvider contract.`);
+        }
+    }
+    return built;
 }
+const REQUIRED_PROVIDER_METHODS = [
+    "write",
+    "writeStream",
+    "read",
+    "readStream",
+    "delete",
+    "exists",
+    "list",
+];
 // Builds the resolver from the ambient AppContext fields — the framework-side
 // equivalent of the GDPR `makeTenantStorageProviderResolver`. The config
 // accessor (and therefore the s3.secretAccessKey secret-read) runs under

package/dist/files/types.js

@@ -207,6 +207,15 @@ export function validateFileContent(fileName, content, accept) {
     const ext = extractExtension(fileName);
     if (!ext)
         return { kind: "ok" };
+    // validateFile accepts aliased declared MIMEs (csv as vnd.ms-excel) on
+    // metadata alone, so the byte check has to live on this exported path too:
+    // a text-format extension with an alias must never carry a binary signature.
+    if (Object.hasOwn(EXTENSION_MIME_ALIASES, ext) && sniffMimeType(content) !== null) {
+        return {
+            kind: "rejected",
+            error: `content_mismatch: ".${ext}" upload bytes are a binary format`,
+        };
+    }
     const signatureMimeTypes = signatureMimeTypesForExtension(ext);
     if (signatureMimeTypes.length === 0)
         return { kind: "ok" };
@@ -261,9 +270,15 @@ extensionOverride) {
     const ext = /^[A-Za-z0-9]+$/.test(rawExt) ? rawExt.toLowerCase() : "bin";
     return `${tenantId}/${entityType}/${entityId}/${fieldName}/${uniqueId}.${ext}`;
 }
+const EXPORTS_PREFIX_SEGMENT = "exports";
 /** Fixed leading segment so one S3 lifecycle rule (Prefix: "exports/") can expire export bundles for every tenant without matching a buildStorageKey() upload. */
 export function tenantExportPrefix(tenantId) {
-    return `exports/${tenantId}/`;
+    // A tenant named like the reserved segment would make the sweep prefix
+    // "exports/" match every tenant's export bundles.
+    if (tenantId === EXPORTS_PREFIX_SEGMENT) {
+        throw new Error(`tenantId "${tenantId}" collides with the reserved storage segment`);
+    }
+    return `${EXPORTS_PREFIX_SEGMENT}/${tenantId}/`;
 }
 /**
  * Every storage-key prefix a tenant's binaries can live under. A tenant-destroy

package/dist/i18n/request-locale.d.ts

@@ -1,7 +1,8 @@
 export declare const DEFAULT_LOCALE = "en";
 export declare function isValidLocaleTag(value: string): boolean;
-/** BCP-47 is case-insensitive; canonicalize the primary subtag to lowercase
- *  so registry lookups (`mailT`, `hasMailTranslations`) hit registered keys. */
+/** BCP-47 is case-insensitive; canonicalize to the RFC 5646 conventional form
+ *  (`de-AT`, `zh-Hant-TW`) so registry lookups (`mailT`, `hasMailTranslations`)
+ *  hit registered keys regardless of header casing. */
 export declare function canonicalizeLocaleTag(tag: string): string;
 /**
  * Picks the best well-formed tag from an Accept-Language header (RFC 9110

package/dist/i18n/request-locale.js

@@ -14,13 +14,23 @@ const MAX_LOCALE_TAG_LENGTH = 35;
 export function isValidLocaleTag(value) {
     return value.length <= MAX_LOCALE_TAG_LENGTH && LOCALE_TAG_RE.test(value);
 }
-/** BCP-47 is case-insensitive; canonicalize the primary subtag to lowercase
- *  so registry lookups (`mailT`, `hasMailTranslations`) hit registered keys. */
+/** BCP-47 is case-insensitive; canonicalize to the RFC 5646 conventional form
+ *  (`de-AT`, `zh-Hant-TW`) so registry lookups (`mailT`, `hasMailTranslations`)
+ *  hit registered keys regardless of header casing. */
 export function canonicalizeLocaleTag(tag) {
-    const dash = tag.indexOf("-");
-    if (dash === -1)
-        return tag.toLowerCase();
-    return `${tag.slice(0, dash).toLowerCase()}${tag.slice(dash)}`;
+    return tag
+        .split("-")
+        .map((subtag, index) => {
+        if (index === 0)
+            return subtag.toLowerCase();
+        if (/^[A-Za-z]{2}$/.test(subtag))
+            return subtag.toUpperCase();
+        if (/^[A-Za-z]{4}$/.test(subtag)) {
+            return subtag.charAt(0).toUpperCase() + subtag.slice(1).toLowerCase();
+        }
+        return subtag.toLowerCase();
+    })
+        .join("-");
 }
 /**
  * Picks the best well-formed tag from an Accept-Language header (RFC 9110

package/dist/i18n/required-surface-keys.js

@@ -46,6 +46,24 @@ function pushSectionTitles(out, section, treatAsKey = false) {
     for (const group of section.groups ?? [])
         pushKey(out, group.title, treatAsKey);
 }
+// Field labels and section titles of a step rendered through the action-form shim.
+function pushActionFormShapedKeys(out, featureName, step) {
+    for (const fieldName of Object.keys(step.fields)) {
+        out.add(fieldLabelKey(featureName, ACTION_FORM_ENTITY, fieldName));
+    }
+    for (const section of step.layout.sections) {
+        if (isExtensionEditSection(section)) {
+            pushKey(out, section.title);
+            continue;
+        }
+        if (section.kind === "relatedList")
+            continue; // rejected at boot, unreachable here
+        pushSectionTitles(out, section);
+        for (const f of sectionFieldSpecs(section)) {
+            out.add(fieldLabelKey(featureName, ACTION_FORM_ENTITY, editFieldName(f)));
+        }
+    }
+}
 function pushRowActionKeys(out, action) {
     pushKey(out, action.label);
     if (action.kind === "writeHandler" || action.kind === undefined) {
@@ -242,26 +260,9 @@ export function requiredKeysFromScreen(featureName, screen, options = {}) {
             break;
         }
         case "secretMint": {
-            // Mirrors the actionForm case above (submitLabel, fields/section titles
-            // via ACTION_FORM_ENTITY) plus the reveal-phase's own i18n surface.
             const mint = screen;
             pushKey(out, mint.submitLabel);
-            for (const fieldName of Object.keys(mint.fields)) {
-                out.add(fieldLabelKey(featureName, ACTION_FORM_ENTITY, fieldName));
-            }
-            for (const section of mint.layout.sections) {
-                if (isExtensionEditSection(section)) {
-                    pushKey(out, section.title);
-                    continue;
-                }
-                if (section.kind === "relatedList")
-                    continue; // rejected at boot, unreachable here
-                pushSectionTitles(out, section);
-                for (const f of sectionFieldSpecs(section)) {
-                    const fieldName = editFieldName(f);
-                    out.add(fieldLabelKey(featureName, ACTION_FORM_ENTITY, fieldName));
-                }
-            }
+            pushActionFormShapedKeys(out, featureName, mint);
             pushKey(out, mint.reveal.title);
             pushKey(out, mint.reveal.warning);
             pushKey(out, mint.reveal.confirmLabel);
@@ -271,28 +272,17 @@ export function requiredKeysFromScreen(featureName, screen, options = {}) {
             if (mint.confirm !== undefined) {
                 pushKey(out, mint.confirm.submitLabel);
                 pushKey(out, mint.confirm.doneMessage);
-                for (const fieldName of Object.keys(mint.confirm.fields)) {
-                    out.add(fieldLabelKey(featureName, ACTION_FORM_ENTITY, fieldName));
-                }
-                for (const section of mint.confirm.layout.sections) {
-                    if (isExtensionEditSection(section)) {
-                        pushKey(out, section.title);
-                        continue;
-                    }
-                    if (section.kind === "relatedList")
-                        continue; // rejected at boot, unreachable here
-                    pushSectionTitles(out, section);
-                    for (const f of sectionFieldSpecs(section)) {
-                        const fieldName = editFieldName(f);
-                        out.add(fieldLabelKey(featureName, ACTION_FORM_ENTITY, fieldName));
-                    }
-                }
+                pushActionFormShapedKeys(out, featureName, mint.confirm);
             }
             break;
         }
         case "configEdit": {
             const config = screen;
             pushKey(out, config.submitLabel);
+            pushKey(out, config.description, treatDotFormAsKey);
+            for (const key of Object.values(config.fieldDescriptions ?? {})) {
+                pushKey(out, key, treatDotFormAsKey);
+            }
             for (const fieldName of Object.keys(config.fields)) {
                 const override = config.fieldLabels?.[fieldName];
                 if (override !== undefined)
@@ -308,6 +298,7 @@ export function requiredKeysFromScreen(featureName, screen, options = {}) {
                 if (section.kind === "relatedList")
                     continue; // rejected at boot, unreachable here
                 pushSectionTitles(out, section, treatDotFormAsKey);
+                pushKey(out, section.description, treatDotFormAsKey);
                 for (const f of sectionFieldSpecs(section)) {
                     const fieldName = editFieldName(f);
                     const override = config.fieldLabels?.[fieldName];
@@ -348,6 +339,7 @@ export function requiredKeysFromScreen(featureName, screen, options = {}) {
                 }
                 if (isWriteFormEditSection(section)) {
                     pushKey(out, section.title);
+                    pushKey(out, section.description);
                     pushKey(out, section.submitLabel);
                     // kumiko-lint-ignore section-fields-raw writeForm sections carry no groups (EditWriteFormSection)
                     for (const f of section.fields) {
@@ -399,7 +391,8 @@ function collectEntityListFilterKeys(feature, out) {
                     out.add(key);
                 }
             }
-            else if (def.type === "select" && Array.isArray(def.options)) {
+            else if ((def.type === "select" || def.type === "multiSelect") &&
+                Array.isArray(def.options)) {
                 for (const value of def.options) {
                     out.add(selectFacetOptionKey(feature.name, screen.entity, fieldName, value));
                 }

package/dist/i18n/settings-hub-keys.js

@@ -16,14 +16,31 @@ export const SETTINGS_HUB_I18N = {
     "config.settings.tenant": { en: "Tenant" },
     "config.settings.user": { en: "Personal" },
     "config.settings.extensionSelectorHint": {
-        en: "Save the provider selection to show its settings below.",
+        en: "Choose a provider and save. Its settings appear below.",
     },
+    "config.settings.provider": { en: "Provider" },
+    "config.settings.saveProvider": { en: "Save provider" },
+    "config.settings.audience.system": {
+        en: "Applies to every tenant unless a tenant sets its own value.",
+    },
+    "config.settings.audience.tenant": {
+        en: "Applies to everyone in this tenant. Platform defaults are shown where nothing is set here.",
+    },
+    "config.settings.audience.user": { en: "Only applies to you." },
     "config.secrets.title": { en: "Secrets" },
     "config.secrets.notSet": { en: "Not set" },
     "config.secrets.set": { en: "Set" },
-    "config.secrets.required": { en: "Required" },
+    "config.secrets.saved": { en: "Saved" },
+    "config.secrets.section": { en: "Secrets" },
+    "config.secrets.description": {
+        en: "Stored encrypted. Saved values are never shown again.",
+    },
+    "config.secrets.stored": { en: "Stored: {preview}" },
     "config.secrets.placeholder": { en: "Enter a value" },
-    "config.secrets.replacePlaceholder": { en: "Enter a new value to replace" },
-    "config.secrets.delete": { en: "Delete" },
+    "config.secrets.replacePlaceholder": { en: "Enter a new value to replace it" },
+    "config.secrets.delete": { en: "Remove" },
+    "config.secrets.deleteConfirm": {
+        en: "Remove this secret? Anything using it stops working, and it cannot be restored.",
+    },
     "screen:secrets.title": { en: "Secrets" },
 };

package/dist/jobs/index.d.ts

@@ -1,2 +1,2 @@
-export type { JobLogEntry, JobMeta, JobOutcomeMeta, JobRunner, JobRunnerOptions, } from "./job-runner.js";
-export { createJobRunner } from "./job-runner.js";
+export type { JobLogEntry, JobMeta, JobOutcomeMeta, JobRunner, JobRunnerOptions, JobSubjectValue, } from "./job-runner.js";
+export { createJobRunner, serializeJobSubject } from "./job-runner.js";

package/dist/jobs/index.js

@@ -1 +1 @@
-export { createJobRunner } from "./job-runner.js";
+export { createJobRunner, serializeJobSubject } from "./job-runner.js";

package/dist/jobs/job-runner.d.ts

@@ -28,6 +28,8 @@ export type JobOutcomeMeta = {
         readonly messageKey: string | null;
     } | undefined;
 };
+export type JobSubjectValue = string | number | boolean | null;
+export declare function serializeJobSubject(subject: Readonly<Record<string, JobSubjectValue>>): string;
 export type JobRunner = {
     start(): Promise<void>;
     stop(): Promise<void>;

package/dist/jobs/job-runner.js

@@ -1,4 +1,4 @@
-import { Queue, Worker } from "bullmq";
+import { Queue, UnrecoverableError, Worker } from "bullmq";
 import { Redis } from "ioredis";
 import { requestContext } from "../api/request-context.js";
 import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
@@ -10,8 +10,7 @@ import { InternalError } from "../errors/index.js";
 import { isKumikoError } from "../errors/kumiko-error.js";
 import { createFileContext } from "../files/file-handle.js";
 import { createFallbackLogger } from "../logging/index.js";
-import { createEscapeHatchReporter } from "../observability/escape-hatch-report.js";
-import { emitJobLastSuccess, emitJobQueueDepth, getFallbackTracer, } from "../observability/index.js";
+import { createEscapeHatchReporter, emitJobLastSuccess, emitJobQueueDepth, getFallbackTracer, registerStandardMetrics, } from "../observability/index.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
 import { RedisKeys } from "../pipeline/redis-keys.js";
 import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
@@ -122,6 +121,14 @@ function createJobLogger(logs, baseContext = {}) {
     };
     return logger;
 }
+// Single source of the stored subject string: the writer (jobSubjectKey) and
+// the tenant failures query both go through it. Matching is exact — field names
+// and JSON types must equal what the writer stored (`"1"` is not `1`).
+export function serializeJobSubject(subject) {
+    return JSON.stringify(Object.keys(subject)
+        .sort()
+        .map((field) => [field, subject[field] ?? null]));
+}
 // Stable identity for the declared payload fields: sorted keys, so two runs
 // with the same subject values produce the same string and the later one
 // replaces the earlier record. Non-primitives would serialize into something
@@ -129,15 +136,31 @@ function createJobLogger(logs, baseContext = {}) {
 function jobSubjectKey(jobName, payload, fields) {
     if (fields === undefined || fields.length === 0)
         return null;
-    const entries = [];
-    for (const field of [...fields].sort()) {
+    const subject = {};
+    for (const field of fields) {
         const value = payload[field] ?? null;
         if (value !== null && typeof value === "object") {
             throw new Error(`Job "${jobName}": tenantVisibleFailure.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
         }
-        entries.push([field, value]);
+        subject[field] = value;
     }
-    return JSON.stringify(entries);
+    return serializeJobSubject(subject);
+}
+// Jobs that never touch ctx.db run fine on a runner built without one; a job
+// that does gets a named error instead of a bare TypeError on undefined.
+function missingDbProxy(jobName) {
+    // @cast-boundary db-operator — every access throws, so no TenantDb member is ever returned
+    return new Proxy({}, {
+        get() {
+            throw new InternalError({
+                message: `[jobs] job runner constructed without a db connection — ctx.db is unavailable for "${jobName}"`,
+            });
+        },
+    });
+}
+function priorAttemptsOf(data) {
+    const value = data["_priorAttempts"];
+    return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
 }
 // Only a translation key ever travels to the tenant: the thrown error's own
 // i18nKey when it carries one, otherwise the key declared at the job. The
@@ -202,6 +225,9 @@ function parseRedisOpts(url) {
 // redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
 // would otherwise hang start() forever with no health endpoint to notice.
 const BOOT_REDIS_TIMEOUT_MS = 10_000;
+// Shutdown must fit a k8s grace period even when Redis is down, so the
+// readiness wait in stop() is capped well below the boot timeout.
+const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
 function timeoutReject(ms, message) {
     let timer;
     const promise = new Promise((_, reject) => {
@@ -224,10 +250,10 @@ const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
 // perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
 // set must retry the same way regardless of how it got enqueued, or it
 // fails for good on the very first error on whichever path skips this.
-function buildRetryBullOpts(jobDef) {
+function buildRetryBullOpts(jobDef, priorAttempts = 0) {
     const opts = {};
     if (jobDef.retries !== undefined)
-        opts.attempts = jobDef.retries + 1;
+        opts.attempts = Math.max(1, jobDef.retries + 1 - priorAttempts);
     if (jobDef.backoff) {
         opts.backoff =
             typeof jobDef.backoff === "string"
@@ -282,6 +308,10 @@ export function createJobRunner(options) {
     // boot); otherwise noop so dispatch/handleJob stay zero-cost without config.
     const tracer = context.tracer ?? getFallbackTracer();
     const errorLogger = createFallbackLogger("job-runner", context.log);
+    // The last-success gauge throws if its metric was never registered; the runner
+    // must not depend on buildServer having done it first. Idempotent.
+    if (context.meter)
+        registerStandardMetrics(context.meter);
     // Set at the top of stop() — a graceful shutdown closes the redis/BullMQ
     // clients itself, which fires the exact same 'error' listeners below with
     // an expected "Connection is closed." Downgrading to debug once stopping
@@ -318,7 +348,12 @@ export function createJobRunner(options) {
                 registry,
                 ...bridgeStub(),
             });
-            // @cast-boundary engine-payload — generic query-handler return for typed convention
+            if (!Array.isArray(result) || result.some((id) => typeof id !== "string")) {
+                throw new InternalError({
+                    message: `${ACTIVE_TENANT_IDS_QUERY_NAME} must return string[] of tenant ids`,
+                });
+            }
+            // @cast-boundary engine-payload — checked to be string[] above; TenantId is a brand
             return result;
         };
     }
@@ -455,6 +490,12 @@ export function createJobRunner(options) {
         }
         return false;
     }
+    async function reportPreRunFailure(jobName, bullJob, err) {
+        const message = err instanceof Error ? err.message : String(err);
+        await options.onJobFailed?.(jobName, bullJob.id ?? "unknown", message, [
+            { level: "error", message, timestamp: Temporal.Now.instant() },
+        ]);
+    }
     async function handleJob(bullJob) {
         const rawName = bullJob.name;
         // Handle perTenant dispatch jobs — fan out to one job per tenant. The
@@ -463,34 +504,51 @@ export function createJobRunner(options) {
         // (it's picked up by this runner's own worker).
         if (rawName.startsWith("_perTenant:")) {
             const actualName = rawName.slice("_perTenant:".length);
-            if (!getActiveTenantIds) {
-                throw new Error(`perTenant job "${actualName}" requires either options.getActiveTenantIds or the ` +
-                    `tenant feature mounted (it registers "${ACTIVE_TENANT_IDS_QUERY_NAME}", which the ` +
-                    "framework uses to resolve active tenants on its own)");
-            }
-            const actualDef = allJobs.get(actualName);
-            if (!actualDef) {
-                throw new Error(`Unknown job: ${actualName}`);
+            // These throws happen before runInSpan, so its onJobFailed never sees them
+            // and a drained test stack would stay green over a fan-out that never ran.
+            try {
+                if (!getActiveTenantIds) {
+                    if (registry.getQueryHandler(ACTIVE_TENANT_IDS_QUERY_NAME)) {
+                        throw new Error(`perTenant job "${actualName}": the tenant feature is mounted but the job runner has no ` +
+                            "`context.db`: pass a DbConnection in the runner context or supply options.getActiveTenantIds");
+                    }
+                    throw new Error(`perTenant job "${actualName}" requires either options.getActiveTenantIds or the ` +
+                        `tenant feature mounted (it registers "${ACTIVE_TENANT_IDS_QUERY_NAME}", which the ` +
+                        "framework uses to resolve active tenants on its own)");
+                }
+                const actualDef = allJobs.get(actualName);
+                if (!actualDef) {
+                    throw new Error(`Unknown job: ${actualName}`);
+                }
+                const tenantIds = await getActiveTenantIds();
+                const targetQueue = queues[laneForJob(actualDef)];
+                // wrapperJobId is only absent for a Job BullMQ somehow never assigned
+                // an id to (not observed in practice) — falling back to a constant
+                // there would collide every such run's children onto the same ids
+                // and silently swallow later runs as "duplicates", which is worse
+                // than the duplicate this is meant to prevent. Skip id-based dedup
+                // for that edge case instead: enqueue plainly, same as before.
+                const wrapperJobId = bullJob.id;
+                for (const tenantId of tenantIds) {
+                    await targetQueue.add(actualName, { ...bullJob.data, _tenantId: tenantId }, {
+                        ...buildRetryBullOpts(actualDef),
+                        // Dedup over wrapper retries only holds as long as the children
+                        // stay in Redis for the wrapper's whole retry window — see the
+                        // COMPLETED_JOB_RETENTION_AGE_SEC invariant check above.
+                        ...(wrapperJobId !== undefined
+                            ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
+                            : {}),
+                    });
+                }
             }
-            const tenantIds = await getActiveTenantIds();
-            const targetQueue = queues[laneForJob(actualDef)];
-            // wrapperJobId is only absent for a Job BullMQ somehow never assigned
-            // an id to (not observed in practice) — falling back to a constant
-            // there would collide every such run's children onto the same ids
-            // and silently swallow later runs as "duplicates", which is worse
-            // than the duplicate this is meant to prevent. Skip id-based dedup
-            // for that edge case instead: enqueue plainly, same as before.
-            const wrapperJobId = bullJob.id;
-            for (const tenantId of tenantIds) {
-                await targetQueue.add(actualName, { ...bullJob.data, _tenantId: tenantId }, {
-                    ...buildRetryBullOpts(actualDef),
-                    // Dedup over wrapper retries only holds as long as the children
-                    // stay in Redis for the wrapper's whole retry window — see the
-                    // COMPLETED_JOB_RETENTION_AGE_SEC invariant check above.
-                    ...(wrapperJobId !== undefined
-                        ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
-                        : {}),
-                });
+            catch (err) {
+                // A retry may still succeed, and a succeeding wrapper reports no completion
+                // that could clear an earlier failure, so only the final attempt is reported.
+                const retries = allJobs.get(actualName)?.retries ?? 0;
+                if (bullJob.attemptsMade + 1 >= retries + 1) {
+                    await reportPreRunFailure(rawName, bullJob, err);
+                }
+                throw err;
             }
             // skip: fan-out dispatcher job, per-tenant children enqueued
             return;
@@ -498,7 +556,9 @@ export function createJobRunner(options) {
         const jobName = rawName;
         const jobDef = allJobs.get(jobName);
         if (!jobDef) {
-            throw new Error(`Unknown job: ${jobName}`);
+            const err = new Error(`Unknown job: ${jobName}`);
+            await reportPreRunFailure(jobName, bullJob, err);
+            throw err;
         }
         // Sequential gate: try to claim the per-name lock. If another worker
         // (or this worker on a different bullJob) holds it, re-enqueue with a
@@ -516,13 +576,12 @@ export function createJobRunner(options) {
                 // same queue the worker just picked from (since only the consuming
                 // lane runs handleJob at all), but route explicitly — no implicit
                 // coupling to "whichever queue the caller happened to be on".
-                // The re-enqueued job starts with a full retry budget; a remaining
-                // budget is deliberately not carried over, since finalAttempt/
-                // tenantVisibleFailure are computed from jobDef.retries against
-                // attemptsMade, not against some inherited remainder.
-                await queues[laneForJob(jobDef)].add(jobName, bullJob.data, {
+                // Attempts already spent travel in _priorAttempts so a lock conflict
+                // on a retry cannot hand out a fresh budget and exceed `retries`.
+                const priorAttempts = priorAttemptsOf(bullJob.data) + bullJob.attemptsMade;
+                await queues[laneForJob(jobDef)].add(jobName, { ...bullJob.data, _priorAttempts: priorAttempts }, {
                     delay: SEQUENTIAL_RETRY_DELAY_MS,
-                    ...buildRetryBullOpts(jobDef),
+                    ...buildRetryBullOpts(jobDef, priorAttempts),
                 });
                 // skip: lock taken, work re-enqueued with delay, current invocation done
                 return;
@@ -537,7 +596,7 @@ export function createJobRunner(options) {
         // retry number, so audit queries distinguish fresh from retry runs
         // without peeking at BullMQ internals.
         const rawData = bullJob.data;
-        const attempt = bullJob.attemptsMade + 1;
+        const attempt = bullJob.attemptsMade + 1 + priorAttemptsOf(rawData);
         const meta = {
             triggeredById: rawData["_triggeredById"], // @cast-boundary dynamic-key
             payload: rawData["_payload"], // @cast-boundary dynamic-key
@@ -558,16 +617,26 @@ export function createJobRunner(options) {
         // subject is read from the handler payload, so it is resolved here where
         // the payload is built, not in the callbacks.
         const tenantVisibleDecl = jobDef.tenantVisibleFailure;
-        const tenantVisibleSubject = tenantVisibleDecl
-            ? jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields)
-            : null;
+        // A bad subject config must surface as a failed run, so it is thrown from
+        // inside runInSpan below instead of here, before any run record exists.
+        let tenantVisibleSubject = null;
+        let subjectConfigError;
+        if (tenantVisibleDecl) {
+            try {
+                tenantVisibleSubject = jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields);
+            }
+            catch (err) {
+                subjectConfigError = err instanceof Error ? err : new Error(String(err));
+            }
+        }
         // BullMQ stops retrying once attemptsMade reaches the configured attempts
         // (`retries + 1`), so this is the attempt whose failure is final.
-        const finalAttempt = attempt >= (jobDef.retries ?? 0) + 1;
+        const finalAttempt = bullJob.retryable === false || attempt >= (jobDef.retries ?? 0) + 1;
         const outcomeMeta = (messageKey) => ({
             tenantId,
             finalAttempt,
-            ...(tenantVisibleDecl && {
+            ...(tenantVisibleDecl &&
+                !subjectConfigError && {
                 tenantVisible: { subject: tenantVisibleSubject, messageKey },
             }),
         });
@@ -658,8 +727,7 @@ export function createJobRunner(options) {
         let memberReader;
         const jobContext = {
             ...context,
-            // Undefined only for a runner built without a db; JobContext requires db non-optional.
-            db: jobDb, // @cast-boundary db-operator
+            db: jobDb ?? missingDbProxy(jobName),
             files,
             derivatives,
             ...(notify !== undefined && { notify }),
@@ -721,6 +789,10 @@ export function createJobRunner(options) {
         const jobCorrelationId = inheritedCorrelationId ?? jobRequestId;
         const runInSpan = async () => {
             try {
+                if (subjectConfigError) {
+                    // Retrying cannot fix a declaration error.
+                    throw new UnrecoverableError(subjectConfigError.message);
+                }
                 if (writeOriginInvalid) {
                     throw new InternalError({
                         message: `Job "${jobName}" received an unparseable _writeOrigin — refusing to run without ` +
@@ -732,13 +804,22 @@ export function createJobRunner(options) {
                     correlationId: jobCorrelationId,
                     // #3043 — events a job writes carry the job as their origin.
                     handler: jobName,
-                    feature: qnScope(jobName),
+                    feature: registry.getJobFeature(jobName) ?? qnScope(jobName),
                     writeOrigin: jobOrigin,
                 }, () => jobDef.handler(payload, jobContext));
                 // Stamped before the observer hook: a throwing onJobComplete must not
                 // make a run that actually succeeded look dead to the liveness alert.
-                if (context.meter)
-                    emitJobLastSuccess(context.meter, jobName);
+                if (context.meter) {
+                    try {
+                        emitJobLastSuccess(context.meter, jobName);
+                    }
+                    catch (err) {
+                        errorLogger.warn("job last-success metric failed", {
+                            job: jobName,
+                            error: err instanceof Error ? err.message : String(err),
+                        });
+                    }
+                }
                 const duration = Date.now() - startTime;
                 await options.onJobComplete?.(jobName, jobId, duration, logs, outcomeMeta(null));
             }
@@ -795,7 +876,13 @@ export function createJobRunner(options) {
             if (!jobDef.bootGate)
                 continue;
             try {
-                await handleJob({ id: bootJobIdForJobName(name), name, data: {}, attemptsMade: 0 });
+                await handleJob({
+                    id: bootJobIdForJobName(name),
+                    name,
+                    data: {},
+                    attemptsMade: 0,
+                    retryable: false,
+                });
             }
             catch (err) {
                 // The worker is already consuming here; leaving its Redis connections
@@ -925,15 +1012,19 @@ export function createJobRunner(options) {
             // rejects the in-flight INFO with "Connection is closed." on an emitter
             // without listeners -> unhandled rejection (fw#1805). Wait for readiness
             // first; a failed/timed-out wait must not block the close itself.
-            const readinessTimeout = timeoutReject(bootRedisTimeoutMs, `job-runner: Queue not ready within ${bootRedisTimeoutMs}ms during stop()`);
+            const stopReadyTimeoutMs = Math.min(bootRedisTimeoutMs, STOP_QUEUE_READY_TIMEOUT_MS);
+            const readinessTimeout = timeoutReject(stopReadyTimeoutMs, `job-runner: Queue not ready within ${stopReadyTimeoutMs}ms during stop()`);
             try {
                 await Promise.race([
                     Promise.all([queues.api.waitUntilReady(), queues.worker.waitUntilReady()]),
                     readinessTimeout.promise,
                 ]);
             }
-            catch {
-                // close() below still has to run
+            catch (err) {
+                // close() below still has to run; stopping is already true, so debug matches the other shutdown logs
+                errorLogger.debug("queue readiness wait failed during stop()", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
             }
             finally {
                 readinessTimeout.cancel();

package/dist/migrations/index.d.ts

@@ -1,4 +1,4 @@
 export { assertKumikoSchemaCurrent, type ChecksumMismatch, detectKumikoDrift, formatKumikoDriftReport, type KumikoDriftReport, SchemaDriftError, } from "./kumiko-drift.js";
-export { createPendingRebuildsTable, type EnqueueProjectionRebuildDeps, type EnqueueProjectionRebuildResult, enqueueProjectionRebuild, listPendingRebuilds, type PendingRebuildRun, PROJECTION_REBUILD_JOB, pendingRebuildsTable, queueRebuildsFromMarkers, type RunPendingRebuildsOptions, rebuildProjectionOrMultiStream, runPendingRebuilds, } from "./pending-rebuilds.js";
+export { createPendingRebuildsTable, type EnqueueProjectionRebuildDeps, type EnqueueProjectionRebuildResult, enqueueProjectionRebuild, listPendingRebuilds, type PendingRebuildRun, PROJECTION_REBUILD_JOB, pendingRebuildsTable, queueRebuildsFromMarkers, type RunPendingRebuildsOptions, rebuildProjectionOrMultiStream, reportPendingRebuildRun, runPendingRebuilds, } from "./pending-rebuilds.js";
 export { createPiiBackfillStateTable, type PiiEventBackfillOptions, type PiiEventBackfillResult, type PiiRebuildRun, piiAnnotationFingerprint, piiBackfillStateTable, runPiiEventBackfill, } from "./pii-event-backfill.js";
 export { buildProjectionTableIndex } from "./projection-table-index.js";