@cosmicdrift/kumiko-framework 0.353.0 → 0.354.0

package/dist/event-store/admin-api.js

@@ -8,7 +8,7 @@
 // Allowlist: samples/*/migration/, scripts/migrations/, die Definition
 // selbst, das Guard-Script selbst.
 import { constraintOf, isUniqueViolation } from "../db/pg-error.js";
-import { claimXactId } from "../db/queries/event-store.js";
+import { claimXactId, runAtomicEventWrite } from "../db/queries/event-store.js";
 import { eventPredecessorExists, findExistingEventVersion, insertRawEventBatch, insertRawFirstEvent, insertRawSubsequentEvent, } from "../db/queries/event-store-admin.js";
 import { IdempotentAppendConflictError, VersionConflictError } from "./errors.js";
 function mapEventUniqueViolation(e, event) {
@@ -26,13 +26,15 @@ export async function appendRaw(runner, event) {
     try {
         // See db/queries/event-store.ts's claimXactId — gap-finality needs a
         // real xact id assigned before this holder's insert.
-        await claimXactId(runner);
-        if (event.expectedVersion === 0) {
-            await insertRawFirst(runner, event, newVersion, eventVersion);
-        }
-        else {
-            await insertRawSubsequent(runner, event, newVersion, eventVersion);
-        }
+        await runAtomicEventWrite(runner, async (tx) => {
+            await claimXactId(tx);
+            if (event.expectedVersion === 0) {
+                await insertRawFirst(tx, event, newVersion, eventVersion);
+            }
+            else {
+                await insertRawSubsequent(tx, event, newVersion, eventVersion);
+            }
+        });
     }
     catch (e) {
         if (isUniqueViolation(e)) {
@@ -90,8 +92,10 @@ export async function appendRawBatch(runner, events) {
         return `($${baseIdx + 1}::uuid, $${baseIdx + 2}, $${baseIdx + 3}::uuid, $${baseIdx + 4}, $${baseIdx + 5}, $${baseIdx + 6}, $${baseIdx + 7}::jsonb, $${baseIdx + 8}::jsonb, $${baseIdx + 9}::timestamptz, $${baseIdx + 10})`;
     });
     try {
-        await claimXactId(runner);
-        await insertRawEventBatch(runner, valuesClauses.join(", "), params);
+        await runAtomicEventWrite(runner, async (tx) => {
+            await claimXactId(tx);
+            await insertRawEventBatch(tx, valuesClauses.join(", "), params);
+        });
     }
     catch (e) {
         if (isUniqueViolation(e)) {

package/dist/event-store/event-store.js

@@ -4,7 +4,7 @@ import { Temporal } from "@cosmicdrift/kumiko-types/temporal";
 import { requestContext } from "../api/request-context.js";
 import { encryptEventPayloadPii } from "../crypto/event-pii.js";
 import { constraintOf, isUniqueViolation } from "../db/pg-error.js";
-import { claimXactId, insertSubsequentEventRow, notifyPgChannel, selectAggregateMaxVersion, selectEventsHighWaterMark, selectStreamMaxVersion, } from "../db/queries/event-store.js";
+import { claimXactId, insertSubsequentEventRow, notifyPgChannel, runAtomicEventWrite, selectAggregateMaxVersion, selectEventsHighWaterMark, selectStreamMaxVersion, } from "../db/queries/event-store.js";
 import { insertOne, selectMany } from "../db/query.js";
 import { isStreamArchived } from "./archive.js";
 import { IdempotentAppendConflictError, VersionConflictError } from "./errors.js";
@@ -50,14 +50,16 @@ export async function append(db, event) {
     const newVersion = toStore.expectedVersion + 1;
     const eventVersion = toStore.eventVersion ?? 1;
     try {
-        await claimXactId(db);
-        const row = toStore.expectedVersion === 0
-            ? await insertFirstEvent(db, toStore, newVersion, eventVersion, seedCreatedAt)
-            : await insertSubsequentEvent(db, toStore, newVersion, eventVersion, seedCreatedAt);
-        // NOTIFY after the INSERT: outside a transaction each statement commits
-        // on its own, so a NOTIFY sent first would wake the dispatcher before
-        // the row exists.
-        await notifyPgChannel(db, EVENTS_PUBSUB_CHANNEL);
+        const row = await runAtomicEventWrite(db, async (runner) => {
+            await claimXactId(runner);
+            const inserted = toStore.expectedVersion === 0
+                ? await insertFirstEvent(runner, toStore, newVersion, eventVersion, seedCreatedAt)
+                : await insertSubsequentEvent(runner, toStore, newVersion, eventVersion, seedCreatedAt);
+            // NOTIFY after the INSERT and inside the same transaction: PG delivers it on
+            // commit, so the dispatcher cannot wake before the row is visible.
+            await notifyPgChannel(runner, EVENTS_PUBSUB_CHANNEL);
+            return inserted;
+        });
         return buildStoredEvent(toStore, newVersion, eventVersion, row);
     }
     catch (e) {

package/dist/event-store/events-schema.js

@@ -1,7 +1,7 @@
 // sql now comes from native dialect
 import { bigserial, index, instant, integer, jsonb, table as pgTable, sql, text, uniqueIndex, uuid, } from "../db/dialect.js";
 import { tableExists } from "../db/index.js";
-import { ensureIdempotencyKeyIndex } from "../db/queries/event-store.js";
+import { ensureIdempotencyKeyIndex, ensureMembershipPayloadTenantIndex, } from "../db/queries/event-store.js";
 import { unsafePushTables } from "../stack/index.js";
 import { createArchivedStreamsTable } from "./archive.js";
 import { createSnapshotsTable } from "./snapshot.js";
@@ -61,6 +61,7 @@ export async function createEventsTable(db) {
     // that predate the idempotency-key index get healed the same way
     // ensureSnapshotVersionColumn heals kumiko_snapshots.
     await ensureIdempotencyKeyIndex(db);
+    await ensureMembershipPayloadTenantIndex(db);
     await createArchivedStreamsTable(db);
     await createSnapshotsTable(db);
 }

package/dist/files/file-handle.d.ts

@@ -2,6 +2,6 @@ import type { FileContext, FileHandle } from "@cosmicdrift/kumiko-types/file-han
 import type { FileStorageProvider } from "./types.js";
 export type { FileContext, FileHandle };
 export declare function createFileHandle(key: string, getProvider: () => Promise<FileStorageProvider>): FileHandle;
-export declare function createFileContext(resolve: () => Promise<FileStorageProvider>): FileContext;
+export declare function createFileContext(resolve: () => Promise<FileStorageProvider>, tenantId?: string): FileContext;
 export declare function deriveKey(key: string, suffix: string): string;
 export declare function storageKeyStemPrefix(key: string): string;

package/dist/files/file-handle.js

@@ -7,8 +7,10 @@
 // the bytes, and big files stay in the storage layer where they belong.
 //
 // `derive(suffix)` is the primitive for thumbnail/variant keys: it inserts a
-// suffix before the file extension — `foo/bar.jpg` + `"medium"` →
-// `foo/bar.medium.jpg`. Stable, reversible, no extra lookup tables.
+// suffix before the file extension — `foo/bar.jpg` + `"medium-<16 hex>"` →
+// `foo/bar.medium-<16 hex>.jpg`. The suffix must be a `variantSuffix(...)` so
+// the erasure sweep recognizes it. Stable, reversible, no extra lookup tables.
+import { isDerivativeSuffix } from "../derivatives/variant-key.js";
 // `getProvider` is a lazily-resolved, memoized accessor — the provider is
 // resolved (config + s3.secretAccessKey secret read) only when a handle method
 // actually does I/O, never on every request that merely builds a ctx.
@@ -19,7 +21,15 @@ export function createFileHandle(key, getProvider) {
         write: async (data, mimeType) => (await getProvider()).write(key, data, mimeType),
         delete: async () => (await getProvider()).delete(key),
         exists: async () => (await getProvider()).exists(key),
-        derive: (suffix) => createFileHandle(deriveKey(key, suffix), getProvider),
+        derive: (suffix) => {
+            // A suffix outside the derivative grammar would write a key that the
+            // forget/tenant-destroy sweep (isDerivativeKeyOf) can never recognize,
+            // so the binary would survive erasure.
+            if (!isDerivativeSuffix(suffix)) {
+                throw new Error(`FileHandle.derive: suffix "${suffix}" must be <name>-<16 hex chars>; build it with variantSuffix(name, spec)`);
+            }
+            return createFileHandle(deriveKey(key, suffix), getProvider);
+        },
     };
 }
 // `resolve` is a tenant-bound thunk (the caller binds the tenant). The result
@@ -27,15 +37,29 @@ export function createFileHandle(key, getProvider) {
 // produces — never a process-global cache, since each FileContext is bound to a
 // single request/event tenant; sharing would leak one tenant's provider (and
 // its bucket/credentials) to another.
-export function createFileContext(resolve) {
+//
+// With `tenantId`, `ref` and `list` only accept keys/prefixes inside `${tenantId}/` —
+// the key space is the isolation boundary in a store shared by several tenants, and
+// a request-derived key or prefix (or "") would otherwise reach every tenant's keys.
+export function createFileContext(resolve, tenantId) {
     let cached;
     const getProvider = () => {
         cached ??= resolve();
         return cached;
     };
     return {
-        ref: (key) => createFileHandle(key, getProvider),
-        list: async (prefix) => (await getProvider()).list(prefix),
+        ref: (key) => {
+            if (tenantId !== undefined && !key.startsWith(`${tenantId}/`)) {
+                throw new Error(`FileContext.ref: key must start with "${tenantId}/"`);
+            }
+            return createFileHandle(key, getProvider);
+        },
+        list: async (prefix) => {
+            if (tenantId !== undefined && !prefix.startsWith(`${tenantId}/`)) {
+                throw new Error(`FileContext.list: prefix must start with "${tenantId}/"`);
+            }
+            return (await getProvider()).list(prefix);
+        },
     };
 }
 // Splits a key into the part before the last dot of its last path segment

package/dist/files/file-routes.js

@@ -13,7 +13,7 @@ import { buildContentDispositionHeader } from "./content-disposition.js";
 import { createFileContext } from "./file-handle.js";
 import { fileRefEntity } from "./file-ref-entity.js";
 import { fileRefsTable } from "./file-ref-table.js";
-import { resolveContentType } from "./resolve-content-type.js";
+import { neutralFileNameForMimeType, resolveContentType } from "./resolve-content-type.js";
 import { buildStorageKey, parseMaxSize, resolveServedContentType, resolveUploadMimeType, validateFile, validateFileContent, } from "./types.js";
 const DEFAULT_PRIVILEGED_ROLES = ["Admin", "SystemAdmin"];
 // 15 minutes — long enough for a download to start, short enough that a
@@ -97,6 +97,7 @@ export function createFileRoutes(options) {
     const api = new Hono();
     const projectionGrants = options.registry ? { projectionRegistry: options.registry } : undefined;
     const tenantDbFor = (tenantId) => createTenantDb(db, tenantId, "tenant", undefined, undefined, undefined, projectionGrants);
+    const fileNameIsPersonal = piiSubjectFields.includes("fileName");
     function kmsContextFor() {
         return { requestId: requestContext.get()?.requestId ?? "file-routes" };
     }
@@ -311,7 +312,7 @@ export function createFileRoutes(options) {
         }
         // Built per request: createFileContext caches the resolved provider, so
         // one shared across requests would serve tenant A's store to tenant B.
-        const files = createFileContext(() => options.resolveProvider(user.tenantId));
+        const files = createFileContext(() => options.resolveProvider(user.tenantId), user.tenantId);
         const derivatives = createDerivativesContext({
             files,
             registry,
@@ -397,10 +398,15 @@ export function createFileRoutes(options) {
         const expiresInSeconds = SIGNED_URL_DEFAULT_EXPIRY_SECONDS;
         const url = await storageProvider.getSignedUrl(fileRef.storageKey, expiresInSeconds, {
             // Hint the provider to set Content-Disposition so the browser prompts
-            // with the original filename instead of the UUID-based storage key.
+            // with a readable filename instead of the UUID-based storage key.
             // Sanitised via buildContentDispositionHeader — the same attacker-
             // controlled fileName reaches the provider's presigned response.
-            contentDisposition: buildContentDispositionHeader(await resolveFileName(fileRef)),
+            // A personal fileName never goes into the URL: the signed query string
+            // lands in storage access logs and proxies and outlives a crypto-shred.
+            // The streaming route keeps the real name in a response header only.
+            contentDisposition: buildContentDispositionHeader(fileNameIsPersonal
+                ? neutralFileNameForMimeType(fileRef.mimeType)
+                : await resolveFileName(fileRef)),
         });
         const expiresAt = new Date(Date.now() + expiresInSeconds * 1000).toISOString();
         return c.json({ url, expiresAt });

package/dist/files/resolve-content-type.d.ts

@@ -1,3 +1,5 @@
+/** A name derived from the MIME type alone, for places a personal file name must not appear. */
+export declare function neutralFileNameForMimeType(mimeType: string): string;
 /**
  * Content type for a file whose sender may not have declared a useful one
  * (mail attachments commonly arrive as application/octet-stream). A concrete

package/dist/files/resolve-content-type.js

@@ -26,6 +26,14 @@ const EXTENSION_FALLBACK = {
     csv: "text/csv",
     txt: "text/plain",
 };
+const EXTENSION_BY_MIME_TYPE = new Map(Object.entries(EXTENSION_FALLBACK)
+    .reverse()
+    .map(([extension, mime]) => [mime, extension]));
+/** A name derived from the MIME type alone, for places a personal file name must not appear. */
+export function neutralFileNameForMimeType(mimeType) {
+    const extension = EXTENSION_BY_MIME_TYPE.get(normalizeMimeType(mimeType));
+    return extension === undefined ? "download" : `download.${extension}`;
+}
 function hasPrefix(bytes, prefix) {
     return prefix.every((byte, i) => bytes[i] === byte);
 }

package/dist/i18n/required-surface-keys.d.ts

@@ -10,6 +10,8 @@ export declare const PROJECTION_DETAIL_ENTITY = "__projection-detail__";
  *  drawn from the host record's display entity). */
 export declare const WRITE_FORM_SECTION_ENTITY = "__write-form-section__";
 export declare function fieldLabelKey(featureName: string, entityName: string, fieldName: string): string;
+export declare function handlerTitleKey(handlerQn: string): string;
+export declare function handlerFieldLabelKey(handlerQn: string, fieldName: string): string;
 export declare function booleanFacetOptionKeys(featureName: string, entityName: string, fieldName: string): readonly string[];
 export declare function selectFacetOptionKey(featureName: string, entityName: string, fieldName: string, value: string): string;
 export declare function screenTitleKey(screenId: string): string;

package/dist/i18n/required-surface-keys.js

@@ -13,6 +13,12 @@ export const WRITE_FORM_SECTION_ENTITY = "__write-form-section__";
 export function fieldLabelKey(featureName, entityName, fieldName) {
     return `${featureName}:entity:${entityName}:field:${fieldName}`;
 }
+export function handlerTitleKey(handlerQn) {
+    return `${handlerQn}:title`;
+}
+export function handlerFieldLabelKey(handlerQn, fieldName) {
+    return `${handlerQn}:field:${fieldName}`;
+}
 export function booleanFacetOptionKeys(featureName, entityName, fieldName) {
     const base = `${featureName}:entity:${entityName}:field:${fieldName}:option`;
     return [`${base}:true`, `${base}:false`];
@@ -116,6 +122,8 @@ function pushDashboardPanelKeys(out, panel) {
 function pushDashboardChartRangeKeys(out, panel) {
     for (const option of panel.ranges?.options ?? [])
         pushKey(out, option.label);
+    for (const kind of Object.values(panel.markerKinds ?? {}))
+        pushKey(out, kind.label);
 }
 function pushDashboardFilterKeys(out, filter) {
     pushKey(out, filter.label);

package/dist/jobs/index.d.ts

@@ -1,2 +1,2 @@
-export type { JobLogEntry, JobMeta, JobOutcomeMeta, JobRunner, JobRunnerOptions, JobSubjectValue, } from "./job-runner.js";
+export type { JobLogEntry, JobMeta, JobOutcomeMeta, JobQueuedMeta, JobRunner, JobRunnerOptions, JobSubjectValue, } from "./job-runner.js";
 export { createJobRunner, serializeJobSubject } from "./job-runner.js";

package/dist/jobs/job-runner.d.ts

@@ -19,6 +19,14 @@ export type JobMeta = {
     payload?: string | undefined;
     attempt?: number | undefined;
     priority?: number | undefined;
+    tenantVisibleRun?: {
+        readonly tenantId: string;
+        readonly subject: string | null;
+    } | undefined;
+};
+export type JobQueuedMeta = {
+    readonly tenantId: string;
+    readonly subject: string | null;
 };
 export type JobOutcomeMeta = {
     readonly tenantId: string;
@@ -27,6 +35,9 @@ export type JobOutcomeMeta = {
         readonly subject: string | null;
         readonly messageKey: string | null;
     } | undefined;
+    readonly tenantVisibleRun?: {
+        readonly subject: string | null;
+    } | undefined;
 };
 export type JobSubjectValue = string | number | boolean | null;
 export declare function serializeJobSubject(subject: Readonly<Record<string, JobSubjectValue>>): string;
@@ -53,5 +64,7 @@ export type JobRunnerOptions = {
     onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
     onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
     onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
+    onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
+    onJobDropped?: (jobName: string, jobId: string) => void;
 };
 export declare function createJobRunner(options: JobRunnerOptions): JobRunner;

package/dist/jobs/job-runner.js

@@ -134,14 +134,14 @@ export function serializeJobSubject(subject) {
 // with the same subject values produce the same string and the later one
 // replaces the earlier record. Non-primitives would serialize into something
 // no caller can reconstruct for a lookup, so they fail the run loudly.
-function jobSubjectKey(jobName, payload, fields) {
+function jobSubjectKey(jobName, payload, fields, declaration) {
     if (fields === undefined || fields.length === 0)
         return null;
     const subject = {};
     for (const field of fields) {
         const value = payload[field] ?? null;
         if (value !== null && typeof value === "object") {
-            throw new Error(`Job "${jobName}": tenantVisibleFailure.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
+            throw new Error(`Job "${jobName}": ${declaration}.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
         }
         subject[field] = value;
     }
@@ -516,12 +516,72 @@ export function createJobRunner(options) {
         }
         return false;
     }
+    // Only err.message, never payload or the logs array: job payloads can carry PII.
+    function logFailedAttempt(entry) {
+        errorLogger.error("job failed", {
+            job: entry.job,
+            jobId: entry.jobId,
+            attempt: entry.attempt,
+            // BullMQ never retries an UnrecoverableError, whatever attempts remain.
+            final: entry.finalAttempt || entry.err instanceof UnrecoverableError,
+            error: entry.err instanceof Error ? entry.err.message : String(entry.err),
+        });
+    }
     async function reportPreRunFailure(jobName, bullJob, err) {
         const message = err instanceof Error ? err.message : String(err);
         await options.onJobFailed?.(jobName, bullJob.id ?? "unknown", message, [
             { level: "error", message, timestamp: Temporal.Now.instant() },
         ]);
     }
+    function handlerPayloadOf(data) {
+        const payload = {};
+        for (const [k, v] of Object.entries(data)) {
+            if (!k.startsWith("_"))
+                payload[k] = v;
+        }
+        return payload;
+    }
+    function tenantIdOfJobData(data) {
+        return (data["_tenantId"] ?? // @cast-boundary dynamic-key
+            data["tenantId"] ?? // @cast-boundary dynamic-key
+            SYSTEM_TENANT_ID);
+    }
+    // Tenant-visible queued state. A callback or subject problem must never fail
+    // the enqueue; a bad subject config surfaces as a failed run instead.
+    async function announceQueued(jobName, jobDef, bullJobId, data) {
+        const declaration = jobDef.tenantVisibleRun;
+        // skip: job did not opt in, no logger wired, or BullMQ returned no id
+        if (!declaration || !options.onJobQueued || bullJobId === undefined)
+            return;
+        const tenantId = tenantIdOfJobData(data);
+        // skip: tenant-less run — no tenant-scoped query could ever read it
+        if (tenantId === SYSTEM_TENANT_ID)
+            return;
+        try {
+            const subject = jobSubjectKey(jobName, handlerPayloadOf(data), declaration.subjectFields, "tenantVisibleRun");
+            await options.onJobQueued(jobName, bullJobId, { tenantId, subject });
+        }
+        catch (err) {
+            errorLogger.warn("tenant-visible queued state not recorded", {
+                job: jobName,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    async function announceDropped(jobName, jobDef, bullJobId) {
+        // skip: job did not opt in or no logger wired
+        if (!jobDef.tenantVisibleRun || !options.onJobDropped)
+            return;
+        try {
+            await options.onJobDropped(jobName, bullJobId);
+        }
+        catch (err) {
+            errorLogger.warn("tenant-visible queued state not cleared", {
+                job: jobName,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
     async function handleJob(bullJob) {
         const rawName = bullJob.name;
         // Handle perTenant dispatch jobs — fan out to one job per tenant. The
@@ -556,7 +616,8 @@ export function createJobRunner(options) {
                 // for that edge case instead: enqueue plainly, same as before.
                 const wrapperJobId = bullJob.id;
                 for (const tenantId of tenantIds) {
-                    await targetQueue.add(actualName, { ...bullJob.data, _tenantId: tenantId }, {
+                    const childData = { ...bullJob.data, _tenantId: tenantId };
+                    const child = await targetQueue.add(actualName, childData, {
                         ...buildRetryBullOpts(actualDef),
                         // Dedup over wrapper retries only holds as long as the children
                         // stay in Redis for the wrapper's whole retry window — see the
@@ -565,13 +626,22 @@ export function createJobRunner(options) {
                             ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
                             : {}),
                     });
+                    await announceQueued(actualName, actualDef, child.id, childData);
                 }
             }
             catch (err) {
                 // A retry may still succeed, and a succeeding wrapper reports no completion
                 // that could clear an earlier failure, so only the final attempt is reported.
                 const retries = allJobs.get(actualName)?.retries ?? 0;
-                if (bullJob.attemptsMade + 1 >= retries + 1) {
+                const finalAttempt = bullJob.attemptsMade + 1 >= retries + 1;
+                logFailedAttempt({
+                    job: rawName,
+                    jobId: bullJob.id ?? "unknown",
+                    attempt: bullJob.attemptsMade + 1,
+                    finalAttempt,
+                    err,
+                });
+                if (finalAttempt) {
                     await reportPreRunFailure(rawName, bullJob, err);
                 }
                 throw err;
@@ -583,6 +653,14 @@ export function createJobRunner(options) {
         const jobDef = allJobs.get(jobName);
         if (!jobDef) {
             const err = new Error(`Unknown job: ${jobName}`);
+            // Final regardless of attempts left: a retry cannot make the job exist.
+            logFailedAttempt({
+                job: jobName,
+                jobId: bullJob.id ?? "unknown",
+                attempt: bullJob.attemptsMade + 1,
+                finalAttempt: true,
+                err,
+            });
             await reportPreRunFailure(jobName, bullJob, err);
             throw err;
         }
@@ -605,10 +683,15 @@ export function createJobRunner(options) {
                 // Attempts already spent travel in _priorAttempts so a lock conflict
                 // on a retry cannot hand out a fresh budget and exceed `retries`.
                 const priorAttempts = priorAttemptsOf(bullJob.data) + bullJob.attemptsMade;
-                await queues[laneForJob(jobDef)].add(jobName, { ...bullJob.data, _priorAttempts: priorAttempts }, {
+                const requeuedData = { ...bullJob.data, _priorAttempts: priorAttempts };
+                const requeued = await queues[laneForJob(jobDef)].add(jobName, requeuedData, {
                     delay: SEQUENTIAL_RETRY_DELAY_MS,
                     ...buildRetryBullOpts(jobDef, priorAttempts),
                 });
+                if (bullJob.id !== undefined && bullJob.id !== requeued.id) {
+                    await announceDropped(jobName, jobDef, bullJob.id);
+                }
+                await announceQueued(jobName, jobDef, requeued.id, requeuedData);
                 // skip: lock taken, work re-enqueued with delay, current invocation done
                 return;
             }
@@ -629,27 +712,31 @@ export function createJobRunner(options) {
             attempt,
         };
         // Build handler payload (without internal meta fields)
-        const payload = {};
-        for (const [k, v] of Object.entries(rawData)) {
-            if (!k.startsWith("_"))
-                payload[k] = v;
-        }
+        const payload = handlerPayloadOf(rawData);
         // Determine tenantId and triggeredBy from meta
-        const tenantId = rawData["_tenantId"] ?? // @cast-boundary dynamic-key
-            payload["tenantId"] ?? // @cast-boundary dynamic-key
-            SYSTEM_TENANT_ID;
+        const tenantId = tenantIdOfJobData(rawData);
         const triggeredById = rawData["_triggeredById"] ?? null; // @cast-boundary dynamic-key
         // Tenant-visible failure record (JobDefinition.tenantVisibleFailure). The
         // subject is read from the handler payload, so it is resolved here where
         // the payload is built, not in the callbacks.
         const tenantVisibleDecl = jobDef.tenantVisibleFailure;
+        const tenantVisibleRunDecl = jobDef.tenantVisibleRun;
         // A bad subject config must surface as a failed run, so it is thrown from
         // inside runInSpan below instead of here, before any run record exists.
         let tenantVisibleSubject = null;
+        let tenantVisibleRunSubject = null;
         let subjectConfigError;
-        if (tenantVisibleDecl) {
+        if (tenantVisibleRunDecl) {
             try {
-                tenantVisibleSubject = jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields);
+                tenantVisibleRunSubject = jobSubjectKey(jobName, payload, tenantVisibleRunDecl.subjectFields, "tenantVisibleRun");
+            }
+            catch (err) {
+                subjectConfigError = err instanceof Error ? err : new Error(String(err));
+            }
+        }
+        if (tenantVisibleDecl && !subjectConfigError) {
+            try {
+                tenantVisibleSubject = jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields, "tenantVisibleFailure");
             }
             catch (err) {
                 subjectConfigError = err instanceof Error ? err : new Error(String(err));
@@ -658,13 +745,16 @@ export function createJobRunner(options) {
         // BullMQ stops retrying once attemptsMade reaches the configured attempts
         // (`retries + 1`), so this is the attempt whose failure is final.
         const finalAttempt = bullJob.retryable === false || attempt >= (jobDef.retries ?? 0) + 1;
-        const outcomeMeta = (messageKey) => ({
+        const outcomeMeta = (messageKey, unrecoverable = false) => ({
             tenantId,
-            finalAttempt,
+            // BullMQ never retries an UnrecoverableError, whatever attempts are left.
+            finalAttempt: finalAttempt || unrecoverable,
             ...(tenantVisibleDecl &&
                 !subjectConfigError && {
                 tenantVisible: { subject: tenantVisibleSubject, messageKey },
             }),
+            ...(tenantVisibleRunDecl &&
+                !subjectConfigError && { tenantVisibleRun: { subject: tenantVisibleRunSubject } }),
         });
         // Carry `_triggerName` from rawData when set — handleEvent injects it on
         // multi-trigger dispatch; exposed as jobContext.triggerName so handlers
@@ -693,7 +783,9 @@ export function createJobRunner(options) {
         // through the same _fileProviderResolver for jobs as for write-handlers,
         // otherwise event-triggered jobs silently get an unresolved ctx.files.
         const fileResolver = context._fileProviderResolver;
-        const files = fileResolver ? createFileContext(() => fileResolver(tenantId)) : context.files;
+        const files = fileResolver
+            ? createFileContext(() => fileResolver(tenantId), tenantId)
+            : context.files;
         const jobSystemUser = createSystemUser(tenantId);
         // Same buildHandlerContext parity as ctx.files above: ctx.notify/ctx.config
         // must resolve for jobs the same way they do for write-handlers, or an
@@ -813,6 +905,9 @@ export function createJobRunner(options) {
                 return memberReader(userId, qn, payload);
             },
         };
+        if (tenantVisibleRunDecl && !subjectConfigError) {
+            meta.tenantVisibleRun = { tenantId, subject: tenantVisibleRunSubject };
+        }
         await options.onJobStart?.(jobName, jobId, meta);
         // Cross-process trace continuation: if the enqueuing code captured a
         // parent span, start the job.execute span as its child. Works for event
@@ -865,7 +960,8 @@ export function createJobRunner(options) {
             catch (err) {
                 const errorMsg = err instanceof Error ? err.message : String(err);
                 logs.push({ level: "error", message: errorMsg, timestamp: Temporal.Now.instant() });
-                await options.onJobFailed?.(jobName, jobId, errorMsg, logs, outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl)));
+                logFailedAttempt({ job: jobName, jobId, attempt, finalAttempt, err });
+                await options.onJobFailed?.(jobName, jobId, errorMsg, logs, outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl), err instanceof UnrecoverableError));
                 throw err;
             }
         };
@@ -938,8 +1034,7 @@ export function createJobRunner(options) {
                 ]);
             }
             catch (err) {
-                // The worker is already consuming here; leaving its Redis connections
-                // open would keep the event loop alive and turn the aborted boot into
+                // Leaving the worker's Redis connections open would keep the event loop alive and turn the aborted boot into
                 // a hanging process instead of a non-zero exit.
                 await worker?.close();
                 worker = null;
@@ -965,6 +1060,9 @@ export function createJobRunner(options) {
             worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), handleJob, {
                 connection: redisOpts,
                 concurrency: 5,
+                // Not consuming until the boot gates passed: a job already queued from a
+                // previous run must not execute against an unmet boot precondition.
+                autorun: false,
             });
             worker.on("error", (err) => {
                 const log = stopping ? errorLogger.debug : errorLogger.error;
@@ -989,6 +1087,11 @@ export function createJobRunner(options) {
             // Ahead of cron/runOnBoot wiring on purpose: a failed gate must not
             // leave recurring schedulers behind.
             await runBootGates(consumerLane);
+            // Floating on purpose: run() resolves only when the worker closes.
+            void worker.run().catch((err) => {
+                const log = stopping ? errorLogger.debug : errorLogger.error;
+                log("worker run loop failed", { error: err instanceof Error ? err.message : String(err) });
+            });
             // Only schedule cron + boot for jobs that belong to this lane. Jobs
             // assigned to the other lane get their cron/boot wiring from the
             // runner running on that lane. Running both here would double-fire.
@@ -1142,6 +1245,7 @@ export function createJobRunner(options) {
                     for (const j of waiting) {
                         if (j.name === jobName && j.id) {
                             await j.remove();
+                            await announceDropped(jobName, jobDef, j.id);
                         }
                     }
                     break;
@@ -1182,6 +1286,7 @@ export function createJobRunner(options) {
                 data["_correlationId"] = reqCtx.correlationId;
             stampGatedWriteOrigin(data);
             const job = await targetQueue.add(jobName, data, bullOpts);
+            await announceQueued(jobName, jobDef, job.id, data);
             return job.id ?? "unknown";
         },
         async handleEvent(eventName, payload, user, triggerEvent) {
@@ -1230,7 +1335,8 @@ export function createJobRunner(options) {
                 stampGatedWriteOrigin(data);
                 // Route to the job's declared lane, not a fixed queue — that's
                 // the whole reason both queues are held.
-                await queues[laneForJob(jobDef)].add(name, data, buildRetryBullOpts(jobDef));
+                const job = await queues[laneForJob(jobDef)].add(name, data, buildRetryBullOpts(jobDef));
+                await announceQueued(name, jobDef, job.id, data);
             }
         },
         attachDispatcher(ref) {