@cosmicdrift/kumiko-framework 0.386.0 → 0.388.0

package/dist/jobs/job-runner.js

@@ -1,27 +1,21 @@
-/// <reference types="temporal-polyfill/global" preserve="true" />
-import { crossTenantReadsGrantFor, escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
-import { DelayedError, Queue, UnrecoverableError, Worker } from "bullmq";
+import { DelayedError, Queue, Worker } from "bullmq";
 import { Redis } from "ioredis";
 import { requestContext } from "../api/request-context.js";
-import { createCrossTenantReads } from "../db/cross-tenant-reads.js";
 import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
-import { createDerivativesContext } from "../derivatives/derivatives-context.js";
 import { bridgeStub } from "../engine/bridge-stub.js";
-import { qnScope } from "../engine/qualified-name.js";
 import { createSystemUser } from "../engine/system-user.js";
 import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
 import { InternalError } from "../errors/index.js";
-import { createTenantFileContext } from "../files/file-handle.js";
-import { createFallbackLogger, scrubLogText } from "../logging/index.js";
-import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
-import { configAccessorFactoryOf } from "../pipeline/config-accessor-factory.js";
+import { createFallbackLogger } from "../logging/index.js";
+import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, } from "../observability/index.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
 import { RedisKeys } from "../pipeline/redis-keys.js";
-import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js";
-import { buildPersonalDataGate, parseWriteOrigin } from "../pipeline/write-origin.js";
 import { cronRepeatOptions } from "./cron-repeat-options.js";
-import { BOOT_GATE_SLOW_WARN_MS, BOOT_REDIS_TIMEOUT_MS, buildRetryBullOpts, COMPLETED_JOB_RETENTION_AGE_SEC, captureTraceContext, createJobLogger, DEFAULT_MAX_DEFERS, DEFER_BACKOFF_MAX_DOUBLINGS, DEFER_COUNT_KEY, DEFER_DELAY_CAP_MS, FAILED_JOB_RETENTION_AGE_SEC, HANDLER_DEFER_COUNT_KEY, isJobTriggerEvent, jobSubjectKey, maxRetryWindowMs, missingDbProxy, parseRedisOpts, payloadMatchesTriggerFilter, priorAttemptsOf, readTraceContext, resolveJobTz, STOP_QUEUE_READY_TIMEOUT_MS, stampGatedWriteOrigin, startQueueDepthPolling, TRACE_CONTEXT_KEY, tenantFailureMessageKey, timeoutReject, } from "./job-runner-helpers.js";
-import { ACTIVE_TENANT_IDS_QUERY_NAME, bootJobIdForJobName, DEFAULT_QUEUE_NAME_PREFIX, markSchedulersSeen, ORPHAN_SCHEDULER_GRACE_MS, perTenantChildJobId, queueNameFor, removeOrphanJobSchedulers, schedulerIdForJobName, } from "./job-runner-schedulers.js";
+import { applyHandlerDeferral, executeJobInSpan } from "./job-runner-execution.js";
+import { claimSequentialSlot, deferOverTenantLimit, fanOutPerTenantJob, PER_TENANT_PREFIX, perTenantTargetOf, releaseSequentialSlot, resolveJobDefinitionOrFail, startJobRun, } from "./job-runner-gates.js";
+import { BOOT_GATE_SLOW_WARN_MS, BOOT_REDIS_TIMEOUT_MS, buildRetryBullOpts, COMPLETED_JOB_RETENTION_AGE_SEC, captureTraceContext, FAILED_JOB_RETENTION_AGE_SEC, maxRetryWindowMs, parseRedisOpts, payloadMatchesTriggerFilter, STOP_QUEUE_READY_TIMEOUT_MS, stampGatedWriteOrigin, startQueueDepthPolling, TRACE_CONTEXT_KEY, timeoutReject, } from "./job-runner-helpers.js";
+import { ACTIVE_TENANT_IDS_QUERY_NAME, bootJobIdForJobName, DEFAULT_QUEUE_NAME_PREFIX, markSchedulersSeen, ORPHAN_SCHEDULER_GRACE_MS, queueNameFor, removeOrphanJobSchedulers, schedulerIdForJobName, } from "./job-runner-schedulers.js";
+import { announceDropped, announceQueued, isOverPerTenantLimit, laneForJob, } from "./job-runner-scope.js";
 import { resolveMaxPerTenant } from "./max-per-tenant.js";
 export { serializeJobSubject } from "./job-runner-helpers.js";
 export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.js";
@@ -120,12 +114,6 @@ export function createJobRunner(options) {
                 "jobRetention.completedAgeSec.");
         }
     }
-    // Resolve the lane for a job — "worker" is the default because that's the
-    // sensible prod lane (heavy async off the request path). Jobs that opted
-    // into "api" must have been validated at registry boot already.
-    function laneForJob(def) {
-        return def.runIn ?? "worker";
-    }
     // Sequential coordination: BullMQ OSS has no `group`, so we serialise
     // same-name jobs ourselves with a per-name Redis lock. Only built when at
     // least one job actually requested it — keeps the no-sequential boot path
@@ -139,8 +127,7 @@ export function createJobRunner(options) {
         lockRedis = new Redis(redisOpts);
         // Without a listener, a post-close 'error' (e.g. a teardown-race
         // "Connection is closed") is unhandled and crashes the process — in
-        // bun:test it gets attributed to whichever test happens to run next
-        // (fw#1805).
+        // bun:test it gets attributed to whichever test happens to run next.
         lockRedis.on("error", (err) => {
             const log = stopping ? errorLogger.debug : errorLogger.error;
             log("lock redis connection error", { error: err.message });
@@ -150,14 +137,6 @@ export function createJobRunner(options) {
         // isolated by prefix on a shared Redis don't contend for the same lock.
         sequentialLock = createDistributedLock(lockRedis, `${RedisKeys.lock}seq:${queueNamePrefix}:${lockScope}:`);
     }
-    // Default lock-TTL for sequential jobs that didn't declare a timeout.
-    // 5 minutes matches BullMQ's default stalledInterval — long enough for
-    // any reasonable handler, short enough that a crashed worker recovers
-    // without manual intervention.
-    const SEQUENTIAL_DEFAULT_TTL_SEC = 305;
-    // How long to wait before re-trying a busy sequential lock. Short enough
-    // to feel responsive, long enough that we don't hammer Redis.
-    const SEQUENTIAL_RETRY_DELAY_MS = 200;
     // Two queue-clients — one per lane. Every runner holds both, regardless of
     // its own consumerLane, so dispatch()/handleEvent() always write to the
     // queue matching the target job's runIn. Client-creation is cheap (shared
@@ -181,7 +160,7 @@ export function createJobRunner(options) {
         }),
     };
     // Same unhandled-'error'-crash hazard as lockRedis above, just via
-    // BullMQ's internal ioredis client (fw#1805).
+    // BullMQ's internal ioredis client.
     for (const queue of Object.values(queues)) {
         queue.on("error", (err) => {
             const log = stopping ? errorLogger.debug : errorLogger.error;
@@ -203,644 +182,53 @@ export function createJobRunner(options) {
     // Set by attachDispatcher() once the boot-level dispatcher exists.
     // JobContext.write/writeAs/queryAs throw until this is set — see JobContext doc.
     let dispatchWriteRef;
-    // Counts active + waiting jobs with this name for this tenant across
-    // BOTH lane queues. Jobs with the same name should only live in one
-    // lane (jobDef.runIn is static), but walking both is cheap and avoids
-    // a subtle bug if someone ever reassigns a job to a different lane
-    // between deploys while old queue contents are still draining.
-    // `activeOnly` + `self` serve the defer check, which runs inside the job being
-    // counted: it must not see waiting jobs or itself, and only counts jobs that
-    // became active before it. Without that order, workers that fetch a burst of
-    // jobs together all see each other as active and all defer, again and again.
-    async function isOverPerTenantLimit(jobName, tenantId, max, scope = {}) {
-        const results = await Promise.all([
-            queues.api.getActive(),
-            queues.worker.getActive(),
-            ...(scope.activeOnly ? [] : [queues.api.getWaiting(), queues.worker.getWaiting()]),
-        ]);
-        let count = 0;
-        for (const list of results) {
-            for (const j of list) {
-                if (j.name !== jobName)
-                    continue;
-                if (scope.self && !isActiveBefore(j, scope.self))
-                    continue;
-                const t = j.data?._tenantId; // @cast-boundary dynamic-key
-                if (t === tenantId) {
-                    count += 1;
-                    if (count >= max)
-                        return true;
-                }
-            }
-        }
-        return false;
-    }
-    function isActiveBefore(other, self) {
-        if (other.id === self.id)
-            return false;
-        if (self.processedOn === undefined)
-            return true;
-        const otherProcessedOn = other.processedOn ?? 0;
-        if (otherProcessedOn !== self.processedOn)
-            return otherProcessedOn < self.processedOn;
-        return (other.id ?? "") < (self.id ?? "");
-    }
-    // Growing delay bounds Redis and worker load while one tenant has a large
-    // backlog: every deferred job would otherwise be re-checked every deferMs.
-    function deferDelayMs(deferMs, deferCount) {
-        const grown = Math.min(deferMs * 2 ** Math.min(deferCount, DEFER_BACKOFF_MAX_DOUBLINGS), DEFER_DELAY_CAP_MS);
-        return grown + Math.floor(Math.random() * deferMs);
-    }
-    function isJobDeferral(value) {
-        return (typeof value === "object" &&
-            value !== null &&
-            "kind" in value &&
-            value.kind === "job-deferral");
-    }
-    function priorDeferCountOf(data, key) {
-        const prior = data[key];
-        return typeof prior === "number" ? prior : 0;
-    }
-    // Only err.message, never payload or the logs array: job payloads can carry PII.
-    function logFailedAttempt(entry) {
-        errorLogger.error("job failed", {
-            job: entry.job,
-            jobId: entry.jobId,
-            attempt: entry.attempt,
-            // BullMQ never retries an UnrecoverableError, whatever attempts remain.
-            final: entry.finalAttempt || entry.err instanceof UnrecoverableError,
-            error: scrubLogText(entry.err instanceof Error ? entry.err.message : String(entry.err)),
-            ...(entry.err instanceof Error &&
-                entry.err.stack && { stack: scrubLogText(entry.err.stack) }),
-        });
-    }
-    async function reportPreRunFailure(jobName, bullJob, err) {
-        const message = err instanceof Error ? err.message : String(err);
-        await options.onJobFailed?.(jobName, bullJob.id ?? "unknown", message, [
-            { level: "error", message, timestamp: Temporal.Now.instant() },
-        ]);
-    }
-    function handlerPayloadOf(data) {
-        const payload = {};
-        for (const [k, v] of Object.entries(data)) {
-            if (!k.startsWith("_"))
-                payload[k] = v;
-        }
-        return payload;
-    }
-    function tenantIdOfJobData(data) {
-        return (data["_tenantId"] ?? // @cast-boundary dynamic-key
-            data["tenantId"] ?? // @cast-boundary dynamic-key
-            SYSTEM_TENANT_ID);
-    }
-    // Tenant-visible queued state. A callback or subject problem must never fail
-    // the enqueue; a bad subject config surfaces as a failed run instead.
-    async function announceQueued(jobName, jobDef, bullJobId, data) {
-        const declaration = jobDef.tenantVisibleRun;
-        // skip: job did not opt in, no logger wired, or BullMQ returned no id
-        if (!declaration || !options.onJobQueued || bullJobId === undefined)
-            return;
-        const tenantId = tenantIdOfJobData(data);
-        // skip: tenant-less run — no tenant-scoped query could ever read it
-        if (tenantId === SYSTEM_TENANT_ID)
-            return;
-        try {
-            const subject = jobSubjectKey(jobName, handlerPayloadOf(data), declaration.subjectFields, "tenantVisibleRun");
-            await options.onJobQueued(jobName, bullJobId, { tenantId, subject });
-        }
-        catch (err) {
-            errorLogger.warn("tenant-visible queued state not recorded", {
-                job: jobName,
-                error: err instanceof Error ? err.message : String(err),
-            });
-        }
-    }
-    async function announceDropped(jobName, jobDef, bullJobId, data) {
-        // skip: job did not opt in or no logger wired
-        if (!jobDef.tenantVisibleRun || !options.onJobDropped)
-            return;
-        const tenantId = tenantIdOfJobData(data);
-        // skip: tenant-less run — announceQueued never recorded a row for it
-        if (tenantId === SYSTEM_TENANT_ID)
-            return;
-        try {
-            await options.onJobDropped(jobName, bullJobId, { tenantId });
-        }
-        catch (err) {
-            errorLogger.warn("tenant-visible queued state not cleared", {
-                job: jobName,
-                error: err instanceof Error ? err.message : String(err),
-            });
-        }
-    }
+    const scope = {
+        registry,
+        context,
+        options,
+        allJobs,
+        queues,
+        consumerLane,
+        tracer,
+        errorLogger,
+        sequentialLock,
+        getActiveTenantIds,
+        escapeHatchReportWindow,
+        systemCronEscapeHatchDedup,
+        getSelfRunner: () => selfRunner,
+        getDispatchWriteRef: () => dispatchWriteRef,
+    };
     async function handleJob(bullJob, token) {
-        const rawName = bullJob.name;
-        // Handle perTenant dispatch jobs — fan out to one job per tenant. The
-        // fan-out re-enqueues into the lane the actual job is assigned to;
-        // the _perTenant wrapper itself always lives in the consumer-lane
-        // (it's picked up by this runner's own worker).
-        if (rawName.startsWith("_perTenant:")) {
-            const actualName = rawName.slice("_perTenant:".length);
-            // These throws happen before runInSpan, so its onJobFailed never sees them
-            // and a drained test stack would stay green over a fan-out that never ran.
-            try {
-                if (!getActiveTenantIds) {
-                    if (registry.getQueryHandler(ACTIVE_TENANT_IDS_QUERY_NAME)) {
-                        throw new Error(`perTenant job "${actualName}": the tenant feature is mounted but the job runner has no ` +
-                            "`context.db`: pass a DbConnection in the runner context or supply options.getActiveTenantIds");
-                    }
-                    throw new Error(`perTenant job "${actualName}" requires either options.getActiveTenantIds or the ` +
-                        `tenant feature mounted (it registers "${ACTIVE_TENANT_IDS_QUERY_NAME}", which the ` +
-                        "framework uses to resolve active tenants on its own)");
-                }
-                const actualDef = allJobs.get(actualName);
-                if (!actualDef) {
-                    throw new Error(`Unknown job: ${actualName}`);
-                }
-                const tenantIds = await getActiveTenantIds();
-                const targetQueue = queues[laneForJob(actualDef)];
-                // wrapperJobId is only absent for a Job BullMQ somehow never assigned
-                // an id to (not observed in practice) — falling back to a constant
-                // there would collide every such run's children onto the same ids
-                // and silently swallow later runs as "duplicates", which is worse
-                // than the duplicate this is meant to prevent. Skip id-based dedup
-                // for that edge case instead: enqueue plainly, same as before.
-                const wrapperJobId = bullJob.id;
-                for (const tenantId of tenantIds) {
-                    const childData = { ...bullJob.data, _tenantId: tenantId };
-                    const child = await targetQueue.add(actualName, childData, {
-                        ...buildRetryBullOpts(actualDef),
-                        // Dedup over wrapper retries only holds as long as the children
-                        // stay in Redis for the wrapper's whole retry window — see the
-                        // COMPLETED_JOB_RETENTION_AGE_SEC invariant check above.
-                        ...(wrapperJobId !== undefined
-                            ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
-                            : {}),
-                    });
-                    await announceQueued(actualName, actualDef, child.id, childData);
-                }
-            }
-            catch (err) {
-                // A retry may still succeed, and a succeeding wrapper reports no completion
-                // that could clear an earlier failure, so only the final attempt is reported.
-                const retries = allJobs.get(actualName)?.retries ?? 0;
-                const finalAttempt = bullJob.attemptsMade + 1 >= retries + 1;
-                logFailedAttempt({
-                    job: rawName,
-                    jobId: bullJob.id ?? "unknown",
-                    attempt: bullJob.attemptsMade + 1,
-                    finalAttempt,
-                    err,
-                });
-                if (finalAttempt) {
-                    await reportPreRunFailure(rawName, bullJob, err);
-                }
-                throw err;
-            }
+        const perTenantTarget = perTenantTargetOf(bullJob.name);
+        if (perTenantTarget !== undefined) {
+            await fanOutPerTenantJob(scope, bullJob, perTenantTarget);
             // skip: fan-out dispatcher job, per-tenant children enqueued
             return;
         }
-        const jobName = rawName;
-        const jobDef = allJobs.get(jobName);
-        if (!jobDef) {
-            const err = new Error(`Unknown job: ${jobName}`);
-            // Final regardless of attempts left: a retry cannot make the job exist.
-            logFailedAttempt({
-                job: jobName,
-                jobId: bullJob.id ?? "unknown",
-                attempt: bullJob.attemptsMade + 1,
-                finalAttempt: true,
-                err,
-            });
-            await reportPreRunFailure(jobName, bullJob, err);
-            throw err;
-        }
-        // maxPerTenant defer: checked before anything is recorded for this run, so a
-        // deferred pickup leaves no span, run-log entry or failure trace and does not
-        // consume an attempt (moveToDelayed + DelayedError, not a retry).
-        const maxPerTenant = resolveMaxPerTenant(jobDef);
-        const deferTenantId = bullJob.data["_tenantId"];
-        if (maxPerTenant?.onLimit === "defer" &&
-            typeof deferTenantId === "string" &&
-            token !== undefined &&
-            bullJob.moveToDelayed &&
-            (await isOverPerTenantLimit(jobName, deferTenantId, maxPerTenant.limit, {
-                activeOnly: true,
-                self: { id: bullJob.id, processedOn: bullJob.processedOn },
-            }))) {
-            const deferCount = priorDeferCountOf(bullJob.data, DEFER_COUNT_KEY);
-            await bullJob.updateData?.({ ...bullJob.data, [DEFER_COUNT_KEY]: deferCount + 1 });
-            await bullJob.moveToDelayed(Date.now() + deferDelayMs(maxPerTenant.deferMs, deferCount), token);
+        const jobDef = await resolveJobDefinitionOrFail(scope, bullJob);
+        if ((await deferOverTenantLimit(scope, jobDef, bullJob, token)) === "deferred") {
             throw new DelayedError();
         }
-        // Sequential gate: try to claim the per-name lock. If another worker
-        // (or this worker on a different bullJob) holds it, re-enqueue with a
-        // small delay and exit *successfully* — using throw would burn the
-        // job's retry budget and pollute failure metrics, but a re-enqueue
-        // looks like an ordinary handoff to BullMQ.
-        let sequentialToken = null;
-        if (jobDef.concurrency === "sequential" && sequentialLock) {
-            const ttlSec = jobDef.timeout
-                ? Math.ceil(jobDef.timeout / 1000) + 5
-                : SEQUENTIAL_DEFAULT_TTL_SEC;
-            sequentialToken = await sequentialLock.acquire(jobName, { ttlSeconds: ttlSec });
-            if (!sequentialToken) {
-                // Re-enqueue onto the job's own lane-queue. In practice that's the
-                // same queue the worker just picked from (since only the consuming
-                // lane runs handleJob at all), but route explicitly — no implicit
-                // coupling to "whichever queue the caller happened to be on".
-                // Attempts already spent travel in _priorAttempts so a lock conflict
-                // on a retry cannot hand out a fresh budget and exceed `retries`.
-                const priorAttempts = priorAttemptsOf(bullJob.data) + bullJob.attemptsMade;
-                const requeuedData = { ...bullJob.data, _priorAttempts: priorAttempts };
-                const requeued = await queues[laneForJob(jobDef)].add(jobName, requeuedData, {
-                    delay: SEQUENTIAL_RETRY_DELAY_MS,
-                    ...buildRetryBullOpts(jobDef, priorAttempts),
-                });
-                if (bullJob.id !== undefined && bullJob.id !== requeued.id) {
-                    await announceDropped(jobName, jobDef, bullJob.id, bullJob.data);
-                }
-                await announceQueued(jobName, jobDef, requeued.id, requeuedData);
-                // skip: lock taken, work re-enqueued with delay, current invocation done
-                return;
-            }
-        }
-        const jobId = bullJob.id ?? "unknown";
-        const startTime = Date.now();
-        const logs = [];
-        // Extract meta from job data. `attempt` is BullMQ's own counter
-        // (1-based on the first run, incremented on each retry) — threading
-        // it through lets the logger tag the run-started event with the
-        // retry number, so audit queries distinguish fresh from retry runs
-        // without peeking at BullMQ internals.
-        const rawData = bullJob.data;
-        const attempt = bullJob.attemptsMade + 1 + priorAttemptsOf(rawData);
-        const meta = {
-            triggeredById: rawData["_triggeredById"], // @cast-boundary dynamic-key
-            payload: rawData["_payload"], // @cast-boundary dynamic-key
-            attempt,
-        };
-        // Build handler payload (without internal meta fields)
-        const payload = handlerPayloadOf(rawData);
-        // Determine tenantId and triggeredBy from meta
-        const tenantId = tenantIdOfJobData(rawData);
-        const triggeredById = rawData["_triggeredById"] ?? null; // @cast-boundary dynamic-key
-        // Tenant-visible failure record (JobDefinition.tenantVisibleFailure). The
-        // subject is read from the handler payload, so it is resolved here where
-        // the payload is built, not in the callbacks.
-        const tenantVisibleDecl = jobDef.tenantVisibleFailure;
-        const tenantVisibleRunDecl = jobDef.tenantVisibleRun;
-        // A bad subject config must surface as a failed run, so it is thrown from
-        // inside runInSpan below instead of here, before any run record exists.
-        let tenantVisibleSubject = null;
-        let tenantVisibleRunSubject = null;
-        let subjectConfigError;
-        if (tenantVisibleRunDecl) {
-            try {
-                tenantVisibleRunSubject = jobSubjectKey(jobName, payload, tenantVisibleRunDecl.subjectFields, "tenantVisibleRun");
-            }
-            catch (err) {
-                subjectConfigError = err instanceof Error ? err : new Error(String(err));
-            }
-        }
-        if (tenantVisibleDecl && !subjectConfigError) {
-            try {
-                tenantVisibleSubject = jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields, "tenantVisibleFailure");
-            }
-            catch (err) {
-                subjectConfigError = err instanceof Error ? err : new Error(String(err));
-            }
-        }
-        // BullMQ stops retrying once attemptsMade reaches the configured attempts
-        // (`retries + 1`), so this is the attempt whose failure is final.
-        const finalAttempt = bullJob.retryable === false || attempt >= (jobDef.retries ?? 0) + 1;
-        const outcomeMeta = (messageKey, unrecoverable = false) => ({
-            tenantId,
-            // BullMQ never retries an UnrecoverableError, whatever attempts are left.
-            finalAttempt: finalAttempt || unrecoverable,
-            ...(tenantVisibleDecl &&
-                !subjectConfigError && {
-                tenantVisible: { subject: tenantVisibleSubject, messageKey },
-            }),
-            ...(tenantVisibleRunDecl &&
-                !subjectConfigError && { tenantVisibleRun: { subject: tenantVisibleRunSubject } }),
-        });
-        // Carry `_triggerName` from rawData when set — handleEvent injects it on
-        // multi-trigger dispatch; exposed as jobContext.triggerName so handlers
-        // don't dig through the raw payload themselves.
-        const triggerName = rawData["_triggerName"]; // @cast-boundary dynamic-key
-        // Same carry-through for `_triggerEvent` — handleEvent injects it when
-        // the job-trigger event consumer supplied the stored event that fired
-        // this run (see createJobTriggerEventConsumer).
-        const rawTriggerEvent = rawData["_triggerEvent"];
-        const triggerEvent = isJobTriggerEvent(rawTriggerEvent) ? rawTriggerEvent : undefined;
-        // Absent = legacy or ungated root; present but invalid fails the run closed.
-        const rawWriteOrigin = rawData["_writeOrigin"]; // @cast-boundary dynamic-key
-        let jobOrigin;
-        let writeOriginInvalid = false;
-        if (rawWriteOrigin !== undefined) {
-            const parsed = parseWriteOrigin(rawWriteOrigin);
-            if (parsed) {
-                jobOrigin = { ...parsed, viaJob: jobName };
-            }
-            else {
-                writeOriginInvalid = true;
-            }
-        }
-        const jobPersonalDataGate = jobOrigin ? buildPersonalDataGate(registry, jobOrigin) : undefined;
-        // Mirror dispatch-shared.ts buildHandlerContext: ctx.files must resolve
-        // through the same _fileProviderResolver for jobs as for write-handlers,
-        // otherwise event-triggered jobs silently get an unresolved ctx.files.
-        const fileResolver = context._fileProviderResolver;
-        const files = fileResolver ? createTenantFileContext(fileResolver, tenantId) : context.files;
-        const jobSystemUser = createSystemUser(tenantId);
-        // Same buildHandlerContext parity as ctx.files above: ctx.notify/ctx.config
-        // must resolve for jobs the same way they do for write-handlers, or an
-        // app-author job calling ctx.notify(...)/ctx.config(...) hits a TypeError
-        // at runtime the write-handler path never would (framework#1532).
-        const notify = context._notifyFactory?.(jobSystemUser, tenantId, selfRunner, undefined, context._fileProviderResolver);
-        const configDb = context.db; // @cast-boundary db-operator
-        // Shared by the config accessor and ctx.derivatives below — both need the
-        // same tenant-scoped db, and building it twice would let the two calls
-        // drift apart. Always "system" mode regardless of the job's own
-        // systemScope() status (pre-existing, not something this change alters)
-        // — isSystemJob below is what actually keeps ctx.systemDb off a
-        // non-system job; it is the only thing standing between this db and an
-        // unchecked cross-tenant escape hatch for such a job. Gated like jobDb so
-        // ctx.systemDb, built from it, is gated too.
-        const tenantScopedDb = configDb
-            ? createTenantDb({
-                db: configDb,
-                tenantId,
-                mode: "system",
-                grants: {
-                    projectionRegistry: registry,
-                    ...(jobPersonalDataGate && { personalDataGate: jobPersonalDataGate }),
-                },
-            })
-            : undefined;
-        const isSystemJob = registry.isJobSystemScoped(jobName);
-        // One reporter for ctx.systemDb and ctx.db.unsafeRaw() so both dedupe in the same window.
-        const isSystemCronRun = triggeredById === null && jobDef.escapeHatch !== undefined && "cron" in jobDef.trigger;
-        const reportEscapeHatch = createEscapeHatchReporter({
-            handler: jobName,
-            tenantId,
-            actor: jobSystemUser.id,
-            sink: context._escapeHatchAuditSink,
-            log: errorLogger,
-            window: escapeHatchReportWindow,
-            ...(triggeredById !== null && { caller: triggeredById }),
-            ...(isSystemCronRun && { processDedup: systemCronEscapeHatchDedup }),
-            ...(context.meter && { meter: context.meter }),
-        });
-        const systemDb = isSystemJob && tenantScopedDb
-            ? createUncheckedSystemDb(tenantScopedDb, undefined, reportEscapeHatch)
-            : undefined;
-        const jobDb = configDb
-            ? createTenantDb({
-                db: configDb,
-                tenantId,
-                mode: "tenant",
-                tracer: context.tracer,
-                meter: context.meter,
-                grants: {
-                    unsafeRaw: escapeHatchFor(jobDef.escapeHatch, "unsafeRaw"),
-                    globalWrites: escapeHatchFor(jobDef.escapeHatch, "globalWrites"),
-                    report: reportEscapeHatch,
-                    projectionRegistry: registry,
-                    ...(jobPersonalDataGate && { personalDataGate: jobPersonalDataGate }),
-                },
-            })
-            : undefined;
-        const crossTenantReadsReason = crossTenantReadsGrantFor(jobDef.escapeHatch)?.reason;
-        const crossTenantReads = tenantScopedDb && crossTenantReadsReason !== undefined
-            ? createCrossTenantReads(tenantScopedDb, crossTenantReadsReason, reportEscapeHatch)
-            : undefined;
-        const configFactory = configAccessorFactoryOf(context);
-        const config = configFactory && tenantScopedDb
-            ? configFactory({
-                user: { id: jobSystemUser.id, tenantId },
-                db: tenantScopedDb,
-                secrets: context.secrets,
-            })
-            : undefined;
-        // Mirror dispatch-shared.ts: ctx.derivatives needs files+db, same
-        // tenant-scoped db the config accessor above uses.
-        const derivatives = files && tenantScopedDb
-            ? createDerivativesContext({
-                files,
-                registry,
-                db: tenantScopedDb,
-                tenantId,
-            })
-            : context.derivatives;
-        // Per-job-run reader, created lazily on queryAsMember's first call and
-        // cached for the rest of this run.
-        let memberReader;
-        const identitySwitchHatch = escapeHatchFor(jobDef.escapeHatch, "systemIdentity");
-        const identitySwitchGrantReason = identitySwitchHatch?.reason ??
-            (isSystemJob ? `r.systemScope() feature of job "${jobName}"` : undefined);
-        const ungatedQueryAs = (user, qn, payload) => {
-            if (!dispatchWriteRef) {
-                throw new Error("JobContext.queryAs called before dispatcher attached — call attachDispatcher() first");
-            }
-            return dispatchWriteRef.queryAs(user, qn, payload, jobOrigin);
-        };
-        const gatedIdentitySwitch = createGatedIdentitySwitch(`job "${jobName}"`, jobSystemUser, isSystemJob || identitySwitchHatch !== undefined, {
-            queryAs: ungatedQueryAs,
-            writeAs: async (user, qn, payload) => {
-                if (!dispatchWriteRef) {
-                    throw new Error("JobContext.writeAs called before dispatcher attached — call attachDispatcher() first");
-                }
-                return dispatchWriteRef.write(user, qn, payload, jobOrigin);
-            },
-        }, { reason: identitySwitchGrantReason, report: reportEscapeHatch });
-        const jobContextWithoutTz = {
-            defer: (delayMs) => {
-                if (!Number.isInteger(delayMs) || delayMs <= 0) {
-                    throw new RangeError(`Job "${jobName}": ctx.defer needs a positive integer delay in ms (got ${delayMs})`);
-                }
-                if (token === undefined || !bullJob.moveToDelayed) {
-                    throw new InternalError({
-                        message: `Job "${jobName}": ctx.defer needs a queued run; it is not available inline (boot gate).`,
-                    });
-                }
-                return { kind: "job-deferral", delayMs };
-            },
-            ...context,
-            db: jobDb ?? missingDbProxy(jobName),
-            files,
-            derivatives,
-            ...(notify !== undefined && { notify }),
-            ...(config !== undefined && { config }),
-            ...(systemDb && { systemDb }),
-            ...(crossTenantReads && { crossTenantReads }),
-            // The runner owns the registry it resolved this job from — expose it so
-            // workers can reach projections/jobs without the app author duplicating
-            // it into `context` (the JobContext contract guarantees `registry`).
-            registry,
-            // Expose the runner so handlers can chain a follow-up job. Symmetric with
-            // how the command-dispatcher hands write-handlers their jobRunner.
-            ...(selfRunner !== undefined && { jobRunner: selfRunner }),
-            systemUser: jobSystemUser,
-            triggeredBy: triggeredById !== null ? { id: triggeredById, tenantId } : null,
-            attempt,
-            finalAttempt,
-            log: createJobLogger(logs),
-            metricsFor: createMetricsForFactory(context.meter),
-            ...(triggerName !== undefined && { triggerName }),
-            ...(triggerEvent !== undefined && { triggerEvent }),
-            write: (qn, payload) => {
-                if (!dispatchWriteRef) {
-                    throw new Error("JobContext.write called before dispatcher attached — call attachDispatcher() first");
-                }
-                return dispatchWriteRef.write(jobSystemUser, qn, payload, jobOrigin);
-            },
-            writeAs: gatedIdentitySwitch.writeAs,
-            queryAs: gatedIdentitySwitch.queryAs,
-            queryAsMember: (userId, qn, payload) => {
-                if (!dispatchWriteRef) {
-                    throw new Error("JobContext.queryAsMember called before dispatcher attached — call attachDispatcher() first");
-                }
-                // Lazy per-run reader, mirroring the handler-context reader.
-                memberReader ??= dispatchWriteRef.createMemberReader(tenantId);
-                return memberReader(userId, qn, payload);
-            },
-        };
-        if (tenantVisibleRunDecl && !subjectConfigError) {
-            meta.tenantVisibleRun = { tenantId, subject: tenantVisibleRunSubject };
+        const slot = await claimSequentialSlot(scope, jobDef, bullJob);
+        if (slot.kind === "requeued") {
+            // skip: lock taken, work re-enqueued with delay, current invocation done
+            return;
         }
-        await options.onJobStart?.(jobName, jobId, meta);
-        // Cross-process trace continuation: if the enqueuing code captured a
-        // parent span, start the job.execute span as its child. Works for event
-        // and manual triggers; cron jobs start a fresh root span.
-        const parentContext = readTraceContext(rawData);
-        // Correlation propagation: the scheduling request's correlationId was
-        // packed into _correlationId at dispatch time. Re-enter requestContext.run
-        // so event writes during this job stamp the same correlation as the
-        // request that scheduled it. Cron/boot jobs (no scheduler) start fresh
-        // — correlationId = new requestId, no parent causation.
-        const inheritedCorrelationId = rawData["_correlationId"] ?? undefined; // @cast-boundary dynamic-key
-        const jobRequestId = requestContext.generateId();
-        const jobCorrelationId = inheritedCorrelationId ?? jobRequestId;
-        const deferred = {};
-        const runInSpan = async () => {
-            try {
-                if (subjectConfigError) {
-                    // Retrying cannot fix a declaration error.
-                    throw new UnrecoverableError(subjectConfigError.message);
-                }
-                if (writeOriginInvalid) {
-                    throw new InternalError({
-                        message: `Job "${jobName}" received an unparseable _writeOrigin — refusing to run without ` +
-                            "a trustworthy anonymous-root gate.",
-                    });
-                }
-                const handlerResult = await requestContext.run({
-                    requestId: jobRequestId,
-                    correlationId: jobCorrelationId,
-                    // #3043 — events a job writes carry the job as their origin.
-                    handler: jobName,
-                    feature: registry.getJobFeature(jobName) ?? qnScope(jobName),
-                    writeOrigin: jobOrigin,
-                    tenantId,
-                    ...(triggeredById !== null && { userId: triggeredById }),
-                }, async () => jobDef.handler(payload, {
-                    ...jobContextWithoutTz,
-                    tz: await resolveJobTz(config, context),
-                }));
-                if (isJobDeferral(handlerResult)) {
-                    const maxDefers = jobDef.maxDefers ?? DEFAULT_MAX_DEFERS;
-                    if (priorDeferCountOf(bullJob.data, HANDLER_DEFER_COUNT_KEY) >= maxDefers) {
-                        // The counter never resets, so a retry would hit the cap again.
-                        throw new UnrecoverableError(`Job "${jobName}" was deferred ${maxDefers} times (maxDefers); giving up.`);
-                    }
-                    deferred.deferral = handlerResult;
-                    // skip: the deferral is applied after the span closes, below
-                    return;
-                }
-                // Stamped before the observer hook: a throwing onJobComplete must not
-                // make a run that actually succeeded look dead to the liveness alert.
-                if (context.meter) {
-                    try {
-                        emitJobLastSuccess(context.meter, jobName);
-                    }
-                    catch (err) {
-                        errorLogger.warn("job last-success metric failed", {
-                            job: jobName,
-                            error: err instanceof Error ? err.message : String(err),
-                        });
-                    }
-                }
-                const duration = Date.now() - startTime;
-                await options.onJobComplete?.(jobName, jobId, duration, logs, outcomeMeta(null));
-            }
-            catch (err) {
-                const errorMsg = err instanceof Error ? err.message : String(err);
-                logs.push({ level: "error", message: errorMsg, timestamp: Temporal.Now.instant() });
-                logFailedAttempt({ job: jobName, jobId, attempt, finalAttempt, err });
-                await options.onJobFailed?.(jobName, jobId, errorMsg, logs, outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl), err instanceof UnrecoverableError));
-                throw err;
-            }
-        };
-        // Unified span creation: withSpan handles start/end + status/exception
-        // recording identically for both parent-context and no-parent paths.
-        // When parentContext is set, the new parent-aware StartSpanOptions
-        // plumbs it through to startSpan — no manual try/finally needed.
         try {
-            await withEscapeHatchAuditScope(() => tracer.withSpan("job.execute", {
-                attributes: {
-                    "job.name": jobName,
-                    "job.id": jobId,
-                    "job.attempt": attempt,
-                    "kumiko.tenant_id": tenantId,
-                    // Lane-routing attributes (Welle 2.6). `run_in` is the job's
-                    // declared lane (explicit or default-"worker"); `consumer_lane`
-                    // is which runner actually executed it. They diverge in
-                    // all-in-one (both lanes live in one process) but must match
-                    // in split deploys — a mismatch in prod logs signals a
-                    // misrouted job that slipped past the boot-validator.
-                    "kumiko.job.run_in": laneForJob(jobDef),
-                    // Omit attribute entirely when no consumer (enqueuer-only runner) —
-                    // SpanAttributeValue doesn't accept undefined.
-                    ...(consumerLane !== undefined ? { "kumiko.job.consumer_lane": consumerLane } : {}),
-                },
-                ...(parentContext ? { parent: parentContext } : {}),
-            }, runInSpan));
-            if (deferred.deferral && token !== undefined && bullJob.moveToDelayed) {
-                // Run-log bookkeeping must not turn a defer into a failure.
-                try {
-                    await options.onJobDeferred?.(jobName, jobId, meta);
-                }
-                catch (err) {
-                    errorLogger.warn("job deferred hook failed", {
-                        job: jobName,
-                        jobId,
-                        error: scrubLogText(err instanceof Error ? err.message : String(err)),
-                    });
-                }
-                await bullJob.updateData?.({
-                    ...bullJob.data,
-                    [HANDLER_DEFER_COUNT_KEY]: priorDeferCountOf(bullJob.data, HANDLER_DEFER_COUNT_KEY) + 1,
-                });
-                await bullJob.moveToDelayed(Date.now() + deferred.deferral.delayMs, token);
+            const run = await startJobRun(scope, jobDef, bullJob, token);
+            const deferral = await executeJobInSpan(scope, run);
+            if ((await applyHandlerDeferral(scope, run, bullJob, deferral, token)) === "deferred") {
                 throw new DelayedError();
             }
         }
         finally {
-            // Release the sequential lock value-matched (Lua compare-and-delete
-            // inside DistributedLock). A TTL-expired lock that's been claimed by
-            // a different owner stays put — releasing it would break sequencing
-            // for the new owner.
-            if (sequentialToken && sequentialLock) {
-                await sequentialLock.release(jobName, sequentialToken);
-            }
+            await releaseSequentialSlot(scope, bullJob.name, slot.lockToken);
         }
     }
     // Boot gates run inline and are awaited, so a throwing gate rejects start()
     // — and with it the entrypoint's start() and runProdApp's boot, before the
     // process reports ready. Enqueuing them (as runOnBoot does) would only fail
-    // a queue job and let the deploy through (fw#2590).
+    // a queue job and let the deploy through.
     async function runBootGates(lane) {
         for (const [name, jobDef] of allJobs) {
             if (laneForJob(jobDef) !== lane)
@@ -900,7 +288,7 @@ export function createJobRunner(options) {
                 continue;
             if ("cron" in jobDef.trigger) {
                 await queue.upsertJobScheduler(schedulerIdForJobName(name), cronRepeatOptions(jobDef.trigger), {
-                    name: jobDef.perTenant ? `_perTenant:${name}` : name,
+                    name: jobDef.perTenant ? `${PER_TENANT_PREFIX}${name}` : name,
                     data: {},
                     // Queue-level defaultJobOptions (jobRetentionOpts) covers
                     // retention; a count here would sweep queue-wide again and
@@ -978,7 +366,7 @@ export function createJobRunner(options) {
             // still-settling blocking connection: it rejects in-flight commands
             // via ioredis's flushQueue() during close(), which isn't a listenable
             // 'error' event — the only fix is to not return until both of the
-            // worker's connections (main + blocking) are ready (fw#1805). This
+            // worker's connections (main + blocking) are ready. This
             // mirrors the wait BullMQ already does internally for
             // upsertJobScheduler()/add() below when the lane has a cron/boot job.
             // Racing a timeout against it keeps an unreachable Redis from hanging
@@ -1012,7 +400,6 @@ export function createJobRunner(options) {
                 });
             });
             await upsertCronSchedulers(consumerQueue, consumerLane);
-            schedulerSeenTimer = startSchedulerSeenRefresh(consumerQueue, consumerLane, expectedSchedulerIds);
             await removeOrphanJobSchedulers(consumerQueue, expectedSchedulerIds, consumerLane, {
                 info: (msg, data) => context.log ? context.log.info(msg, data) : errorLogger.debug(msg, data),
                 warn: errorLogger.warn,
@@ -1035,7 +422,7 @@ export function createJobRunner(options) {
                 if (laneForJob(jobDef) !== consumerLane)
                     continue;
                 if (jobDef.runOnBoot) {
-                    const bootName = jobDef.perTenant ? `_perTenant:${name}` : name;
+                    const bootName = jobDef.perTenant ? `${PER_TENANT_PREFIX}${name}` : name;
                     const bootJobId = bootJobIdForJobName(name);
                     const client = await consumerQueue.client;
                     const alreadyEnqueued = await client.hexists(bootEnqueuedKey, bootJobId);
@@ -1053,6 +440,8 @@ export function createJobRunner(options) {
             if (context.meter) {
                 queueDepthTimer = await startQueueDepthPolling(consumerQueue, consumerLane, context.meter);
             }
+            // Last on purpose: a start() that fails earlier must not leave a timer stamping for a runner that is not running.
+            schedulerSeenTimer = startSchedulerSeenRefresh(consumerQueue, consumerLane, expectedSchedulerIds);
         },
         async stop() {
             stopping = true;
@@ -1070,7 +459,7 @@ export function createJobRunner(options) {
             }
             // BullMQ close() while a Queue's connection is still 'initializing'
             // rejects the in-flight INFO with "Connection is closed." on an emitter
-            // without listeners -> unhandled rejection (fw#1805). Wait for readiness
+            // without listeners -> unhandled rejection. Wait for readiness
             // first; a failed/timed-out wait must not block the close itself.
             const stopReadyTimeoutMs = Math.min(bootRedisTimeoutMs, STOP_QUEUE_READY_TIMEOUT_MS);
             await settleAbandonedBootGates(stopReadyTimeoutMs);
@@ -1112,7 +501,7 @@ export function createJobRunner(options) {
             if (jobDef.perTenant) {
                 const perTenantData = { ...(payload ?? {}) };
                 stampGatedWriteOrigin(perTenantData);
-                const job = await targetQueue.add(`_perTenant:${jobName}`, perTenantData, buildRetryBullOpts(jobDef));
+                const job = await targetQueue.add(`${PER_TENANT_PREFIX}${jobName}`, perTenantData, buildRetryBullOpts(jobDef));
                 return job.id ?? "unknown";
             }
             // maxPerTenant guard: cap concurrent + waiting jobs of the same name
@@ -1126,7 +515,7 @@ export function createJobRunner(options) {
             if (maxPerTenant?.onLimit === "skip") {
                 const tenantId = payload?._tenantId; // @cast-boundary dynamic-key
                 if (tenantId !== undefined &&
-                    (await isOverPerTenantLimit(jobName, tenantId, maxPerTenant.limit))) {
+                    (await isOverPerTenantLimit(scope, jobName, tenantId, maxPerTenant.limit))) {
                     return "skipped:max-per-tenant";
                 }
             }
@@ -1147,7 +536,7 @@ export function createJobRunner(options) {
                     for (const j of waiting) {
                         if (j.name === jobName && j.id) {
                             await j.remove();
-                            await announceDropped(jobName, jobDef, j.id, j.data);
+                            await announceDropped(scope, jobName, jobDef, j.id, j.data);
                         }
                     }
                     break;
@@ -1190,7 +579,7 @@ export function createJobRunner(options) {
                 data["_correlationId"] = reqCtx.correlationId;
             stampGatedWriteOrigin(data);
             const job = await targetQueue.add(jobName, data, bullOpts);
-            await announceQueued(jobName, jobDef, job.id, data);
+            await announceQueued(scope, jobName, jobDef, job, data);
             return job.id ?? "unknown";
         },
         async handleEvent(eventName, payload, user, triggerEvent) {
@@ -1203,8 +592,8 @@ export function createJobRunner(options) {
             for (const [name, jobDef] of allJobs) {
                 if (!("on" in jobDef.trigger))
                     continue;
-                // skip: andere Trigger-Formen (cron, manual) reagieren nicht auf
-                // Events. Nur "on"-Trigger werden hier matched.
+                // skip: other trigger types (cron, manual) do not respond to events.
+                // Only "on" triggers are matched here.
                 const triggerOn = jobDef.trigger.on;
                 const matches = Array.isArray(triggerOn)
                     ? triggerOn.includes(eventName)
@@ -1218,10 +607,9 @@ export function createJobRunner(options) {
                     data["_tenantId"] = user.tenantId;
                     data["_triggeredById"] = user.id;
                 }
-                // Multi-Trigger: payload bekommt _triggerName damit der Handler
-                // weiß, welcher der N Trigger gefeuert hat. Bei Single-Trigger
-                // setzen wir es auch — kostet nichts und vereinfacht Handler-Code
-                // (kein "ist es Multi?"-Branch nötig).
+                // Multi-trigger: add _triggerName to the payload so the handler knows which
+                // of the N triggers fired. Set it for single triggers too; this costs
+                // nothing and keeps handler code simple (no "is it multi?" branch).
                 data["_triggerName"] = eventName;
                 if (triggerEvent)
                     data["_triggerEvent"] = triggerEvent;
@@ -1233,7 +621,7 @@ export function createJobRunner(options) {
                 // jobs must respect the per-tenant cap or the limit is one-sided.
                 const maxPerTenant = resolveMaxPerTenant(jobDef);
                 if (maxPerTenant?.onLimit === "skip" && user?.tenantId !== undefined) {
-                    if (await isOverPerTenantLimit(name, String(user.tenantId), maxPerTenant.limit)) {
+                    if (await isOverPerTenantLimit(scope, name, String(user.tenantId), maxPerTenant.limit)) {
                         continue;
                     }
                 }
@@ -1241,7 +629,7 @@ export function createJobRunner(options) {
                 // Route to the job's declared lane, not a fixed queue — that's
                 // the whole reason both queues are held.
                 const job = await queues[laneForJob(jobDef)].add(name, data, buildRetryBullOpts(jobDef));
-                await announceQueued(name, jobDef, job.id, data);
+                await announceQueued(scope, name, jobDef, job, data);
             }
         },
         attachDispatcher(ref) {