@cosmicdrift/kumiko-framework 0.327.0 → 0.328.1

package/dist/jobs/job-runner.js

@@ -0,0 +1,1110 @@
+import { Queue, Worker } from "bullmq";
+import { Redis } from "ioredis";
+import { requestContext } from "../api/request-context.js";
+import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
+import { createDerivativesContext } from "../derivatives/derivatives-context.js";
+import { qnScope } from "../engine/qualified-name.js";
+import { createSystemUser } from "../engine/system-user.js";
+import { SYSTEM_TENANT_ID, } from "../engine/types/index.js";
+import { InternalError } from "../errors/index.js";
+import { isKumikoError } from "../errors/kumiko-error.js";
+import { createFileContext } from "../files/file-handle.js";
+import { createFallbackLogger } from "../logging/index.js";
+import { createEscapeHatchReporter } from "../observability/escape-hatch-report.js";
+import { emitJobLastSuccess, emitJobQueueDepth, getFallbackTracer, } from "../observability/index.js";
+import { createDistributedLock } from "../pipeline/distributed-lock.js";
+import { RedisKeys } from "../pipeline/redis-keys.js";
+import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
+import { bridgeStub } from "../testing/handler-context.js";
+// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
+// (a security-relevant escape hatch that must fail the run closed), a
+// malformed `_triggerEvent` only costs an idempotency-key convenience — the
+// run proceeds without it rather than failing over what would be a framework
+// bug, not untrusted input.
+function isJobTriggerEvent(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const candidate = value;
+    if (typeof candidate["id"] !== "string")
+        return false;
+    const headers = candidate["headers"];
+    if (typeof headers !== "object" || headers === null)
+        return false;
+    return Object.values(headers).every((v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean");
+}
+// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
+function stampGatedWriteOrigin(data) {
+    delete data["_writeOrigin"];
+    const origin = requestContext.get()?.writeOrigin;
+    if (origin && isPersonalDataGated(origin)) {
+        data["_writeOrigin"] = origin;
+    }
+}
+// Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
+// ("kumiko-jobs") — it must match between enqueuers and consumers, and an
+// accidental drift would silently drop jobs. Tests override via
+// `queueNamePrefix` for per-run isolation (stale jobs from a prior run
+// don't leak into a new test because the queue name includes a timestamp).
+const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
+function queueNameFor(prefix, lane) {
+    return `${prefix}-${lane}`;
+}
+// perTenant jobs need a source of truth for "which tenants are active" —
+// see createJobRunner's defaultGetActiveTenantIds below. The tenant feature
+// (bundled-features package) registers a query under this exact name when
+// mounted. framework must not depend on bundled-features (wrong direction —
+// bundled-features already depends on framework), so this is a name-only
+// coupling: createJobRunner looks the handler up in the registry by string
+// instead of importing it.
+const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
+/**
+ * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
+ * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
+ * treated as old repeatables — spawning a new scheduler entry every tick and
+ * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
+ * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
+ */
+export function schedulerIdForJobName(jobName) {
+    return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
+}
+// Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
+// a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
+export function bootJobIdForJobName(jobName) {
+    return `boot-${jobName.replace(/[.:]/g, "-")}`;
+}
+// ponytail: migration shim, remove after fw#1603 deploy is everywhere.
+function legacySchedulerIdForJobName(jobName) {
+    return `scheduler-${jobName.replace(/\./g, "-")}`;
+}
+// Deterministic per-tenant child job id, derived from the perTenant
+// wrapper's own BullMQ job id. A cron tick's wrapper id is
+// `repeat:<schedulerId>:<millis>` (stable for that tick, including across
+// BullMQ's own retry/stalled-job redelivery of the *same* job); a manual
+// dispatch's wrapper id is BullMQ's auto-generated id, equally stable
+// across retries of that job. Reusing it means a wrapper that runs twice
+// for the same trigger (retry after failure, a Redis drop, an instance
+// restarting mid-run) re-derives the *same* child ids, and BullMQ's
+// existing-jobId add() no-ops the second batch instead of creating
+// duplicates — safe only as long as the child job hash outlives the
+// wrapper's retry window, enforced at construction against
+// COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
+// createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
+// (fw#1603/#1604) — the wrapper id already contains ":", so strip
+// separators from both halves before joining instead of interpolating raw.
+function perTenantChildJobId(wrapperJobId, tenantId) {
+    const sanitize = (value) => value.replace(/[.:]/g, "-");
+    return `child-${sanitize(wrapperJobId)}-${sanitize(tenantId)}`;
+}
+function createJobLogger(logs, baseContext = {}) {
+    function push(level, msg, data) {
+        const merged = { ...baseContext, ...data };
+        const message = Object.keys(merged).length > 0 ? `${msg} ${JSON.stringify(merged)}` : msg;
+        logs.push({ level, message, timestamp: Temporal.Now.instant() });
+    }
+    const logger = {
+        info(msg, data) {
+            push("info", msg, data);
+        },
+        warn(msg, data) {
+            push("warn", msg, data);
+        },
+        error(msg, data) {
+            push("error", msg, data);
+        },
+        // No-op by design, not missing coverage: JobLogEntry.level (and the
+        // persisted run-completed/-failed event schema in bundled-features/
+        // jobs/events.ts) only has info|warn|error — a job's debug() calls have
+        // nowhere durable to land.
+        debug() { },
+        child(context) {
+            return createJobLogger(logs, { ...baseContext, ...context });
+        },
+    };
+    return logger;
+}
+// Stable identity for the declared payload fields: sorted keys, so two runs
+// with the same subject values produce the same string and the later one
+// replaces the earlier record. Non-primitives would serialize into something
+// no caller can reconstruct for a lookup, so they fail the run loudly.
+function jobSubjectKey(jobName, payload, fields) {
+    if (fields === undefined || fields.length === 0)
+        return null;
+    const entries = [];
+    for (const field of [...fields].sort()) {
+        const value = payload[field] ?? null;
+        if (value !== null && typeof value === "object") {
+            throw new Error(`Job "${jobName}": tenantVisibleFailure.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
+        }
+        entries.push([field, value]);
+    }
+    return JSON.stringify(entries);
+}
+// Only a translation key ever travels to the tenant: the thrown error's own
+// i18nKey when it carries one, otherwise the key declared at the job. The
+// error message itself can echo provider or payload content and stays on the
+// run row and in the run log.
+function tenantFailureMessageKey(err, declaration) {
+    // skip: job did not opt in — nothing tenant-visible to record
+    if (!declaration)
+        return null;
+    if (isKumikoError(err) && err.i18nKey.length > 0)
+        return err.i18nKey;
+    return declaration.messageKey;
+}
+// Serialized trace context lives under this key in the BullMQ job data.
+// Leading underscore matches the existing internal-meta convention
+// (_triggeredById, _tenantId, _payload).
+const TRACE_CONTEXT_KEY = "_traceContext";
+function readTraceContext(data) {
+    const raw = data[TRACE_CONTEXT_KEY];
+    if (!raw || typeof raw !== "object")
+        return undefined;
+    const ctx = raw; // @cast-boundary engine-payload
+    if (!ctx.traceId || !ctx.spanId)
+        return undefined;
+    return { traceId: ctx.traceId, spanId: ctx.spanId };
+}
+function captureTraceContext(tracer) {
+    const span = tracer.getActiveSpan();
+    if (!span?.traceId)
+        return undefined;
+    return { traceId: span.traceId, spanId: span.spanId };
+}
+const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
+// kumiko_job_queue_depth: only the lane's own consumer polls — the count is
+// a global Redis-backed value (BullMQ, not per-process), so one reporter per
+// lane is enough. Fires once immediately (metrics exist before the first
+// poll tick) then on an interval; the caller stores + clears the handle.
+async function startQueueDepthPolling(queue, lane, meter) {
+    const pollQueueDepth = async () => {
+        try {
+            const counts = await queue.getJobCounts("waiting", "active", "delayed", "failed", "paused");
+            emitJobQueueDepth(meter, lane, counts);
+        }
+        catch {
+            // skip: transient Redis hiccup — next poll retries
+        }
+    };
+    await pollQueueDepth();
+    return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
+}
+function parseRedisOpts(url) {
+    const parsed = new URL(url);
+    const result = {
+        host: parsed.hostname,
+        port: Number(parsed.port) || 6379,
+    };
+    if (parsed.pathname.length > 1) {
+        result.db = Number(parsed.pathname.slice(1));
+    }
+    return result;
+}
+// redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
+// would otherwise hang start() forever with no health endpoint to notice.
+const BOOT_REDIS_TIMEOUT_MS = 10_000;
+function timeoutReject(ms, message) {
+    let timer;
+    const promise = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(message)), ms);
+    });
+    return {
+        promise,
+        cancel: () => {
+            if (timer !== undefined)
+                clearTimeout(timer);
+        },
+    };
+}
+// Default base delay when a job opts into backoff without an explicit
+// delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
+// NaN/undefined, which is falsy — the job retries immediately instead of
+// waiting.
+const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
+// Shared by every enqueue path (dispatch(), handleEvent(), cron, runOnBoot,
+// perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
+// set must retry the same way regardless of how it got enqueued, or it
+// fails for good on the very first error on whichever path skips this.
+function buildRetryBullOpts(jobDef) {
+    const opts = {};
+    if (jobDef.retries !== undefined)
+        opts.attempts = jobDef.retries + 1;
+    if (jobDef.backoff) {
+        opts.backoff =
+            typeof jobDef.backoff === "string"
+                ? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
+                : {
+                    type: jobDef.backoff.type,
+                    delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
+                };
+    }
+    return opts;
+}
+// Equality filter on top-level payload fields for a job's trigger.where —
+// lets N jobs share one broad event QN, partitioned by a payload
+// discriminant instead of needing N distinct event types.
+function payloadMatchesTriggerFilter(where, payload) {
+    if (!where)
+        return true;
+    return Object.entries(where).every(([key, value]) => payload[key] === value);
+}
+// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
+// OTHER jobs in the same completed/failed zset (moveToFinished lua,
+// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
+// the sweep only runs lazily on a later finish into that set. Per-job
+// retention is therefore meaningless on a shared queue — only queue-wide,
+// age-only retention (no count) is safe: a count would evict perTenant
+// children still inside their wrapper's retry window, and boot jobs. The
+// completed age must stay above every perTenant wrapper's own worst-case
+// retry window (see maxRetryWindowMs and the invariant check in
+// createJobRunner below), because child dedup (perTenantChildJobId) relies
+// on the children still existing in Redis when a wrapper retry re-derives
+// their ids. The margin here is generous — queue wait time isn't bounded by
+// backoff — to cover realistic windows. Audit trail lives in read_job_runs,
+// not BullMQ (fw#3199).
+const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
+const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
+// Total wall-clock time BullMQ can hold a perTenant wrapper across all its
+// retries, consistent with buildRetryBullOpts's own backoff computation.
+function maxRetryWindowMs(jobDef) {
+    if (!jobDef.backoff || jobDef.retries === undefined)
+        return 0;
+    const delayMs = (typeof jobDef.backoff === "string" ? undefined : jobDef.backoff.delayMs) ??
+        DEFAULT_JOB_BACKOFF_DELAY_MS;
+    const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
+    return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
+}
+export function createJobRunner(options) {
+    const { registry, context, redisUrl, consumerLane } = options;
+    const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;
+    const bootRedisTimeoutMs = options.bootRedisTimeoutMs ?? BOOT_REDIS_TIMEOUT_MS;
+    const redisOpts = parseRedisOpts(redisUrl);
+    // Use the context's tracer when present (observability-provider injected at
+    // boot); otherwise noop so dispatch/handleJob stay zero-cost without config.
+    const tracer = context.tracer ?? getFallbackTracer();
+    const errorLogger = createFallbackLogger("job-runner", context.log);
+    // Set at the top of stop() — a graceful shutdown closes the redis/BullMQ
+    // clients itself, which fires the exact same 'error' listeners below with
+    // an expected "Connection is closed." Downgrading to debug once stopping
+    // is true keeps those out of error-rate alerts without losing them.
+    let stopping = false;
+    // Default for perTenant fan-out when the caller didn't supply
+    // options.getActiveTenantIds: if the tenant feature is mounted (its
+    // active-tenant-ids query is in the registry), resolve tenants through
+    // that instead of forcing every app to wire the option by hand. Same
+    // hand-built-context pattern delivery-service.ts uses to invoke a
+    // registered handler without a live dispatcher — no dispatcher exists yet
+    // at createJobRunner() time, and this call runs with system-level access
+    // by construction, so going through the dispatcher's per-request identity
+    // checks would add nothing.
+    function defaultGetActiveTenantIds() {
+        const handler = registry.getQueryHandler(ACTIVE_TENANT_IDS_QUERY_NAME);
+        const db = context.db; // @cast-boundary db-operator
+        if (!handler || !db)
+            return undefined;
+        return async () => {
+            const systemUser = createSystemUser(SYSTEM_TENANT_ID);
+            const systemModeDb = createTenantDb(db, SYSTEM_TENANT_ID, "system");
+            const reportEscapeHatch = createEscapeHatchReporter({
+                handler: ACTIVE_TENANT_IDS_QUERY_NAME,
+                tenantId: SYSTEM_TENANT_ID,
+                actor: systemUser.id,
+                sink: context._escapeHatchAuditSink,
+                log: context.log,
+            });
+            const result = await handler.handler({ type: ACTIVE_TENANT_IDS_QUERY_NAME, payload: {}, user: systemUser }, {
+                db: systemModeDb,
+                dbOutsideTransaction: systemModeDb,
+                systemDb: createUncheckedSystemDb(systemModeDb, undefined, reportEscapeHatch),
+                registry,
+                ...bridgeStub(),
+            });
+            // @cast-boundary engine-payload — generic query-handler return for typed convention
+            return result;
+        };
+    }
+    const getActiveTenantIds = options.getActiveTenantIds ?? defaultGetActiveTenantIds();
+    const allJobs = registry.getAllJobs();
+    function positiveIntRetentionSec(value, fallback, label) {
+        const resolved = value ?? fallback;
+        if (!Number.isInteger(resolved) || resolved <= 0) {
+            throw new Error(`job-runner: jobRetention.${label} must be a positive integer, got ${resolved}`);
+        }
+        return resolved;
+    }
+    const completedAgeSec = positiveIntRetentionSec(options.jobRetention?.completedAgeSec, COMPLETED_JOB_RETENTION_AGE_SEC, "completedAgeSec");
+    const failedAgeSec = positiveIntRetentionSec(options.jobRetention?.failedAgeSec, FAILED_JOB_RETENTION_AGE_SEC, "failedAgeSec");
+    // perTenant child dedup (perTenantChildJobId) only holds as long as the
+    // children are still in Redis when a wrapper retry re-derives their ids,
+    // which requires the completed-job retention to outlast the wrapper's own
+    // worst-case retry window. Enforced here, before any Redis connection
+    // opens below, so a misconfigured job fails boot loudly instead of
+    // silently losing dedup in prod.
+    for (const [name, jobDef] of allJobs) {
+        if (!jobDef.perTenant)
+            continue;
+        const windowMs = maxRetryWindowMs(jobDef);
+        if (windowMs >= completedAgeSec * 1000) {
+            throw new Error(`job-runner: perTenant job "${name}" has a retry window of ${windowMs}ms, which is >= ` +
+                `the completed-job retention (${completedAgeSec}s). Child dedup relies on children ` +
+                "staying in Redis for the whole retry window — lower retries/backoff or raise " +
+                "jobRetention.completedAgeSec.");
+        }
+    }
+    // Resolve the lane for a job — "worker" is the default because that's the
+    // sensible prod lane (heavy async off the request path). Jobs that opted
+    // into "api" must have been validated at registry boot already.
+    function laneForJob(def) {
+        return def.runIn ?? "worker";
+    }
+    // Sequential coordination: BullMQ OSS has no `group`, so we serialise
+    // same-name jobs ourselves with a per-name Redis lock. Only built when at
+    // least one job actually requested it — keeps the no-sequential boot path
+    // free of the extra Redis client. Scoped under the consumer lane so two
+    // runners on different lanes cannot collide on the same lock-key for
+    // unrelated jobs.
+    const hasSequential = [...allJobs.values()].some((def) => def.concurrency === "sequential");
+    let lockRedis = null;
+    let sequentialLock = null;
+    if (hasSequential) {
+        lockRedis = new Redis(redisOpts);
+        // Without a listener, a post-close 'error' (e.g. a teardown-race
+        // "Connection is closed") is unhandled and crashes the process — in
+        // bun:test it gets attributed to whichever test happens to run next
+        // (fw#1805).
+        lockRedis.on("error", (err) => {
+            const log = stopping ? errorLogger.debug : errorLogger.error;
+            log("lock redis connection error", { error: err.message });
+        });
+        const lockScope = consumerLane ?? "enqueue";
+        // Scoped by queueNamePrefix like the queues themselves, so runners
+        // isolated by prefix on a shared Redis don't contend for the same lock.
+        sequentialLock = createDistributedLock(lockRedis, `${RedisKeys.lock}seq:${queueNamePrefix}:${lockScope}:`);
+    }
+    // Default lock-TTL for sequential jobs that didn't declare a timeout.
+    // 5 minutes matches BullMQ's default stalledInterval — long enough for
+    // any reasonable handler, short enough that a crashed worker recovers
+    // without manual intervention.
+    const SEQUENTIAL_DEFAULT_TTL_SEC = 305;
+    // How long to wait before re-trying a busy sequential lock. Short enough
+    // to feel responsive, long enough that we don't hammer Redis.
+    const SEQUENTIAL_RETRY_DELAY_MS = 200;
+    // Two queue-clients — one per lane. Every runner holds both, regardless of
+    // its own consumerLane, so dispatch()/handleEvent() always write to the
+    // queue matching the target job's runIn. Client-creation is cheap (shared
+    // ioredis connection via bullmq), so this doesn't scale with number of
+    // processes.
+    // Queue-wide, age-only retention (see COMPLETED_JOB_RETENTION_AGE_SEC
+    // above for why) — merged into every add()/addBulk()/upsertJobScheduler()
+    // template on both lanes.
+    const jobRetentionOpts = {
+        removeOnComplete: { age: completedAgeSec },
+        removeOnFail: { age: failedAgeSec },
+    };
+    const queues = {
+        api: new Queue(queueNameFor(queueNamePrefix, "api"), {
+            connection: redisOpts,
+            defaultJobOptions: jobRetentionOpts,
+        }),
+        worker: new Queue(queueNameFor(queueNamePrefix, "worker"), {
+            connection: redisOpts,
+            defaultJobOptions: jobRetentionOpts,
+        }),
+    };
+    // Same unhandled-'error'-crash hazard as lockRedis above, just via
+    // BullMQ's internal ioredis client (fw#1805).
+    for (const queue of Object.values(queues)) {
+        queue.on("error", (err) => {
+            const log = stopping ? errorLogger.debug : errorLogger.error;
+            log("queue redis connection error", { error: err.message });
+        });
+    }
+    let worker = null;
+    let queueDepthTimer = null;
+    // Forward reference to the runner's own API, exposed on the job-handler ctx
+    // so a handler can dispatch a follow-up job (job→job chaining, e.g.
+    // delivery.render → delivery.send). Assigned just before return; reads happen
+    // at job-execution time (after start()), so it is always defined by then.
+    let selfRunner;
+    // Set by attachDispatcher() once the boot-level dispatcher exists.
+    // JobContext.write/writeAs/queryAs throw until this is set — see JobContext doc.
+    let dispatchWriteRef;
+    // Counts active + waiting jobs with this name for this tenant across
+    // BOTH lane queues. Jobs with the same name should only live in one
+    // lane (jobDef.runIn is static), but walking both is cheap and avoids
+    // a subtle bug if someone ever reassigns a job to a different lane
+    // between deploys while old queue contents are still draining.
+    async function isOverPerTenantLimit(jobName, tenantId, max) {
+        const results = await Promise.all([
+            queues.api.getActive(),
+            queues.api.getWaiting(),
+            queues.worker.getActive(),
+            queues.worker.getWaiting(),
+        ]);
+        let count = 0;
+        for (const list of results) {
+            for (const j of list) {
+                if (j.name !== jobName)
+                    continue;
+                const t = j.data?._tenantId; // @cast-boundary dynamic-key
+                if (t === tenantId) {
+                    count += 1;
+                    if (count >= max)
+                        return true;
+                }
+            }
+        }
+        return false;
+    }
+    async function handleJob(bullJob) {
+        const rawName = bullJob.name;
+        // Handle perTenant dispatch jobs — fan out to one job per tenant. The
+        // fan-out re-enqueues into the lane the actual job is assigned to;
+        // the _perTenant wrapper itself always lives in the consumer-lane
+        // (it's picked up by this runner's own worker).
+        if (rawName.startsWith("_perTenant:")) {
+            const actualName = rawName.slice("_perTenant:".length);
+            if (!getActiveTenantIds) {
+                throw new Error(`perTenant job "${actualName}" requires either options.getActiveTenantIds or the ` +
+                    `tenant feature mounted (it registers "${ACTIVE_TENANT_IDS_QUERY_NAME}", which the ` +
+                    "framework uses to resolve active tenants on its own)");
+            }
+            const actualDef = allJobs.get(actualName);
+            if (!actualDef) {
+                throw new Error(`Unknown job: ${actualName}`);
+            }
+            const tenantIds = await getActiveTenantIds();
+            const targetQueue = queues[laneForJob(actualDef)];
+            // wrapperJobId is only absent for a Job BullMQ somehow never assigned
+            // an id to (not observed in practice) — falling back to a constant
+            // there would collide every such run's children onto the same ids
+            // and silently swallow later runs as "duplicates", which is worse
+            // than the duplicate this is meant to prevent. Skip id-based dedup
+            // for that edge case instead: enqueue plainly, same as before.
+            const wrapperJobId = bullJob.id;
+            for (const tenantId of tenantIds) {
+                await targetQueue.add(actualName, { ...bullJob.data, _tenantId: tenantId }, {
+                    ...buildRetryBullOpts(actualDef),
+                    // Dedup over wrapper retries only holds as long as the children
+                    // stay in Redis for the wrapper's whole retry window — see the
+                    // COMPLETED_JOB_RETENTION_AGE_SEC invariant check above.
+                    ...(wrapperJobId !== undefined
+                        ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
+                        : {}),
+                });
+            }
+            // skip: fan-out dispatcher job, per-tenant children enqueued
+            return;
+        }
+        const jobName = rawName;
+        const jobDef = allJobs.get(jobName);
+        if (!jobDef) {
+            throw new Error(`Unknown job: ${jobName}`);
+        }
+        // Sequential gate: try to claim the per-name lock. If another worker
+        // (or this worker on a different bullJob) holds it, re-enqueue with a
+        // small delay and exit *successfully* — using throw would burn the
+        // job's retry budget and pollute failure metrics, but a re-enqueue
+        // looks like an ordinary handoff to BullMQ.
+        let sequentialToken = null;
+        if (jobDef.concurrency === "sequential" && sequentialLock) {
+            const ttlSec = jobDef.timeout
+                ? Math.ceil(jobDef.timeout / 1000) + 5
+                : SEQUENTIAL_DEFAULT_TTL_SEC;
+            sequentialToken = await sequentialLock.acquire(jobName, { ttlSeconds: ttlSec });
+            if (!sequentialToken) {
+                // Re-enqueue onto the job's own lane-queue. In practice that's the
+                // same queue the worker just picked from (since only the consuming
+                // lane runs handleJob at all), but route explicitly — no implicit
+                // coupling to "whichever queue the caller happened to be on".
+                // The re-enqueued job starts with a full retry budget; a remaining
+                // budget is deliberately not carried over, since finalAttempt/
+                // tenantVisibleFailure are computed from jobDef.retries against
+                // attemptsMade, not against some inherited remainder.
+                await queues[laneForJob(jobDef)].add(jobName, bullJob.data, {
+                    delay: SEQUENTIAL_RETRY_DELAY_MS,
+                    ...buildRetryBullOpts(jobDef),
+                });
+                // skip: lock taken, work re-enqueued with delay, current invocation done
+                return;
+            }
+        }
+        const jobId = bullJob.id ?? "unknown";
+        const startTime = Date.now();
+        const logs = [];
+        // Extract meta from job data. `attempt` is BullMQ's own counter
+        // (1-based on the first run, incremented on each retry) — threading
+        // it through lets the logger tag the run-started event with the
+        // retry number, so audit queries distinguish fresh from retry runs
+        // without peeking at BullMQ internals.
+        const rawData = bullJob.data;
+        const attempt = bullJob.attemptsMade + 1;
+        const meta = {
+            triggeredById: rawData["_triggeredById"], // @cast-boundary dynamic-key
+            payload: rawData["_payload"], // @cast-boundary dynamic-key
+            attempt,
+        };
+        // Build handler payload (without internal meta fields)
+        const payload = {};
+        for (const [k, v] of Object.entries(rawData)) {
+            if (!k.startsWith("_"))
+                payload[k] = v;
+        }
+        // Determine tenantId and triggeredBy from meta
+        const tenantId = rawData["_tenantId"] ?? // @cast-boundary dynamic-key
+            payload["tenantId"] ?? // @cast-boundary dynamic-key
+            SYSTEM_TENANT_ID;
+        const triggeredById = rawData["_triggeredById"] ?? null; // @cast-boundary dynamic-key
+        // Tenant-visible failure record (JobDefinition.tenantVisibleFailure). The
+        // subject is read from the handler payload, so it is resolved here where
+        // the payload is built, not in the callbacks.
+        const tenantVisibleDecl = jobDef.tenantVisibleFailure;
+        const tenantVisibleSubject = tenantVisibleDecl
+            ? jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields)
+            : null;
+        // BullMQ stops retrying once attemptsMade reaches the configured attempts
+        // (`retries + 1`), so this is the attempt whose failure is final.
+        const finalAttempt = attempt >= (jobDef.retries ?? 0) + 1;
+        const outcomeMeta = (messageKey) => ({
+            tenantId,
+            finalAttempt,
+            ...(tenantVisibleDecl && {
+                tenantVisible: { subject: tenantVisibleSubject, messageKey },
+            }),
+        });
+        // Carry `_triggerName` from rawData when set — handleEvent injects it on
+        // multi-trigger dispatch; exposed as jobContext.triggerName so handlers
+        // don't dig through the raw payload themselves.
+        const triggerName = rawData["_triggerName"]; // @cast-boundary dynamic-key
+        // Same carry-through for `_triggerEvent` — handleEvent injects it when
+        // the job-trigger event consumer supplied the stored event that fired
+        // this run (see createJobTriggerEventConsumer).
+        const rawTriggerEvent = rawData["_triggerEvent"];
+        const triggerEvent = isJobTriggerEvent(rawTriggerEvent) ? rawTriggerEvent : undefined;
+        // Absent = legacy or ungated root; present but invalid fails the run closed.
+        const rawWriteOrigin = rawData["_writeOrigin"]; // @cast-boundary dynamic-key
+        let jobOrigin;
+        let writeOriginInvalid = false;
+        if (rawWriteOrigin !== undefined) {
+            const parsed = parseWriteOrigin(rawWriteOrigin);
+            if (parsed) {
+                jobOrigin = { ...parsed, viaJob: jobName };
+            }
+            else {
+                writeOriginInvalid = true;
+            }
+        }
+        const jobPersonalDataGate = jobOrigin ? buildPersonalDataGate(registry, jobOrigin) : undefined;
+        // Mirror dispatch-shared.ts buildHandlerContext: ctx.files must resolve
+        // through the same _fileProviderResolver for jobs as for write-handlers,
+        // otherwise event-triggered jobs silently get an unresolved ctx.files.
+        const fileResolver = context._fileProviderResolver;
+        const files = fileResolver ? createFileContext(() => fileResolver(tenantId)) : context.files;
+        const jobSystemUser = createSystemUser(tenantId);
+        // Same buildHandlerContext parity as ctx.files above: ctx.notify/ctx.config
+        // must resolve for jobs the same way they do for write-handlers, or an
+        // app-author job calling ctx.notify(...)/ctx.config(...) hits a TypeError
+        // at runtime the write-handler path never would (framework#1532).
+        const notify = context._notifyFactory?.(jobSystemUser, tenantId);
+        const configDb = context.db; // @cast-boundary db-operator
+        // Shared by the config accessor and ctx.derivatives below — both need the
+        // same tenant-scoped db, and building it twice would let the two calls
+        // drift apart. Always "system" mode regardless of the job's own
+        // systemScope() status (pre-existing, not something this change alters)
+        // — isSystemJob below is what actually keeps ctx.systemDb off a
+        // non-system job; it is the only thing standing between this db and an
+        // unchecked cross-tenant escape hatch for such a job. Gated like jobDb so
+        // ctx.systemDb, built from it, is gated too.
+        const tenantScopedDb = configDb
+            ? createTenantDb(configDb, tenantId, "system", undefined, undefined, undefined, jobPersonalDataGate ? { personalDataGate: jobPersonalDataGate } : undefined)
+            : undefined;
+        const isSystemJob = registry.isJobSystemScoped(jobName);
+        // One reporter for ctx.systemDb and ctx.db.unsafeRaw() so both dedupe in the same window.
+        const reportEscapeHatch = createEscapeHatchReporter({
+            handler: jobName,
+            tenantId,
+            actor: jobSystemUser.id,
+            sink: context._escapeHatchAuditSink,
+            log: context.log,
+        });
+        const systemDb = isSystemJob && tenantScopedDb
+            ? createUncheckedSystemDb(tenantScopedDb, undefined, reportEscapeHatch)
+            : undefined;
+        const jobDb = configDb
+            ? createTenantDb(configDb, tenantId, "tenant", context.tracer, context.meter, undefined, {
+                unsafeRaw: jobDef.escapeHatch,
+                report: reportEscapeHatch,
+                ...(jobPersonalDataGate && { personalDataGate: jobPersonalDataGate }),
+            })
+            : undefined;
+        const config = context._configAccessorFactory && tenantScopedDb
+            ? context._configAccessorFactory({
+                user: { id: jobSystemUser.id, tenantId },
+                db: tenantScopedDb,
+                secrets: context.secrets,
+            })
+            : undefined;
+        // Mirror dispatch-shared.ts: ctx.derivatives needs files+db, same
+        // tenant-scoped db the config accessor above uses.
+        const derivatives = files && tenantScopedDb
+            ? createDerivativesContext({
+                files,
+                registry,
+                db: tenantScopedDb,
+                tenantId,
+            })
+            : context.derivatives;
+        // Per-job-run reader, created lazily on queryAsMember's first call and
+        // cached for the rest of this run.
+        let memberReader;
+        const jobContext = {
+            ...context,
+            // Undefined only for a runner built without a db; JobContext requires db non-optional.
+            db: jobDb, // @cast-boundary db-operator
+            files,
+            derivatives,
+            ...(notify !== undefined && { notify }),
+            ...(config !== undefined && { config }),
+            ...(systemDb && { systemDb }),
+            // The runner owns the registry it resolved this job from — expose it so
+            // workers can reach projections/jobs without the app author duplicating
+            // it into `context` (the JobContext contract guarantees `registry`).
+            registry,
+            // Expose the runner so handlers can chain a follow-up job. Symmetric with
+            // how the command-dispatcher hands write-handlers their jobRunner.
+            ...(selfRunner !== undefined && { jobRunner: selfRunner }),
+            systemUser: jobSystemUser,
+            triggeredBy: triggeredById !== null ? { id: triggeredById, tenantId } : null,
+            attempt,
+            finalAttempt,
+            log: createJobLogger(logs),
+            ...(triggerName !== undefined && { triggerName }),
+            ...(triggerEvent !== undefined && { triggerEvent }),
+            write: (qn, payload) => {
+                if (!dispatchWriteRef) {
+                    throw new Error("JobContext.write called before dispatcher attached — call attachDispatcher() first");
+                }
+                return dispatchWriteRef.write(jobSystemUser, qn, payload, jobOrigin);
+            },
+            writeAs: (user, qn, payload) => {
+                if (!dispatchWriteRef) {
+                    throw new Error("JobContext.writeAs called before dispatcher attached — call attachDispatcher() first");
+                }
+                return dispatchWriteRef.write(user, qn, payload, jobOrigin);
+            },
+            queryAs: (user, qn, payload) => {
+                if (!dispatchWriteRef) {
+                    throw new Error("JobContext.queryAs called before dispatcher attached — call attachDispatcher() first");
+                }
+                return dispatchWriteRef.queryAs(user, qn, payload, jobOrigin);
+            },
+            queryAsMember: (userId, qn, payload) => {
+                if (!dispatchWriteRef) {
+                    throw new Error("JobContext.queryAsMember called before dispatcher attached — call attachDispatcher() first");
+                }
+                // Lazy per-run reader, mirroring the handler-context reader.
+                memberReader ??= dispatchWriteRef.createMemberReader(tenantId);
+                return memberReader(userId, qn, payload);
+            },
+        };
+        await options.onJobStart?.(jobName, jobId, meta);
+        // Cross-process trace continuation: if the enqueuing code captured a
+        // parent span, start the job.execute span as its child. Works for event
+        // and manual triggers; cron jobs start a fresh root span.
+        const parentContext = readTraceContext(rawData);
+        // Correlation propagation: the scheduling request's correlationId was
+        // packed into _correlationId at dispatch time. Re-enter requestContext.run
+        // so event writes during this job stamp the same correlation as the
+        // request that scheduled it. Cron/boot jobs (no scheduler) start fresh
+        // — correlationId = new requestId, no parent causation.
+        const inheritedCorrelationId = rawData["_correlationId"] ?? undefined; // @cast-boundary dynamic-key
+        const jobRequestId = requestContext.generateId();
+        const jobCorrelationId = inheritedCorrelationId ?? jobRequestId;
+        const runInSpan = async () => {
+            try {
+                if (writeOriginInvalid) {
+                    throw new InternalError({
+                        message: `Job "${jobName}" received an unparseable _writeOrigin — refusing to run without ` +
+                            "a trustworthy anonymous-root gate.",
+                    });
+                }
+                await requestContext.run({
+                    requestId: jobRequestId,
+                    correlationId: jobCorrelationId,
+                    // #3043 — events a job writes carry the job as their origin.
+                    handler: jobName,
+                    feature: qnScope(jobName),
+                    writeOrigin: jobOrigin,
+                }, () => jobDef.handler(payload, jobContext));
+                // Stamped before the observer hook: a throwing onJobComplete must not
+                // make a run that actually succeeded look dead to the liveness alert.
+                if (context.meter)
+                    emitJobLastSuccess(context.meter, jobName);
+                const duration = Date.now() - startTime;
+                await options.onJobComplete?.(jobName, jobId, duration, logs, outcomeMeta(null));
+            }
+            catch (err) {
+                const errorMsg = err instanceof Error ? err.message : String(err);
+                logs.push({ level: "error", message: errorMsg, timestamp: Temporal.Now.instant() });
+                await options.onJobFailed?.(jobName, jobId, errorMsg, logs, outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl)));
+                throw err;
+            }
+        };
+        // Unified span creation: withSpan handles start/end + status/exception
+        // recording identically for both parent-context and no-parent paths.
+        // When parentContext is set, the new parent-aware StartSpanOptions
+        // plumbs it through to startSpan — no manual try/finally needed.
+        try {
+            await tracer.withSpan("job.execute", {
+                attributes: {
+                    "job.name": jobName,
+                    "job.id": jobId,
+                    "job.attempt": attempt,
+                    "kumiko.tenant_id": tenantId,
+                    // Lane-routing attributes (Welle 2.6). `run_in` is the job's
+                    // declared lane (explicit or default-"worker"); `consumer_lane`
+                    // is which runner actually executed it. They diverge in
+                    // all-in-one (both lanes live in one process) but must match
+                    // in split deploys — a mismatch in prod logs signals a
+                    // misrouted job that slipped past the boot-validator.
+                    "kumiko.job.run_in": laneForJob(jobDef),
+                    // Omit attribute entirely when no consumer (enqueuer-only runner) —
+                    // SpanAttributeValue doesn't accept undefined.
+                    ...(consumerLane !== undefined ? { "kumiko.job.consumer_lane": consumerLane } : {}),
+                },
+                ...(parentContext ? { parent: parentContext } : {}),
+            }, runInSpan);
+        }
+        finally {
+            // Release the sequential lock value-matched (Lua compare-and-delete
+            // inside DistributedLock). A TTL-expired lock that's been claimed by
+            // a different owner stays put — releasing it would break sequencing
+            // for the new owner.
+            if (sequentialToken && sequentialLock) {
+                await sequentialLock.release(jobName, sequentialToken);
+            }
+        }
+    }
+    // Boot gates run inline and are awaited, so a throwing gate rejects start()
+    // — and with it the entrypoint's start() and runProdApp's boot, before the
+    // process reports ready. Enqueuing them (as runOnBoot does) would only fail
+    // a queue job and let the deploy through (fw#2590).
+    async function runBootGates(lane) {
+        for (const [name, jobDef] of allJobs) {
+            if (laneForJob(jobDef) !== lane)
+                continue;
+            if (!jobDef.bootGate)
+                continue;
+            try {
+                await handleJob({ id: bootJobIdForJobName(name), name, data: {}, attemptsMade: 0 });
+            }
+            catch (err) {
+                // The worker is already consuming here; leaving its Redis connections
+                // open would keep the event loop alive and turn the aborted boot into
+                // a hanging process instead of a non-zero exit.
+                await worker?.close();
+                worker = null;
+                throw err;
+            }
+        }
+    }
+    const runnerApi = {
+        async start() {
+            // skip: enqueuer-only runner — no BullMQ worker, no cron schedules,
+            // no boot jobs. The API-process (runLocalJobs=false) lands here; it
+            // still holds the queue-clients so dispatch()/handleEvent() can
+            // target the worker-lane queue, but nothing local consumes.
+            if (!consumerLane) {
+                return;
+            }
+            const consumerQueue = queues[consumerLane];
+            worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), handleJob, {
+                connection: redisOpts,
+                concurrency: 5,
+            });
+            worker.on("error", (err) => {
+                const log = stopping ? errorLogger.debug : errorLogger.error;
+                log("worker redis connection error", { error: err.message });
+            });
+            // A caller that calls stop() right after start() otherwise races the
+            // still-settling blocking connection: it rejects in-flight commands
+            // via ioredis's flushQueue() during close(), which isn't a listenable
+            // 'error' event — the only fix is to not return until both of the
+            // worker's connections (main + blocking) are ready (fw#1805). This
+            // mirrors the wait BullMQ already does internally for
+            // upsertJobScheduler()/add() below when the lane has a cron/boot job.
+            // Racing a timeout against it keeps an unreachable Redis from hanging
+            // start() forever — there's no worker health endpoint to notice.
+            const bootTimeout = timeoutReject(bootRedisTimeoutMs, `job-runner: Redis not reachable within ${bootRedisTimeoutMs}ms (lane=${consumerLane})`);
+            try {
+                await Promise.race([worker.waitUntilReady(), bootTimeout.promise]);
+            }
+            finally {
+                bootTimeout.cancel();
+            }
+            // Ahead of cron/runOnBoot wiring on purpose: a failed gate must not
+            // leave recurring schedulers behind.
+            await runBootGates(consumerLane);
+            // Only schedule cron + boot for jobs that belong to this lane. Jobs
+            // assigned to the other lane get their cron/boot wiring from the
+            // runner running on that lane. Running both here would double-fire.
+            for (const [name, jobDef] of allJobs) {
+                if (laneForJob(jobDef) !== consumerLane)
+                    continue;
+                if ("cron" in jobDef.trigger) {
+                    const schedulerId = schedulerIdForJobName(name);
+                    const legacyId = legacySchedulerIdForJobName(name);
+                    // Drop pre-sanitize scheduler ids so colon-form ghosts stop firing.
+                    if (legacyId !== schedulerId) {
+                        try {
+                            await consumerQueue.removeJobScheduler(legacyId);
+                        }
+                        catch {
+                            // skip: legacy scheduler absent (fresh install / already purged)
+                        }
+                    }
+                    await consumerQueue.upsertJobScheduler(schedulerId, { pattern: jobDef.trigger.cron }, {
+                        name: jobDef.perTenant ? `_perTenant:${name}` : name,
+                        data: {},
+                        // Queue-level defaultJobOptions (jobRetentionOpts) covers
+                        // retention; a count here would sweep queue-wide again and
+                        // evict perTenant children and boot jobs.
+                        opts: {
+                            ...buildRetryBullOpts(jobDef),
+                        },
+                    });
+                }
+            }
+            // Persistent marker outside the swept completed/failed sets: a plain
+            // Redis hash at one key per consumer queue, fields = boot job ids
+            // already enqueued (BullMQ's IRedisClient has no set commands, only
+            // hash/hexists — a hash with dummy values does the same job). The job
+            // hash itself (removeOnComplete/-Fail: age-bound) is not a safe dedup
+            // target any more — retention now evicts it eventually, which would
+            // otherwise re-run a boot job "once per dataset" every time it ages
+            // out. Order matters: HEXISTS check, then add(), then HSET.
+            // Concurrent starts still dedupe on the existing job hash from add()'s
+            // own jobId no-op; a crash between add() and HSET dedupes again on the
+            // *next* boot (the job hash is still there) instead of losing the
+            // boot job forever, which HSET-first would risk if the process died
+            // before add() ran.
+            const bootEnqueuedKey = consumerQueue.toKey("kumiko-boot-enqueued");
+            for (const [name, jobDef] of allJobs) {
+                if (laneForJob(jobDef) !== consumerLane)
+                    continue;
+                if (jobDef.runOnBoot) {
+                    const bootName = jobDef.perTenant ? `_perTenant:${name}` : name;
+                    const bootJobId = bootJobIdForJobName(name);
+                    const client = await consumerQueue.client;
+                    const alreadyEnqueued = await client.hexists(bootEnqueuedKey, bootJobId);
+                    if (alreadyEnqueued)
+                        continue;
+                    await consumerQueue.add(bootName, {}, {
+                        jobId: bootJobId,
+                        ...buildRetryBullOpts(jobDef),
+                    });
+                    await client.hset(bootEnqueuedKey, { [bootJobId]: 1 });
+                }
+            }
+            // Skipped when no meter is wired (context.meter is optional, e.g. in
+            // tests without an observability provider).
+            if (context.meter) {
+                queueDepthTimer = await startQueueDepthPolling(consumerQueue, consumerLane, context.meter);
+            }
+        },
+        async stop() {
+            stopping = true;
+            if (queueDepthTimer) {
+                clearInterval(queueDepthTimer);
+                queueDepthTimer = null;
+            }
+            if (worker) {
+                await worker.close();
+                worker = null;
+            }
+            // BullMQ close() while a Queue's connection is still 'initializing'
+            // rejects the in-flight INFO with "Connection is closed." on an emitter
+            // without listeners -> unhandled rejection (fw#1805). Wait for readiness
+            // first; a failed/timed-out wait must not block the close itself.
+            const readinessTimeout = timeoutReject(bootRedisTimeoutMs, `job-runner: Queue not ready within ${bootRedisTimeoutMs}ms during stop()`);
+            try {
+                await Promise.race([
+                    Promise.all([queues.api.waitUntilReady(), queues.worker.waitUntilReady()]),
+                    readinessTimeout.promise,
+                ]);
+            }
+            catch {
+                // close() below still has to run
+            }
+            finally {
+                readinessTimeout.cancel();
+            }
+            await Promise.all([queues.api.close(), queues.worker.close()]);
+            if (lockRedis) {
+                // quit() drains in-flight commands; disconnect() would cancel them
+                // and risk a half-released lock.
+                await lockRedis.quit();
+                lockRedis = null;
+            }
+        },
+        async dispatch(jobName, payload, meta) {
+            const jobDef = allJobs.get(jobName);
+            if (!jobDef) {
+                throw new Error(`Unknown job: ${jobName}`);
+            }
+            // Route to the job's declared lane, not the runner's consumer lane —
+            // an api-runner is allowed to enqueue a worker-lane job and vice
+            // versa (that's the whole point of both queues being held).
+            const targetQueue = queues[laneForJob(jobDef)];
+            // perTenant: dispatch the fan-out wrapper instead
+            if (jobDef.perTenant) {
+                const perTenantData = { ...(payload ?? {}) };
+                stampGatedWriteOrigin(perTenantData);
+                const job = await targetQueue.add(`_perTenant:${jobName}`, perTenantData, buildRetryBullOpts(jobDef));
+                return job.id ?? "unknown";
+            }
+            // maxPerTenant guard: cap concurrent + waiting jobs of the same name
+            // for the same tenant. Orthogonal to the concurrency mode below — runs
+            // first because if we're over the limit nothing else matters.
+            // Requires a `_tenantId` in the payload to know which bucket to count
+            // against; without it the guard is inactive (system jobs, ambient
+            // dispatch). Fan-out children of perTenant jobs land here on their
+            // recursive queue.add and DO carry _tenantId.
+            if (jobDef.maxPerTenant !== undefined) {
+                const tenantId = payload?._tenantId; // @cast-boundary dynamic-key
+                if (tenantId !== undefined &&
+                    (await isOverPerTenantLimit(jobName, tenantId, jobDef.maxPerTenant))) {
+                    return "skipped:max-per-tenant";
+                }
+            }
+            const concurrency = jobDef.concurrency ?? "parallel";
+            const bullOpts = {};
+            switch (concurrency) {
+                case "skip": {
+                    const active = await targetQueue.getActive();
+                    const waiting = await targetQueue.getWaiting();
+                    const isRunning = [...active, ...waiting].some((j) => j.name === jobName);
+                    if (isRunning) {
+                        return "skipped";
+                    }
+                    break;
+                }
+                case "replace": {
+                    const waiting = await targetQueue.getWaiting();
+                    for (const j of waiting) {
+                        if (j.name === jobName && j.id) {
+                            await j.remove();
+                        }
+                    }
+                    break;
+                }
+                // case "sequential" is rejected at boot — see createJobRunner. Once
+                // the OSS-compatible implementation lands (per-name Redis lock),
+                // re-add the dispatch branch here.
+                case "debounce": {
+                    const debounceMs = jobDef.debounceMs ?? 5000;
+                    bullOpts["debounce"] = { id: jobName, ttl: debounceMs };
+                    break;
+                }
+                default:
+                    break;
+            }
+            Object.assign(bullOpts, buildRetryBullOpts(jobDef));
+            if (jobDef.timeout)
+                bullOpts["timeout"] = jobDef.timeout;
+            if (meta?.priority !== undefined)
+                bullOpts["priority"] = meta.priority;
+            // Pack meta into job data with _ prefix
+            const data = { ...payload };
+            if (meta?.triggeredById !== undefined)
+                data["_triggeredById"] = meta.triggeredById;
+            if (meta?.payload !== undefined)
+                data["_payload"] = meta.payload;
+            // Carry the enqueuing span context into the worker so job.execute shows
+            // as a child of the caller.
+            const traceCtx = captureTraceContext(tracer);
+            if (traceCtx)
+                data[TRACE_CONTEXT_KEY] = traceCtx;
+            // Propagate correlation from the scheduling request into the job
+            // execution context. The worker re-enters requestContext.run with
+            // this value so ctx.appendEvent / executor writes during the job
+            // stamp the same correlation as the HTTP request that scheduled it.
+            const reqCtx = requestContext.get();
+            if (reqCtx?.correlationId)
+                data["_correlationId"] = reqCtx.correlationId;
+            stampGatedWriteOrigin(data);
+            const job = await targetQueue.add(jobName, data, bullOpts);
+            return job.id ?? "unknown";
+        },
+        async handleEvent(eventName, payload, user, triggerEvent) {
+            const traceCtx = captureTraceContext(tracer);
+            // Same correlation propagation as dispatch(): events triggered from
+            // within a request (or an MSP-apply running under requestContext.run)
+            // get their correlationId into job data so the job execution keeps
+            // the same causal chain.
+            const reqCtx = requestContext.get();
+            for (const [name, jobDef] of allJobs) {
+                if (!("on" in jobDef.trigger))
+                    continue;
+                // skip: andere Trigger-Formen (cron, manual) reagieren nicht auf
+                // Events. Nur "on"-Trigger werden hier matched.
+                const triggerOn = jobDef.trigger.on;
+                const matches = Array.isArray(triggerOn)
+                    ? triggerOn.includes(eventName)
+                    : triggerOn === eventName;
+                if (!matches)
+                    continue;
+                if (!payloadMatchesTriggerFilter(jobDef.trigger.where, payload))
+                    continue;
+                const data = { ...payload };
+                if (user) {
+                    data["_tenantId"] = user.tenantId;
+                    data["_triggeredById"] = user.id;
+                }
+                // Multi-Trigger: payload bekommt _triggerName damit der Handler
+                // weiß, welcher der N Trigger gefeuert hat. Bei Single-Trigger
+                // setzen wir es auch — kostet nichts und vereinfacht Handler-Code
+                // (kein "ist es Multi?"-Branch nötig).
+                data["_triggerName"] = eventName;
+                if (triggerEvent)
+                    data["_triggerEvent"] = triggerEvent;
+                if (traceCtx)
+                    data[TRACE_CONTEXT_KEY] = traceCtx;
+                if (reqCtx?.correlationId)
+                    data["_correlationId"] = reqCtx.correlationId;
+                // Same maxPerTenant guard as dispatch — events that fan into many
+                // jobs must respect the per-tenant cap or the limit is one-sided.
+                if (jobDef.maxPerTenant !== undefined && user?.tenantId !== undefined) {
+                    if (await isOverPerTenantLimit(name, String(user.tenantId), jobDef.maxPerTenant)) {
+                        continue;
+                    }
+                }
+                stampGatedWriteOrigin(data);
+                // Route to the job's declared lane, not a fixed queue — that's
+                // the whole reason both queues are held.
+                await queues[laneForJob(jobDef)].add(name, data, buildRetryBullOpts(jobDef));
+            }
+        },
+        attachDispatcher(ref) {
+            dispatchWriteRef = ref;
+        },
+        async countPendingJobs() {
+            let total = 0;
+            for (const queue of Object.values(queues)) {
+                const counts = await queue.getJobCounts("waiting", "active", "prioritized", "waiting-children");
+                total +=
+                    (counts["waiting"] ?? 0) +
+                        (counts["active"] ?? 0) +
+                        (counts["prioritized"] ?? 0) +
+                        (counts["waiting-children"] ?? 0);
+                // getJobCounts has no id-prefix filter for "delayed" — getJobs(["delayed"])
+                // loads the full Job objects (payload included) just to read .id; fine for
+                // test-draining's low volume, not a pattern for a prod-path hot loop.
+                const delayed = await queue.getJobs(["delayed"]);
+                total += delayed.filter((job) => !(job.id ?? "").startsWith("repeat:")).length;
+            }
+            return total;
+        },
+    };
+    selfRunner = runnerApi;
+    return runnerApi;
+}