@cosmicdrift/kumiko-framework 0.328.0 → 0.329.0

package/src/jobs/job-runner.ts

@@ -1,1390 +0,0 @@
-import type { WriteOrigin } from "@cosmicdrift/kumiko-types/event-store-types";
-import { type JobsOptions, Queue, Worker } from "bullmq";
-import { Redis } from "ioredis";
-import { requestContext } from "../api/request-context";
-import type { DbConnection, DbRow } from "../db/connection";
-import { createTenantDb, createUncheckedSystemDb, type TenantDb } from "../db/tenant-db";
-import { createDerivativesContext } from "../derivatives/derivatives-context";
-import { qnScope } from "../engine/qualified-name";
-import { createSystemUser } from "../engine/system-user";
-import {
-  type AppContext,
-  type DispatchWriteRef,
-  type JobContext,
-  type JobDefinition,
-  type JobRunIn,
-  type JobTriggerEvent,
-  type MemberReader,
-  type Registry,
-  type SessionUser,
-  SYSTEM_TENANT_ID,
-  type TenantId,
-} from "../engine/types";
-import { InternalError } from "../errors";
-import { isKumikoError } from "../errors/kumiko-error";
-import { createFileContext } from "../files/file-handle";
-import { createFallbackLogger } from "../logging";
-import type { Logger } from "../logging/types";
-import {
-  emitJobLastSuccess,
-  emitJobQueueDepth,
-  getFallbackTracer,
-  type Meter,
-  type SerializedTraceContext,
-  type Tracer,
-} from "../observability";
-import { createEscapeHatchReporter } from "../observability/escape-hatch-report";
-import { createDistributedLock, type DistributedLock } from "../pipeline/distributed-lock";
-import { RedisKeys } from "../pipeline/redis-keys";
-import {
-  buildPersonalDataGate,
-  isPersonalDataGated,
-  parseWriteOrigin,
-} from "../pipeline/write-origin";
-import { bridgeStub } from "../testing/handler-context";
-
-// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
-// (a security-relevant escape hatch that must fail the run closed), a
-// malformed `_triggerEvent` only costs an idempotency-key convenience — the
-// run proceeds without it rather than failing over what would be a framework
-// bug, not untrusted input.
-function isJobTriggerEvent(value: unknown): value is JobTriggerEvent {
-  if (typeof value !== "object" || value === null) return false;
-  const candidate = value as Record<string, unknown>;
-  if (typeof candidate["id"] !== "string") return false;
-  const headers = candidate["headers"];
-  if (typeof headers !== "object" || headers === null) return false;
-  return Object.values(headers as Record<string, unknown>).every(
-    (v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean",
-  );
-}
-
-// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
-function stampGatedWriteOrigin(data: Record<string, unknown>): void {
-  delete data["_writeOrigin"];
-  const origin = requestContext.get()?.writeOrigin;
-  if (origin && isPersonalDataGated(origin)) {
-    data["_writeOrigin"] = origin;
-  }
-}
-
-// Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
-// ("kumiko-jobs") — it must match between enqueuers and consumers, and an
-// accidental drift would silently drop jobs. Tests override via
-// `queueNamePrefix` for per-run isolation (stale jobs from a prior run
-// don't leak into a new test because the queue name includes a timestamp).
-const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
-function queueNameFor(prefix: string, lane: JobRunIn): string {
-  return `${prefix}-${lane}`;
-}
-
-// perTenant jobs need a source of truth for "which tenants are active" —
-// see createJobRunner's defaultGetActiveTenantIds below. The tenant feature
-// (bundled-features package) registers a query under this exact name when
-// mounted. framework must not depend on bundled-features (wrong direction —
-// bundled-features already depends on framework), so this is a name-only
-// coupling: createJobRunner looks the handler up in the registry by string
-// instead of importing it.
-const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
-
-/**
- * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
- * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
- * treated as old repeatables — spawning a new scheduler entry every tick and
- * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
- * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
- */
-export function schedulerIdForJobName(jobName: string): string {
-  return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
-}
-
-// Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
-// a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
-export function bootJobIdForJobName(jobName: string): string {
-  return `boot-${jobName.replace(/[.:]/g, "-")}`;
-}
-
-// ponytail: migration shim, remove after fw#1603 deploy is everywhere.
-function legacySchedulerIdForJobName(jobName: string): string {
-  return `scheduler-${jobName.replace(/\./g, "-")}`;
-}
-
-// Deterministic per-tenant child job id, derived from the perTenant
-// wrapper's own BullMQ job id. A cron tick's wrapper id is
-// `repeat:<schedulerId>:<millis>` (stable for that tick, including across
-// BullMQ's own retry/stalled-job redelivery of the *same* job); a manual
-// dispatch's wrapper id is BullMQ's auto-generated id, equally stable
-// across retries of that job. Reusing it means a wrapper that runs twice
-// for the same trigger (retry after failure, a Redis drop, an instance
-// restarting mid-run) re-derives the *same* child ids, and BullMQ's
-// existing-jobId add() no-ops the second batch instead of creating
-// duplicates — safe only as long as the child job hash outlives the
-// wrapper's retry window, enforced at construction against
-// COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
-// createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
-// (fw#1603/#1604) — the wrapper id already contains ":", so strip
-// separators from both halves before joining instead of interpolating raw.
-function perTenantChildJobId(wrapperJobId: string, tenantId: string): string {
-  const sanitize = (value: string) => value.replace(/[.:]/g, "-");
-  return `child-${sanitize(wrapperJobId)}-${sanitize(tenantId)}`;
-}
-
-export type JobLogEntry = {
-  level: "info" | "warn" | "error";
-  message: string;
-  timestamp: Temporal.Instant;
-};
-
-function createJobLogger(logs: JobLogEntry[], baseContext: Record<string, unknown> = {}): Logger {
-  function push(level: "info" | "warn" | "error", msg: string, data?: Record<string, unknown>) {
-    const merged = { ...baseContext, ...data };
-    const message = Object.keys(merged).length > 0 ? `${msg} ${JSON.stringify(merged)}` : msg;
-    logs.push({ level, message, timestamp: Temporal.Now.instant() });
-  }
-  const logger: Logger = {
-    info(msg, data) {
-      push("info", msg, data);
-    },
-    warn(msg, data) {
-      push("warn", msg, data);
-    },
-    error(msg, data) {
-      push("error", msg, data);
-    },
-    // No-op by design, not missing coverage: JobLogEntry.level (and the
-    // persisted run-completed/-failed event schema in bundled-features/
-    // jobs/events.ts) only has info|warn|error — a job's debug() calls have
-    // nowhere durable to land.
-    debug() {},
-    child(context) {
-      return createJobLogger(logs, { ...baseContext, ...context });
-    },
-  };
-  return logger;
-}
-
-export type JobMeta = {
-  triggeredById?: string | undefined;
-  payload?: string | undefined;
-  // BullMQ numbers retries from 1 upward; the logger threads this into
-  // the run-started event so audit queries can distinguish "fresh run" vs.
-  // "nth retry" without joining back to BullMQ-internals.
-  attempt?: number | undefined;
-  // BullMQ job priority (lower = processed first). Set per-dispatch so the same
-  // job definition can be enqueued at different urgencies (e.g. delivery maps
-  // critical/normal/low onto it).
-  priority?: number | undefined;
-};
-
-// What a finished run tells the run-logger about the tenant-visible failure
-// record (`JobDefinition.tenantVisibleFailure`). `tenantVisible` is set only
-// when the job opted in; `messageKey` is null on the success path, where the
-// record is cleared rather than written.
-export type JobOutcomeMeta = {
-  readonly tenantId: string;
-  readonly finalAttempt: boolean;
-  readonly tenantVisible?:
-    | { readonly subject: string | null; readonly messageKey: string | null }
-    | undefined;
-};
-
-// Stable identity for the declared payload fields: sorted keys, so two runs
-// with the same subject values produce the same string and the later one
-// replaces the earlier record. Non-primitives would serialize into something
-// no caller can reconstruct for a lookup, so they fail the run loudly.
-function jobSubjectKey(
-  jobName: string,
-  payload: Record<string, unknown>,
-  fields: readonly string[] | undefined,
-): string | null {
-  if (fields === undefined || fields.length === 0) return null;
-  const entries: [string, string | number | boolean | null][] = [];
-  for (const field of [...fields].sort()) {
-    const value = payload[field] ?? null;
-    if (value !== null && typeof value === "object") {
-      throw new Error(
-        `Job "${jobName}": tenantVisibleFailure.subjectFields["${field}"] must be a primitive, got ${typeof value}`,
-      );
-    }
-    entries.push([field, value as string | number | boolean | null]);
-  }
-  return JSON.stringify(entries);
-}
-
-// Only a translation key ever travels to the tenant: the thrown error's own
-// i18nKey when it carries one, otherwise the key declared at the job. The
-// error message itself can echo provider or payload content and stays on the
-// run row and in the run log.
-function tenantFailureMessageKey(
-  err: unknown,
-  declaration: JobDefinition["tenantVisibleFailure"],
-): string | null {
-  // skip: job did not opt in — nothing tenant-visible to record
-  if (!declaration) return null;
-  if (isKumikoError(err) && err.i18nKey.length > 0) return err.i18nKey;
-  return declaration.messageKey;
-}
-
-export type JobRunner = {
-  start(): Promise<void>;
-  stop(): Promise<void>;
-  dispatch(jobName: string, payload?: Record<string, unknown>, meta?: JobMeta): Promise<string>;
-  handleEvent(
-    eventName: string,
-    payload: Record<string, unknown>,
-    user?: SessionUser,
-    triggerEvent?: JobTriggerEvent,
-  ): Promise<void>;
-  // Wires JobContext.write/writeAs/queryAs to the real dispatcher — called
-  // once at boot, after the dispatcher exists (job-runner construction happens
-  // before it). Before this runs, JobContext.write/writeAs/queryAs throw.
-  attachDispatcher(ref: DispatchWriteRef): void;
-  // Read-only snapshot across both lane queues for test-stack draining
-  // (stack.drainJobs()). Excludes delayed jobs whose BullMQ id starts with
-  // "repeat:" — those are future cron ticks, not work in flight, and would
-  // make drainJobs wait forever on a stack with a scheduled job.
-  countPendingJobs(): Promise<number>;
-};
-
-export type JobRunnerOptions = {
-  registry: Registry;
-  context: AppContext;
-  redisUrl: string;
-  // Which lane this runner CONSUMES — i.e. starts a BullMQ worker for and
-  // schedules cron/boot jobs on. Undefined = enqueuer-only: the runner
-  // still holds queue-clients for BOTH lanes so dispatch()/handleEvent()
-  // can enqueue jobs destined for either lane, but no BullMQ worker is
-  // started and no cron schedules fire. API processes that don't
-  // runLocalJobs leave this unset; worker processes set "worker"; api-
-  // processes with runLocalJobs set "api".
-  consumerLane?: JobRunIn | undefined;
-  // Override the queue-name prefix. Prod uses the default ("kumiko-jobs").
-  // Tests set a unique prefix (e.g. `"test-${Date.now()}"`) for isolation —
-  // two parallel test-runners never see each other's jobs.
-  queueNamePrefix?: string | undefined;
-  // Override how long start() waits for the worker's Redis connection
-  // before failing boot. Defaults to BOOT_REDIS_TIMEOUT_MS; tests shrink it
-  // to keep an unreachable-Redis assertion fast.
-  bootRedisTimeoutMs?: number | undefined;
-  // Override how long completed/failed jobs stay in Redis before BullMQ's
-  // lazy queue-wide sweep evicts them. Defaults to
-  // COMPLETED_JOB_RETENTION_AGE_SEC / FAILED_JOB_RETENTION_AGE_SEC; tests
-  // shrink both to exercise the sweep without a real 24h/7d wait.
-  jobRetention?:
-    | { completedAgeSec?: number | undefined; failedAgeSec?: number | undefined }
-    | undefined;
-  getActiveTenantIds?: () => Promise<TenantId[]>;
-  onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
-  onJobComplete?: (
-    jobName: string,
-    jobId: string,
-    duration: number,
-    logs: JobLogEntry[],
-    outcome?: JobOutcomeMeta,
-  ) => void;
-  onJobFailed?: (
-    jobName: string,
-    jobId: string,
-    error: string,
-    logs: JobLogEntry[],
-    outcome?: JobOutcomeMeta,
-  ) => void;
-};
-
-// Serialized trace context lives under this key in the BullMQ job data.
-// Leading underscore matches the existing internal-meta convention
-// (_triggeredById, _tenantId, _payload).
-const TRACE_CONTEXT_KEY = "_traceContext";
-
-function readTraceContext(data: Record<string, unknown>): SerializedTraceContext | undefined {
-  const raw = data[TRACE_CONTEXT_KEY];
-  if (!raw || typeof raw !== "object") return undefined;
-  const ctx = raw as Partial<SerializedTraceContext>; // @cast-boundary engine-payload
-  if (!ctx.traceId || !ctx.spanId) return undefined;
-  return { traceId: ctx.traceId, spanId: ctx.spanId };
-}
-
-function captureTraceContext(tracer: Tracer): SerializedTraceContext | undefined {
-  const span = tracer.getActiveSpan();
-  if (!span?.traceId) return undefined;
-  return { traceId: span.traceId, spanId: span.spanId };
-}
-
-const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
-
-// kumiko_job_queue_depth: only the lane's own consumer polls — the count is
-// a global Redis-backed value (BullMQ, not per-process), so one reporter per
-// lane is enough. Fires once immediately (metrics exist before the first
-// poll tick) then on an interval; the caller stores + clears the handle.
-async function startQueueDepthPolling(
-  queue: Queue,
-  lane: JobRunIn,
-  meter: Meter,
-): Promise<ReturnType<typeof setInterval>> {
-  const pollQueueDepth = async (): Promise<void> => {
-    try {
-      const counts = await queue.getJobCounts("waiting", "active", "delayed", "failed", "paused");
-      emitJobQueueDepth(meter, lane, counts);
-    } catch {
-      // skip: transient Redis hiccup — next poll retries
-    }
-  };
-  await pollQueueDepth();
-  return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
-}
-
-function parseRedisOpts(url: string): { host: string; port: number; db?: number | undefined } {
-  const parsed = new URL(url);
-  const result: { host: string; port: number; db?: number | undefined } = {
-    host: parsed.hostname,
-    port: Number(parsed.port) || 6379,
-  };
-  if (parsed.pathname.length > 1) {
-    result.db = Number(parsed.pathname.slice(1));
-  }
-  return result;
-}
-
-// redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
-// would otherwise hang start() forever with no health endpoint to notice.
-const BOOT_REDIS_TIMEOUT_MS = 10_000;
-
-function timeoutReject(
-  ms: number,
-  message: string,
-): { promise: Promise<never>; cancel: () => void } {
-  let timer: ReturnType<typeof setTimeout> | undefined;
-  const promise = new Promise<never>((_, reject) => {
-    timer = setTimeout(() => reject(new Error(message)), ms);
-  });
-  return {
-    promise,
-    cancel: () => {
-      if (timer !== undefined) clearTimeout(timer);
-    },
-  };
-}
-
-// Default base delay when a job opts into backoff without an explicit
-// delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
-// NaN/undefined, which is falsy — the job retries immediately instead of
-// waiting.
-const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
-
-// Shared by every enqueue path (dispatch(), handleEvent(), cron, runOnBoot,
-// perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
-// set must retry the same way regardless of how it got enqueued, or it
-// fails for good on the very first error on whichever path skips this.
-function buildRetryBullOpts(jobDef: JobDefinition): Pick<JobsOptions, "attempts" | "backoff"> {
-  const opts: Pick<JobsOptions, "attempts" | "backoff"> = {};
-  if (jobDef.retries !== undefined) opts.attempts = jobDef.retries + 1;
-  if (jobDef.backoff) {
-    opts.backoff =
-      typeof jobDef.backoff === "string"
-        ? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
-        : {
-            type: jobDef.backoff.type,
-            delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
-          };
-  }
-  return opts;
-}
-
-// Equality filter on top-level payload fields for a job's trigger.where —
-// lets N jobs share one broad event QN, partitioned by a payload
-// discriminant instead of needing N distinct event types.
-function payloadMatchesTriggerFilter(
-  where: Readonly<Record<string, string | number | boolean>> | undefined,
-  payload: Record<string, unknown>,
-): boolean {
-  if (!where) return true;
-  return Object.entries(where).every(([key, value]) => payload[key] === value);
-}
-
-// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
-// OTHER jobs in the same completed/failed zset (moveToFinished lua,
-// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
-// the sweep only runs lazily on a later finish into that set. Per-job
-// retention is therefore meaningless on a shared queue — only queue-wide,
-// age-only retention (no count) is safe: a count would evict perTenant
-// children still inside their wrapper's retry window, and boot jobs. The
-// completed age must stay above every perTenant wrapper's own worst-case
-// retry window (see maxRetryWindowMs and the invariant check in
-// createJobRunner below), because child dedup (perTenantChildJobId) relies
-// on the children still existing in Redis when a wrapper retry re-derives
-// their ids. The margin here is generous — queue wait time isn't bounded by
-// backoff — to cover realistic windows. Audit trail lives in read_job_runs,
-// not BullMQ (fw#3199).
-const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
-const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
-
-// Total wall-clock time BullMQ can hold a perTenant wrapper across all its
-// retries, consistent with buildRetryBullOpts's own backoff computation.
-function maxRetryWindowMs(jobDef: JobDefinition): number {
-  if (!jobDef.backoff || jobDef.retries === undefined) return 0;
-  const delayMs =
-    (typeof jobDef.backoff === "string" ? undefined : jobDef.backoff.delayMs) ??
-    DEFAULT_JOB_BACKOFF_DELAY_MS;
-  const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
-  return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
-}
-
-export function createJobRunner(options: JobRunnerOptions): JobRunner {
-  const { registry, context, redisUrl, consumerLane } = options;
-  const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;
-  const bootRedisTimeoutMs = options.bootRedisTimeoutMs ?? BOOT_REDIS_TIMEOUT_MS;
-  const redisOpts = parseRedisOpts(redisUrl);
-  // Use the context's tracer when present (observability-provider injected at
-  // boot); otherwise noop so dispatch/handleJob stay zero-cost without config.
-  const tracer: Tracer = context.tracer ?? getFallbackTracer();
-  const errorLogger = createFallbackLogger("job-runner", context.log);
-  // Set at the top of stop() — a graceful shutdown closes the redis/BullMQ
-  // clients itself, which fires the exact same 'error' listeners below with
-  // an expected "Connection is closed." Downgrading to debug once stopping
-  // is true keeps those out of error-rate alerts without losing them.
-  let stopping = false;
-
-  // Default for perTenant fan-out when the caller didn't supply
-  // options.getActiveTenantIds: if the tenant feature is mounted (its
-  // active-tenant-ids query is in the registry), resolve tenants through
-  // that instead of forcing every app to wire the option by hand. Same
-  // hand-built-context pattern delivery-service.ts uses to invoke a
-  // registered handler without a live dispatcher — no dispatcher exists yet
-  // at createJobRunner() time, and this call runs with system-level access
-  // by construction, so going through the dispatcher's per-request identity
-  // checks would add nothing.
-  function defaultGetActiveTenantIds(): (() => Promise<TenantId[]>) | undefined {
-    const handler = registry.getQueryHandler(ACTIVE_TENANT_IDS_QUERY_NAME);
-    const db = context.db as DbConnection | undefined; // @cast-boundary db-operator
-    if (!handler || !db) return undefined;
-    return async () => {
-      const systemUser = createSystemUser(SYSTEM_TENANT_ID);
-      const systemModeDb = createTenantDb(db, SYSTEM_TENANT_ID, "system");
-      const reportEscapeHatch = createEscapeHatchReporter({
-        handler: ACTIVE_TENANT_IDS_QUERY_NAME,
-        tenantId: SYSTEM_TENANT_ID,
-        actor: systemUser.id,
-        sink: context._escapeHatchAuditSink,
-        log: context.log,
-      });
-      const result = await handler.handler(
-        { type: ACTIVE_TENANT_IDS_QUERY_NAME, payload: {}, user: systemUser },
-        {
-          db: systemModeDb,
-          dbOutsideTransaction: systemModeDb,
-          systemDb: createUncheckedSystemDb(systemModeDb, undefined, reportEscapeHatch),
-          registry,
-          ...bridgeStub(),
-        },
-      );
-      // @cast-boundary engine-payload — generic query-handler return for typed convention
-      return result as TenantId[];
-    };
-  }
-  const getActiveTenantIds = options.getActiveTenantIds ?? defaultGetActiveTenantIds();
-
-  const allJobs = registry.getAllJobs();
-
-  function positiveIntRetentionSec(
-    value: number | undefined,
-    fallback: number,
-    label: string,
-  ): number {
-    const resolved = value ?? fallback;
-    if (!Number.isInteger(resolved) || resolved <= 0) {
-      throw new Error(
-        `job-runner: jobRetention.${label} must be a positive integer, got ${resolved}`,
-      );
-    }
-    return resolved;
-  }
-  const completedAgeSec = positiveIntRetentionSec(
-    options.jobRetention?.completedAgeSec,
-    COMPLETED_JOB_RETENTION_AGE_SEC,
-    "completedAgeSec",
-  );
-  const failedAgeSec = positiveIntRetentionSec(
-    options.jobRetention?.failedAgeSec,
-    FAILED_JOB_RETENTION_AGE_SEC,
-    "failedAgeSec",
-  );
-
-  // perTenant child dedup (perTenantChildJobId) only holds as long as the
-  // children are still in Redis when a wrapper retry re-derives their ids,
-  // which requires the completed-job retention to outlast the wrapper's own
-  // worst-case retry window. Enforced here, before any Redis connection
-  // opens below, so a misconfigured job fails boot loudly instead of
-  // silently losing dedup in prod.
-  for (const [name, jobDef] of allJobs) {
-    if (!jobDef.perTenant) continue;
-    const windowMs = maxRetryWindowMs(jobDef);
-    if (windowMs >= completedAgeSec * 1000) {
-      throw new Error(
-        `job-runner: perTenant job "${name}" has a retry window of ${windowMs}ms, which is >= ` +
-          `the completed-job retention (${completedAgeSec}s). Child dedup relies on children ` +
-          "staying in Redis for the whole retry window — lower retries/backoff or raise " +
-          "jobRetention.completedAgeSec.",
-      );
-    }
-  }
-
-  // Resolve the lane for a job — "worker" is the default because that's the
-  // sensible prod lane (heavy async off the request path). Jobs that opted
-  // into "api" must have been validated at registry boot already.
-  function laneForJob(def: { readonly runIn?: JobRunIn | undefined }): JobRunIn {
-    return def.runIn ?? "worker";
-  }
-
-  // Sequential coordination: BullMQ OSS has no `group`, so we serialise
-  // same-name jobs ourselves with a per-name Redis lock. Only built when at
-  // least one job actually requested it — keeps the no-sequential boot path
-  // free of the extra Redis client. Scoped under the consumer lane so two
-  // runners on different lanes cannot collide on the same lock-key for
-  // unrelated jobs.
-  const hasSequential = [...allJobs.values()].some((def) => def.concurrency === "sequential");
-  let lockRedis: Redis | null = null;
-  let sequentialLock: DistributedLock | null = null;
-  if (hasSequential) {
-    lockRedis = new Redis(redisOpts);
-    // Without a listener, a post-close 'error' (e.g. a teardown-race
-    // "Connection is closed") is unhandled and crashes the process — in
-    // bun:test it gets attributed to whichever test happens to run next
-    // (fw#1805).
-    lockRedis.on("error", (err) => {
-      const log = stopping ? errorLogger.debug : errorLogger.error;
-      log("lock redis connection error", { error: err.message });
-    });
-    const lockScope = consumerLane ?? "enqueue";
-    // Scoped by queueNamePrefix like the queues themselves, so runners
-    // isolated by prefix on a shared Redis don't contend for the same lock.
-    sequentialLock = createDistributedLock(
-      lockRedis,
-      `${RedisKeys.lock}seq:${queueNamePrefix}:${lockScope}:`,
-    );
-  }
-  // Default lock-TTL for sequential jobs that didn't declare a timeout.
-  // 5 minutes matches BullMQ's default stalledInterval — long enough for
-  // any reasonable handler, short enough that a crashed worker recovers
-  // without manual intervention.
-  const SEQUENTIAL_DEFAULT_TTL_SEC = 305;
-  // How long to wait before re-trying a busy sequential lock. Short enough
-  // to feel responsive, long enough that we don't hammer Redis.
-  const SEQUENTIAL_RETRY_DELAY_MS = 200;
-
-  // Two queue-clients — one per lane. Every runner holds both, regardless of
-  // its own consumerLane, so dispatch()/handleEvent() always write to the
-  // queue matching the target job's runIn. Client-creation is cheap (shared
-  // ioredis connection via bullmq), so this doesn't scale with number of
-  // processes.
-  // Queue-wide, age-only retention (see COMPLETED_JOB_RETENTION_AGE_SEC
-  // above for why) — merged into every add()/addBulk()/upsertJobScheduler()
-  // template on both lanes.
-  const jobRetentionOpts: Pick<JobsOptions, "removeOnComplete" | "removeOnFail"> = {
-    removeOnComplete: { age: completedAgeSec },
-    removeOnFail: { age: failedAgeSec },
-  };
-  const queues: Readonly<Record<JobRunIn, Queue>> = {
-    api: new Queue(queueNameFor(queueNamePrefix, "api"), {
-      connection: redisOpts,
-      defaultJobOptions: jobRetentionOpts,
-    }),
-    worker: new Queue(queueNameFor(queueNamePrefix, "worker"), {
-      connection: redisOpts,
-      defaultJobOptions: jobRetentionOpts,
-    }),
-  };
-  // Same unhandled-'error'-crash hazard as lockRedis above, just via
-  // BullMQ's internal ioredis client (fw#1805).
-  for (const queue of Object.values(queues)) {
-    queue.on("error", (err) => {
-      const log = stopping ? errorLogger.debug : errorLogger.error;
-      log("queue redis connection error", { error: err.message });
-    });
-  }
-  let worker: Worker | null = null;
-  let queueDepthTimer: ReturnType<typeof setInterval> | null = null;
-  // Forward reference to the runner's own API, exposed on the job-handler ctx
-  // so a handler can dispatch a follow-up job (job→job chaining, e.g.
-  // delivery.render → delivery.send). Assigned just before return; reads happen
-  // at job-execution time (after start()), so it is always defined by then.
-  let selfRunner: JobRunner | undefined;
-  // Set by attachDispatcher() once the boot-level dispatcher exists.
-  // JobContext.write/writeAs/queryAs throw until this is set — see JobContext doc.
-  let dispatchWriteRef: DispatchWriteRef | undefined;
-
-  // Counts active + waiting jobs with this name for this tenant across
-  // BOTH lane queues. Jobs with the same name should only live in one
-  // lane (jobDef.runIn is static), but walking both is cheap and avoids
-  // a subtle bug if someone ever reassigns a job to a different lane
-  // between deploys while old queue contents are still draining.
-  async function isOverPerTenantLimit(
-    jobName: string,
-    tenantId: string,
-    max: number,
-  ): Promise<boolean> {
-    const results = await Promise.all([
-      queues.api.getActive(),
-      queues.api.getWaiting(),
-      queues.worker.getActive(),
-      queues.worker.getWaiting(),
-    ]);
-    let count = 0;
-    for (const list of results) {
-      for (const j of list) {
-        if (j.name !== jobName) continue;
-        const t = (j.data as { _tenantId?: string } | undefined)?._tenantId; // @cast-boundary dynamic-key
-        if (t === tenantId) {
-          count += 1;
-          if (count >= max) return true;
-        }
-      }
-    }
-    return false;
-  }
-
-  // Structural instead of BullMQ's `Job` so boot gates can invoke the same
-  // execution path inline. A real `Job` stays assignable, so handleJob is
-  // still a valid BullMQ Processor.
-  type JobInvocation = {
-    readonly id?: string | undefined;
-    readonly name: string;
-    readonly data: Record<string, unknown>;
-    readonly attemptsMade: number;
-  };
-
-  async function handleJob(bullJob: JobInvocation): Promise<void> {
-    const rawName = bullJob.name;
-
-    // Handle perTenant dispatch jobs — fan out to one job per tenant. The
-    // fan-out re-enqueues into the lane the actual job is assigned to;
-    // the _perTenant wrapper itself always lives in the consumer-lane
-    // (it's picked up by this runner's own worker).
-    if (rawName.startsWith("_perTenant:")) {
-      const actualName = rawName.slice("_perTenant:".length);
-      if (!getActiveTenantIds) {
-        throw new Error(
-          `perTenant job "${actualName}" requires either options.getActiveTenantIds or the ` +
-            `tenant feature mounted (it registers "${ACTIVE_TENANT_IDS_QUERY_NAME}", which the ` +
-            "framework uses to resolve active tenants on its own)",
-        );
-      }
-      const actualDef = allJobs.get(actualName);
-      if (!actualDef) {
-        throw new Error(`Unknown job: ${actualName}`);
-      }
-      const tenantIds = await getActiveTenantIds();
-      const targetQueue = queues[laneForJob(actualDef)];
-      // wrapperJobId is only absent for a Job BullMQ somehow never assigned
-      // an id to (not observed in practice) — falling back to a constant
-      // there would collide every such run's children onto the same ids
-      // and silently swallow later runs as "duplicates", which is worse
-      // than the duplicate this is meant to prevent. Skip id-based dedup
-      // for that edge case instead: enqueue plainly, same as before.
-      const wrapperJobId = bullJob.id;
-      for (const tenantId of tenantIds) {
-        await targetQueue.add(
-          actualName,
-          { ...bullJob.data, _tenantId: tenantId },
-          {
-            ...buildRetryBullOpts(actualDef),
-            // Dedup over wrapper retries only holds as long as the children
-            // stay in Redis for the wrapper's whole retry window — see the
-            // COMPLETED_JOB_RETENTION_AGE_SEC invariant check above.
-            ...(wrapperJobId !== undefined
-              ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
-              : {}),
-          },
-        );
-      }
-      // skip: fan-out dispatcher job, per-tenant children enqueued
-      return;
-    }
-
-    const jobName = rawName;
-    const jobDef = allJobs.get(jobName);
-    if (!jobDef) {
-      throw new Error(`Unknown job: ${jobName}`);
-    }
-
-    // Sequential gate: try to claim the per-name lock. If another worker
-    // (or this worker on a different bullJob) holds it, re-enqueue with a
-    // small delay and exit *successfully* — using throw would burn the
-    // job's retry budget and pollute failure metrics, but a re-enqueue
-    // looks like an ordinary handoff to BullMQ.
-    let sequentialToken: string | null = null;
-    if (jobDef.concurrency === "sequential" && sequentialLock) {
-      const ttlSec = jobDef.timeout
-        ? Math.ceil(jobDef.timeout / 1000) + 5
-        : SEQUENTIAL_DEFAULT_TTL_SEC;
-      sequentialToken = await sequentialLock.acquire(jobName, { ttlSeconds: ttlSec });
-      if (!sequentialToken) {
-        // Re-enqueue onto the job's own lane-queue. In practice that's the
-        // same queue the worker just picked from (since only the consuming
-        // lane runs handleJob at all), but route explicitly — no implicit
-        // coupling to "whichever queue the caller happened to be on".
-        // The re-enqueued job starts with a full retry budget; a remaining
-        // budget is deliberately not carried over, since finalAttempt/
-        // tenantVisibleFailure are computed from jobDef.retries against
-        // attemptsMade, not against some inherited remainder.
-        await queues[laneForJob(jobDef)].add(jobName, bullJob.data, {
-          delay: SEQUENTIAL_RETRY_DELAY_MS,
-          ...buildRetryBullOpts(jobDef),
-        });
-        // skip: lock taken, work re-enqueued with delay, current invocation done
-        return;
-      }
-    }
-
-    const jobId = bullJob.id ?? "unknown";
-    const startTime = Date.now();
-    const logs: JobLogEntry[] = [];
-
-    // Extract meta from job data. `attempt` is BullMQ's own counter
-    // (1-based on the first run, incremented on each retry) — threading
-    // it through lets the logger tag the run-started event with the
-    // retry number, so audit queries distinguish fresh from retry runs
-    // without peeking at BullMQ internals.
-    const rawData = bullJob.data as DbRow;
-    const attempt = bullJob.attemptsMade + 1;
-    const meta: JobMeta = {
-      triggeredById: rawData["_triggeredById"] as string | undefined, // @cast-boundary dynamic-key
-      payload: rawData["_payload"] as string | undefined, // @cast-boundary dynamic-key
-      attempt,
-    };
-
-    // Build handler payload (without internal meta fields)
-    const payload: Record<string, unknown> = {};
-    for (const [k, v] of Object.entries(rawData)) {
-      if (!k.startsWith("_")) payload[k] = v;
-    }
-
-    // Determine tenantId and triggeredBy from meta
-    const tenantId =
-      (rawData["_tenantId"] as string | undefined) ?? // @cast-boundary dynamic-key
-      (payload["tenantId"] as string | undefined) ?? // @cast-boundary dynamic-key
-      SYSTEM_TENANT_ID;
-    const triggeredById = (rawData["_triggeredById"] as string | undefined) ?? null; // @cast-boundary dynamic-key
-
-    // Tenant-visible failure record (JobDefinition.tenantVisibleFailure). The
-    // subject is read from the handler payload, so it is resolved here where
-    // the payload is built, not in the callbacks.
-    const tenantVisibleDecl = jobDef.tenantVisibleFailure;
-    const tenantVisibleSubject = tenantVisibleDecl
-      ? jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields)
-      : null;
-    // BullMQ stops retrying once attemptsMade reaches the configured attempts
-    // (`retries + 1`), so this is the attempt whose failure is final.
-    const finalAttempt = attempt >= (jobDef.retries ?? 0) + 1;
-    const outcomeMeta = (messageKey: string | null): JobOutcomeMeta => ({
-      tenantId,
-      finalAttempt,
-      ...(tenantVisibleDecl && {
-        tenantVisible: { subject: tenantVisibleSubject, messageKey },
-      }),
-    });
-
-    // Carry `_triggerName` from rawData when set — handleEvent injects it on
-    // multi-trigger dispatch; exposed as jobContext.triggerName so handlers
-    // don't dig through the raw payload themselves.
-    const triggerName = rawData["_triggerName"] as string | undefined; // @cast-boundary dynamic-key
-
-    // Same carry-through for `_triggerEvent` — handleEvent injects it when
-    // the job-trigger event consumer supplied the stored event that fired
-    // this run (see createJobTriggerEventConsumer).
-    const rawTriggerEvent = rawData["_triggerEvent"];
-    const triggerEvent = isJobTriggerEvent(rawTriggerEvent) ? rawTriggerEvent : undefined;
-
-    // Absent = legacy or ungated root; present but invalid fails the run closed.
-    const rawWriteOrigin = rawData["_writeOrigin"]; // @cast-boundary dynamic-key
-    let jobOrigin: WriteOrigin | undefined;
-    let writeOriginInvalid = false;
-    if (rawWriteOrigin !== undefined) {
-      const parsed = parseWriteOrigin(rawWriteOrigin);
-      if (parsed) {
-        jobOrigin = { ...parsed, viaJob: jobName };
-      } else {
-        writeOriginInvalid = true;
-      }
-    }
-    const jobPersonalDataGate = jobOrigin ? buildPersonalDataGate(registry, jobOrigin) : undefined;
-
-    // Mirror dispatch-shared.ts buildHandlerContext: ctx.files must resolve
-    // through the same _fileProviderResolver for jobs as for write-handlers,
-    // otherwise event-triggered jobs silently get an unresolved ctx.files.
-    const fileResolver = context._fileProviderResolver;
-    const files = fileResolver ? createFileContext(() => fileResolver(tenantId)) : context.files;
-    const jobSystemUser = createSystemUser(tenantId);
-    // Same buildHandlerContext parity as ctx.files above: ctx.notify/ctx.config
-    // must resolve for jobs the same way they do for write-handlers, or an
-    // app-author job calling ctx.notify(...)/ctx.config(...) hits a TypeError
-    // at runtime the write-handler path never would (framework#1532).
-    const notify = context._notifyFactory?.(jobSystemUser, tenantId);
-    const configDb = context.db as DbConnection | undefined; // @cast-boundary db-operator
-    // Shared by the config accessor and ctx.derivatives below — both need the
-    // same tenant-scoped db, and building it twice would let the two calls
-    // drift apart. Always "system" mode regardless of the job's own
-    // systemScope() status (pre-existing, not something this change alters)
-    // — isSystemJob below is what actually keeps ctx.systemDb off a
-    // non-system job; it is the only thing standing between this db and an
-    // unchecked cross-tenant escape hatch for such a job. Gated like jobDb so
-    // ctx.systemDb, built from it, is gated too.
-    const tenantScopedDb = configDb
-      ? createTenantDb(
-          configDb,
-          tenantId,
-          "system",
-          undefined,
-          undefined,
-          undefined,
-          jobPersonalDataGate ? { personalDataGate: jobPersonalDataGate } : undefined,
-        )
-      : undefined;
-    const isSystemJob = registry.isJobSystemScoped(jobName);
-    // One reporter for ctx.systemDb and ctx.db.unsafeRaw() so both dedupe in the same window.
-    const reportEscapeHatch = createEscapeHatchReporter({
-      handler: jobName,
-      tenantId,
-      actor: jobSystemUser.id,
-      sink: context._escapeHatchAuditSink,
-      log: context.log,
-    });
-    const systemDb =
-      isSystemJob && tenantScopedDb
-        ? createUncheckedSystemDb(tenantScopedDb, undefined, reportEscapeHatch)
-        : undefined;
-    const jobDb = configDb
-      ? createTenantDb(configDb, tenantId, "tenant", context.tracer, context.meter, undefined, {
-          unsafeRaw: jobDef.escapeHatch,
-          report: reportEscapeHatch,
-          ...(jobPersonalDataGate && { personalDataGate: jobPersonalDataGate }),
-        })
-      : undefined;
-    const config =
-      context._configAccessorFactory && tenantScopedDb
-        ? context._configAccessorFactory({
-            user: { id: jobSystemUser.id, tenantId },
-            db: tenantScopedDb,
-            secrets: context.secrets,
-          })
-        : undefined;
-    // Mirror dispatch-shared.ts: ctx.derivatives needs files+db, same
-    // tenant-scoped db the config accessor above uses.
-    const derivatives =
-      files && tenantScopedDb
-        ? createDerivativesContext({
-            files,
-            registry,
-            db: tenantScopedDb,
-            tenantId,
-          })
-        : context.derivatives;
-    // Per-job-run reader, created lazily on queryAsMember's first call and
-    // cached for the rest of this run.
-    let memberReader: MemberReader | undefined;
-    const jobContext: JobContext = {
-      ...context,
-      // Undefined only for a runner built without a db; JobContext requires db non-optional.
-      db: jobDb as TenantDb, // @cast-boundary db-operator
-      files,
-      derivatives,
-      ...(notify !== undefined && { notify }),
-      ...(config !== undefined && { config }),
-      ...(systemDb && { systemDb }),
-      // The runner owns the registry it resolved this job from — expose it so
-      // workers can reach projections/jobs without the app author duplicating
-      // it into `context` (the JobContext contract guarantees `registry`).
-      registry,
-      // Expose the runner so handlers can chain a follow-up job. Symmetric with
-      // how the command-dispatcher hands write-handlers their jobRunner.
-      ...(selfRunner !== undefined && { jobRunner: selfRunner }),
-      systemUser: jobSystemUser,
-      triggeredBy: triggeredById !== null ? { id: triggeredById, tenantId } : null,
-      attempt,
-      finalAttempt,
-      log: createJobLogger(logs),
-      ...(triggerName !== undefined && { triggerName }),
-      ...(triggerEvent !== undefined && { triggerEvent }),
-      write: (qn: string, payload: unknown) => {
-        if (!dispatchWriteRef) {
-          throw new Error(
-            "JobContext.write called before dispatcher attached — call attachDispatcher() first",
-          );
-        }
-        return dispatchWriteRef.write(jobSystemUser, qn, payload, jobOrigin);
-      },
-      writeAs: (user: SessionUser, qn: string, payload: unknown) => {
-        if (!dispatchWriteRef) {
-          throw new Error(
-            "JobContext.writeAs called before dispatcher attached — call attachDispatcher() first",
-          );
-        }
-        return dispatchWriteRef.write(user, qn, payload, jobOrigin);
-      },
-      queryAs: (user: SessionUser, qn: string, payload: unknown) => {
-        if (!dispatchWriteRef) {
-          throw new Error(
-            "JobContext.queryAs called before dispatcher attached — call attachDispatcher() first",
-          );
-        }
-        return dispatchWriteRef.queryAs(user, qn, payload, jobOrigin);
-      },
-      queryAsMember: (userId: string, qn: string, payload: unknown) => {
-        if (!dispatchWriteRef) {
-          throw new Error(
-            "JobContext.queryAsMember called before dispatcher attached — call attachDispatcher() first",
-          );
-        }
-        // Lazy per-run reader, mirroring the handler-context reader.
-        memberReader ??= dispatchWriteRef.createMemberReader(tenantId);
-        return memberReader(userId, qn, payload);
-      },
-    };
-
-    await options.onJobStart?.(jobName, jobId, meta);
-
-    // Cross-process trace continuation: if the enqueuing code captured a
-    // parent span, start the job.execute span as its child. Works for event
-    // and manual triggers; cron jobs start a fresh root span.
-    const parentContext = readTraceContext(rawData);
-
-    // Correlation propagation: the scheduling request's correlationId was
-    // packed into _correlationId at dispatch time. Re-enter requestContext.run
-    // so event writes during this job stamp the same correlation as the
-    // request that scheduled it. Cron/boot jobs (no scheduler) start fresh
-    // — correlationId = new requestId, no parent causation.
-    const inheritedCorrelationId = (rawData["_correlationId"] as string | undefined) ?? undefined; // @cast-boundary dynamic-key
-    const jobRequestId = requestContext.generateId();
-    const jobCorrelationId = inheritedCorrelationId ?? jobRequestId;
-
-    const runInSpan = async (): Promise<void> => {
-      try {
-        if (writeOriginInvalid) {
-          throw new InternalError({
-            message:
-              `Job "${jobName}" received an unparseable _writeOrigin — refusing to run without ` +
-              "a trustworthy anonymous-root gate.",
-          });
-        }
-        await requestContext.run(
-          {
-            requestId: jobRequestId,
-            correlationId: jobCorrelationId,
-            // #3043 — events a job writes carry the job as their origin.
-            handler: jobName,
-            feature: qnScope(jobName),
-            writeOrigin: jobOrigin,
-          },
-          () => jobDef.handler(payload, jobContext),
-        );
-        // Stamped before the observer hook: a throwing onJobComplete must not
-        // make a run that actually succeeded look dead to the liveness alert.
-        if (context.meter) emitJobLastSuccess(context.meter, jobName);
-        const duration = Date.now() - startTime;
-        await options.onJobComplete?.(jobName, jobId, duration, logs, outcomeMeta(null));
-      } catch (err) {
-        const errorMsg = err instanceof Error ? err.message : String(err);
-        logs.push({ level: "error", message: errorMsg, timestamp: Temporal.Now.instant() });
-        await options.onJobFailed?.(
-          jobName,
-          jobId,
-          errorMsg,
-          logs,
-          outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl)),
-        );
-        throw err;
-      }
-    };
-
-    // Unified span creation: withSpan handles start/end + status/exception
-    // recording identically for both parent-context and no-parent paths.
-    // When parentContext is set, the new parent-aware StartSpanOptions
-    // plumbs it through to startSpan — no manual try/finally needed.
-    try {
-      await tracer.withSpan(
-        "job.execute",
-        {
-          attributes: {
-            "job.name": jobName,
-            "job.id": jobId,
-            "job.attempt": attempt,
-            "kumiko.tenant_id": tenantId,
-            // Lane-routing attributes (Welle 2.6). `run_in` is the job's
-            // declared lane (explicit or default-"worker"); `consumer_lane`
-            // is which runner actually executed it. They diverge in
-            // all-in-one (both lanes live in one process) but must match
-            // in split deploys — a mismatch in prod logs signals a
-            // misrouted job that slipped past the boot-validator.
-            "kumiko.job.run_in": laneForJob(jobDef),
-            // Omit attribute entirely when no consumer (enqueuer-only runner) —
-            // SpanAttributeValue doesn't accept undefined.
-            ...(consumerLane !== undefined ? { "kumiko.job.consumer_lane": consumerLane } : {}),
-          },
-          ...(parentContext ? { parent: parentContext } : {}),
-        },
-        runInSpan,
-      );
-    } finally {
-      // Release the sequential lock value-matched (Lua compare-and-delete
-      // inside DistributedLock). A TTL-expired lock that's been claimed by
-      // a different owner stays put — releasing it would break sequencing
-      // for the new owner.
-      if (sequentialToken && sequentialLock) {
-        await sequentialLock.release(jobName, sequentialToken);
-      }
-    }
-  }
-
-  // Boot gates run inline and are awaited, so a throwing gate rejects start()
-  // — and with it the entrypoint's start() and runProdApp's boot, before the
-  // process reports ready. Enqueuing them (as runOnBoot does) would only fail
-  // a queue job and let the deploy through (fw#2590).
-  async function runBootGates(lane: JobRunIn): Promise<void> {
-    for (const [name, jobDef] of allJobs) {
-      if (laneForJob(jobDef) !== lane) continue;
-      if (!jobDef.bootGate) continue;
-      try {
-        await handleJob({ id: bootJobIdForJobName(name), name, data: {}, attemptsMade: 0 });
-      } catch (err) {
-        // The worker is already consuming here; leaving its Redis connections
-        // open would keep the event loop alive and turn the aborted boot into
-        // a hanging process instead of a non-zero exit.
-        await worker?.close();
-        worker = null;
-        throw err;
-      }
-    }
-  }
-
-  const runnerApi: JobRunner = {
-    async start(): Promise<void> {
-      // skip: enqueuer-only runner — no BullMQ worker, no cron schedules,
-      // no boot jobs. The API-process (runLocalJobs=false) lands here; it
-      // still holds the queue-clients so dispatch()/handleEvent() can
-      // target the worker-lane queue, but nothing local consumes.
-      if (!consumerLane) {
-        return;
-      }
-
-      const consumerQueue = queues[consumerLane];
-      worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), handleJob, {
-        connection: redisOpts,
-        concurrency: 5,
-      });
-      worker.on("error", (err) => {
-        const log = stopping ? errorLogger.debug : errorLogger.error;
-        log("worker redis connection error", { error: err.message });
-      });
-      // A caller that calls stop() right after start() otherwise races the
-      // still-settling blocking connection: it rejects in-flight commands
-      // via ioredis's flushQueue() during close(), which isn't a listenable
-      // 'error' event — the only fix is to not return until both of the
-      // worker's connections (main + blocking) are ready (fw#1805). This
-      // mirrors the wait BullMQ already does internally for
-      // upsertJobScheduler()/add() below when the lane has a cron/boot job.
-      // Racing a timeout against it keeps an unreachable Redis from hanging
-      // start() forever — there's no worker health endpoint to notice.
-      const bootTimeout = timeoutReject(
-        bootRedisTimeoutMs,
-        `job-runner: Redis not reachable within ${bootRedisTimeoutMs}ms (lane=${consumerLane})`,
-      );
-      try {
-        await Promise.race([worker.waitUntilReady(), bootTimeout.promise]);
-      } finally {
-        bootTimeout.cancel();
-      }
-
-      // Ahead of cron/runOnBoot wiring on purpose: a failed gate must not
-      // leave recurring schedulers behind.
-      await runBootGates(consumerLane);
-
-      // Only schedule cron + boot for jobs that belong to this lane. Jobs
-      // assigned to the other lane get their cron/boot wiring from the
-      // runner running on that lane. Running both here would double-fire.
-      for (const [name, jobDef] of allJobs) {
-        if (laneForJob(jobDef) !== consumerLane) continue;
-        if ("cron" in jobDef.trigger) {
-          const schedulerId = schedulerIdForJobName(name);
-          const legacyId = legacySchedulerIdForJobName(name);
-          // Drop pre-sanitize scheduler ids so colon-form ghosts stop firing.
-          if (legacyId !== schedulerId) {
-            try {
-              await consumerQueue.removeJobScheduler(legacyId);
-            } catch {
-              // skip: legacy scheduler absent (fresh install / already purged)
-            }
-          }
-          await consumerQueue.upsertJobScheduler(
-            schedulerId,
-            { pattern: jobDef.trigger.cron },
-            {
-              name: jobDef.perTenant ? `_perTenant:${name}` : name,
-              data: {},
-              // Queue-level defaultJobOptions (jobRetentionOpts) covers
-              // retention; a count here would sweep queue-wide again and
-              // evict perTenant children and boot jobs.
-              opts: {
-                ...buildRetryBullOpts(jobDef),
-              },
-            },
-          );
-        }
-      }
-
-      // Persistent marker outside the swept completed/failed sets: a plain
-      // Redis hash at one key per consumer queue, fields = boot job ids
-      // already enqueued (BullMQ's IRedisClient has no set commands, only
-      // hash/hexists — a hash with dummy values does the same job). The job
-      // hash itself (removeOnComplete/-Fail: age-bound) is not a safe dedup
-      // target any more — retention now evicts it eventually, which would
-      // otherwise re-run a boot job "once per dataset" every time it ages
-      // out. Order matters: HEXISTS check, then add(), then HSET.
-      // Concurrent starts still dedupe on the existing job hash from add()'s
-      // own jobId no-op; a crash between add() and HSET dedupes again on the
-      // *next* boot (the job hash is still there) instead of losing the
-      // boot job forever, which HSET-first would risk if the process died
-      // before add() ran.
-      const bootEnqueuedKey = consumerQueue.toKey("kumiko-boot-enqueued");
-      for (const [name, jobDef] of allJobs) {
-        if (laneForJob(jobDef) !== consumerLane) continue;
-        if (jobDef.runOnBoot) {
-          const bootName = jobDef.perTenant ? `_perTenant:${name}` : name;
-          const bootJobId = bootJobIdForJobName(name);
-          const client = await consumerQueue.client;
-          const alreadyEnqueued = await client.hexists(bootEnqueuedKey, bootJobId);
-          if (alreadyEnqueued) continue;
-          await consumerQueue.add(
-            bootName,
-            {},
-            {
-              jobId: bootJobId,
-              ...buildRetryBullOpts(jobDef),
-            },
-          );
-          await client.hset(bootEnqueuedKey, { [bootJobId]: 1 });
-        }
-      }
-
-      // Skipped when no meter is wired (context.meter is optional, e.g. in
-      // tests without an observability provider).
-      if (context.meter) {
-        queueDepthTimer = await startQueueDepthPolling(consumerQueue, consumerLane, context.meter);
-      }
-    },
-
-    async stop(): Promise<void> {
-      stopping = true;
-      if (queueDepthTimer) {
-        clearInterval(queueDepthTimer);
-        queueDepthTimer = null;
-      }
-      if (worker) {
-        await worker.close();
-        worker = null;
-      }
-      // BullMQ close() while a Queue's connection is still 'initializing'
-      // rejects the in-flight INFO with "Connection is closed." on an emitter
-      // without listeners -> unhandled rejection (fw#1805). Wait for readiness
-      // first; a failed/timed-out wait must not block the close itself.
-      const readinessTimeout = timeoutReject(
-        bootRedisTimeoutMs,
-        `job-runner: Queue not ready within ${bootRedisTimeoutMs}ms during stop()`,
-      );
-      try {
-        await Promise.race([
-          Promise.all([queues.api.waitUntilReady(), queues.worker.waitUntilReady()]),
-          readinessTimeout.promise,
-        ]);
-      } catch {
-        // close() below still has to run
-      } finally {
-        readinessTimeout.cancel();
-      }
-      await Promise.all([queues.api.close(), queues.worker.close()]);
-      if (lockRedis) {
-        // quit() drains in-flight commands; disconnect() would cancel them
-        // and risk a half-released lock.
-        await lockRedis.quit();
-        lockRedis = null;
-      }
-    },
-
-    async dispatch(
-      jobName: string,
-      payload?: Record<string, unknown>,
-      meta?: JobMeta,
-    ): Promise<string> {
-      const jobDef = allJobs.get(jobName);
-      if (!jobDef) {
-        throw new Error(`Unknown job: ${jobName}`);
-      }
-
-      // Route to the job's declared lane, not the runner's consumer lane —
-      // an api-runner is allowed to enqueue a worker-lane job and vice
-      // versa (that's the whole point of both queues being held).
-      const targetQueue = queues[laneForJob(jobDef)];
-
-      // perTenant: dispatch the fan-out wrapper instead
-      if (jobDef.perTenant) {
-        const perTenantData: Record<string, unknown> = { ...(payload ?? {}) };
-        stampGatedWriteOrigin(perTenantData);
-        const job = await targetQueue.add(
-          `_perTenant:${jobName}`,
-          perTenantData,
-          buildRetryBullOpts(jobDef),
-        );
-        return job.id ?? "unknown";
-      }
-
-      // maxPerTenant guard: cap concurrent + waiting jobs of the same name
-      // for the same tenant. Orthogonal to the concurrency mode below — runs
-      // first because if we're over the limit nothing else matters.
-      // Requires a `_tenantId` in the payload to know which bucket to count
-      // against; without it the guard is inactive (system jobs, ambient
-      // dispatch). Fan-out children of perTenant jobs land here on their
-      // recursive queue.add and DO carry _tenantId.
-      if (jobDef.maxPerTenant !== undefined) {
-        const tenantId = (payload as { _tenantId?: string } | undefined)?._tenantId; // @cast-boundary dynamic-key
-        if (
-          tenantId !== undefined &&
-          (await isOverPerTenantLimit(jobName, tenantId, jobDef.maxPerTenant))
-        ) {
-          return "skipped:max-per-tenant";
-        }
-      }
-
-      const concurrency = jobDef.concurrency ?? "parallel";
-      const bullOpts: Record<string, unknown> = {};
-
-      switch (concurrency) {
-        case "skip": {
-          const active = await targetQueue.getActive();
-          const waiting = await targetQueue.getWaiting();
-          const isRunning = [...active, ...waiting].some((j) => j.name === jobName);
-          if (isRunning) {
-            return "skipped";
-          }
-          break;
-        }
-        case "replace": {
-          const waiting = await targetQueue.getWaiting();
-          for (const j of waiting) {
-            if (j.name === jobName && j.id) {
-              await j.remove();
-            }
-          }
-          break;
-        }
-        // case "sequential" is rejected at boot — see createJobRunner. Once
-        // the OSS-compatible implementation lands (per-name Redis lock),
-        // re-add the dispatch branch here.
-        case "debounce": {
-          const debounceMs = jobDef.debounceMs ?? 5000;
-          bullOpts["debounce"] = { id: jobName, ttl: debounceMs };
-          break;
-        }
-        default:
-          break;
-      }
-
-      Object.assign(bullOpts, buildRetryBullOpts(jobDef));
-      if (jobDef.timeout) bullOpts["timeout"] = jobDef.timeout;
-      if (meta?.priority !== undefined) bullOpts["priority"] = meta.priority;
-
-      // Pack meta into job data with _ prefix
-      const data: Record<string, unknown> = { ...payload };
-      if (meta?.triggeredById !== undefined) data["_triggeredById"] = meta.triggeredById;
-      if (meta?.payload !== undefined) data["_payload"] = meta.payload;
-      // Carry the enqueuing span context into the worker so job.execute shows
-      // as a child of the caller.
-      const traceCtx = captureTraceContext(tracer);
-      if (traceCtx) data[TRACE_CONTEXT_KEY] = traceCtx;
-      // Propagate correlation from the scheduling request into the job
-      // execution context. The worker re-enters requestContext.run with
-      // this value so ctx.appendEvent / executor writes during the job
-      // stamp the same correlation as the HTTP request that scheduled it.
-      const reqCtx = requestContext.get();
-      if (reqCtx?.correlationId) data["_correlationId"] = reqCtx.correlationId;
-      stampGatedWriteOrigin(data);
-
-      const job = await targetQueue.add(jobName, data, bullOpts);
-      return job.id ?? "unknown";
-    },
-
-    async handleEvent(
-      eventName: string,
-      payload: Record<string, unknown>,
-      user?: SessionUser,
-      triggerEvent?: JobTriggerEvent,
-    ): Promise<void> {
-      const traceCtx = captureTraceContext(tracer);
-      // Same correlation propagation as dispatch(): events triggered from
-      // within a request (or an MSP-apply running under requestContext.run)
-      // get their correlationId into job data so the job execution keeps
-      // the same causal chain.
-      const reqCtx = requestContext.get();
-      for (const [name, jobDef] of allJobs) {
-        if (!("on" in jobDef.trigger)) continue;
-        // skip: andere Trigger-Formen (cron, manual) reagieren nicht auf
-        // Events. Nur "on"-Trigger werden hier matched.
-        const triggerOn = jobDef.trigger.on;
-        const matches = Array.isArray(triggerOn)
-          ? triggerOn.includes(eventName)
-          : triggerOn === eventName;
-        if (!matches) continue;
-        if (!payloadMatchesTriggerFilter(jobDef.trigger.where, payload)) continue;
-        const data: Record<string, unknown> = { ...payload };
-        if (user) {
-          data["_tenantId"] = user.tenantId;
-          data["_triggeredById"] = user.id;
-        }
-        // Multi-Trigger: payload bekommt _triggerName damit der Handler
-        // weiß, welcher der N Trigger gefeuert hat. Bei Single-Trigger
-        // setzen wir es auch — kostet nichts und vereinfacht Handler-Code
-        // (kein "ist es Multi?"-Branch nötig).
-        data["_triggerName"] = eventName;
-        if (triggerEvent) data["_triggerEvent"] = triggerEvent;
-        if (traceCtx) data[TRACE_CONTEXT_KEY] = traceCtx;
-        if (reqCtx?.correlationId) data["_correlationId"] = reqCtx.correlationId;
-        // Same maxPerTenant guard as dispatch — events that fan into many
-        // jobs must respect the per-tenant cap or the limit is one-sided.
-        if (jobDef.maxPerTenant !== undefined && user?.tenantId !== undefined) {
-          if (await isOverPerTenantLimit(name, String(user.tenantId), jobDef.maxPerTenant)) {
-            continue;
-          }
-        }
-        stampGatedWriteOrigin(data);
-        // Route to the job's declared lane, not a fixed queue — that's
-        // the whole reason both queues are held.
-        await queues[laneForJob(jobDef)].add(name, data, buildRetryBullOpts(jobDef));
-      }
-    },
-    attachDispatcher(ref: DispatchWriteRef): void {
-      dispatchWriteRef = ref;
-    },
-    async countPendingJobs(): Promise<number> {
-      let total = 0;
-      for (const queue of Object.values(queues)) {
-        const counts = await queue.getJobCounts(
-          "waiting",
-          "active",
-          "prioritized",
-          "waiting-children",
-        );
-        total +=
-          (counts["waiting"] ?? 0) +
-          (counts["active"] ?? 0) +
-          (counts["prioritized"] ?? 0) +
-          (counts["waiting-children"] ?? 0);
-        // getJobCounts has no id-prefix filter for "delayed" — getJobs(["delayed"])
-        // loads the full Job objects (payload included) just to read .id; fine for
-        // test-draining's low volume, not a pattern for a prod-path hot loop.
-        const delayed = await queue.getJobs(["delayed"]);
-        total += delayed.filter((job) => !(job.id ?? "").startsWith("repeat:")).length;
-      }
-      return total;
-    },
-  };
-
-  selfRunner = runnerApi;
-  return runnerApi;
-}