@cosmicdrift/kumiko-framework 0.384.0 → 0.385.0

package/dist/jobs/job-runner.d.ts

@@ -1,77 +1,6 @@
 /// <reference types="temporal-polyfill/global" preserve="true" />
-import { type AppContext, type DispatchWriteRef, type JobRunIn, type JobTriggerEvent, type Registry, type SessionUser, type TenantId } from "../engine/types/index.ts";
-/**
- * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
- * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
- * treated as old repeatables — spawning a new scheduler entry every tick and
- * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
- * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
- */
-export declare function schedulerIdForJobName(jobName: string): string;
-export declare function bootJobIdForJobName(jobName: string): string;
-export declare const ORPHAN_SCHEDULER_GRACE_MS: number;
-export type JobLogEntry = {
-    level: "info" | "warn" | "error";
-    message: string;
-    timestamp: Temporal.Instant;
-};
-export type JobMeta = {
-    triggeredById?: string | undefined;
-    payload?: string | undefined;
-    attempt?: number | undefined;
-    priority?: number | undefined;
-    jobId?: string | undefined;
-    tenantVisibleRun?: {
-        readonly tenantId: string;
-        readonly subject: string | null;
-    } | undefined;
-};
-export type JobQueuedMeta = {
-    readonly tenantId: string;
-    readonly subject: string | null;
-};
-export type JobDroppedMeta = {
-    readonly tenantId: string;
-};
-export type JobOutcomeMeta = {
-    readonly tenantId: string;
-    readonly finalAttempt: boolean;
-    readonly tenantVisible?: {
-        readonly subject: string | null;
-        readonly messageKey: string | null;
-    } | undefined;
-    readonly tenantVisibleRun?: {
-        readonly subject: string | null;
-    } | undefined;
-};
-export type JobSubjectValue = string | number | boolean | null;
-export declare function serializeJobSubject(subject: Readonly<Record<string, JobSubjectValue>>): string;
-export type JobRunner = {
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    dispatch(jobName: string, payload?: Record<string, unknown>, meta?: JobMeta): Promise<string>;
-    handleEvent(eventName: string, payload: Record<string, unknown>, user?: SessionUser, triggerEvent?: JobTriggerEvent): Promise<void>;
-    attachDispatcher(ref: DispatchWriteRef): void;
-    countPendingJobs(): Promise<number>;
-};
-export type JobRunnerOptions = {
-    registry: Registry;
-    context: AppContext;
-    redisUrl: string;
-    consumerLane?: JobRunIn | undefined;
-    queueNamePrefix?: string | undefined;
-    bootRedisTimeoutMs?: number | undefined;
-    schedulerSeenRefreshMs?: number | undefined;
-    jobRetention?: {
-        completedAgeSec?: number | undefined;
-        failedAgeSec?: number | undefined;
-    } | undefined;
-    getActiveTenantIds?: () => Promise<TenantId[]>;
-    onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
-    onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
-    onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
-    onJobDeferred?: (jobName: string, jobId: string, meta: JobMeta) => void;
-    onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
-    onJobDropped?: (jobName: string, jobId: string, meta: JobDroppedMeta) => void;
-};
+import type { JobRunner, JobRunnerOptions } from "./job-runner-types.ts";
+export { serializeJobSubject } from "./job-runner-helpers.ts";
+export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.ts";
+export type { JobDroppedMeta, JobLogEntry, JobMeta, JobOutcomeMeta, JobQueuedMeta, JobRunner, JobRunnerOptions, JobSubjectValue, } from "./job-runner-types.ts";
 export declare function createJobRunner(options: JobRunnerOptions): JobRunner;

package/dist/jobs/job-runner.js

@@ -1,6 +1,5 @@
 /// <reference types="temporal-polyfill/global" preserve="true" />
 import { escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
-import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
 import { DelayedError, Queue, UnrecoverableError, Worker } from "bullmq";
 import { Redis } from "ioredis";
 import { requestContext } from "../api/request-context.js";
@@ -9,398 +8,22 @@ import { createDerivativesContext } from "../derivatives/derivatives-context.js"
 import { bridgeStub } from "../engine/bridge-stub.js";
 import { qnScope } from "../engine/qualified-name.js";
 import { createSystemUser } from "../engine/system-user.js";
-import { SYSTEM_TENANT_ID, } from "../engine/types/index.js";
+import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
 import { InternalError } from "../errors/index.js";
-import { isKumikoError } from "../errors/kumiko-error.js";
 import { createTenantFileContext } from "../files/file-handle.js";
 import { createFallbackLogger, scrubLogText } from "../logging/index.js";
-import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
+import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
 import { configAccessorFactoryOf } from "../pipeline/config-accessor-factory.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
 import { RedisKeys } from "../pipeline/redis-keys.js";
 import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js";
-import { TENANT_TIMEZONE_CONFIG_KEY } from "../pipeline/tenant-timezone-config-key.js";
-import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
-import { createTzContext, isValidIanaTimeZone } from "../time/index.js";
+import { buildPersonalDataGate, parseWriteOrigin } from "../pipeline/write-origin.js";
 import { cronRepeatOptions } from "./cron-repeat-options.js";
+import { BOOT_GATE_SLOW_WARN_MS, BOOT_REDIS_TIMEOUT_MS, buildRetryBullOpts, COMPLETED_JOB_RETENTION_AGE_SEC, captureTraceContext, createCrossTenantReads, createJobLogger, DEFAULT_MAX_DEFERS, DEFER_BACKOFF_MAX_DOUBLINGS, DEFER_COUNT_KEY, DEFER_DELAY_CAP_MS, FAILED_JOB_RETENTION_AGE_SEC, HANDLER_DEFER_COUNT_KEY, isJobTriggerEvent, jobSubjectKey, maxRetryWindowMs, missingDbProxy, parseRedisOpts, payloadMatchesTriggerFilter, priorAttemptsOf, readTraceContext, resolveJobTz, STOP_QUEUE_READY_TIMEOUT_MS, stampGatedWriteOrigin, startQueueDepthPolling, TRACE_CONTEXT_KEY, tenantFailureMessageKey, timeoutReject, } from "./job-runner-helpers.js";
+import { ACTIVE_TENANT_IDS_QUERY_NAME, bootJobIdForJobName, DEFAULT_QUEUE_NAME_PREFIX, markSchedulersSeen, ORPHAN_SCHEDULER_GRACE_MS, perTenantChildJobId, queueNameFor, removeOrphanJobSchedulers, schedulerIdForJobName, } from "./job-runner-schedulers.js";
 import { resolveMaxPerTenant } from "./max-per-tenant.js";
-// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
-// (a security-relevant escape hatch that must fail the run closed), a
-// malformed `_triggerEvent` only costs an idempotency-key convenience — the
-// run proceeds without it rather than failing over what would be a framework
-// bug, not untrusted input.
-function isJobTriggerEvent(value) {
-    if (!isPlainObject(value))
-        return false;
-    if (typeof value["id"] !== "string")
-        return false;
-    const headers = value["headers"];
-    if (typeof headers !== "object" || headers === null)
-        return false;
-    return Object.values(headers).every((v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean");
-}
-// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
-function stampGatedWriteOrigin(data) {
-    delete data["_writeOrigin"];
-    const origin = requestContext.get()?.writeOrigin;
-    if (origin && isPersonalDataGated(origin)) {
-        data["_writeOrigin"] = origin;
-    }
-}
-// Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
-// ("kumiko-jobs") — it must match between enqueuers and consumers, and an
-// accidental drift would silently drop jobs. Tests override via
-// `queueNamePrefix` for per-run isolation (stale jobs from a prior run
-// don't leak into a new test because the queue name includes a timestamp).
-const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
-function queueNameFor(prefix, lane) {
-    return `${prefix}-${lane}`;
-}
-// perTenant jobs need a source of truth for "which tenants are active" —
-// see createJobRunner's defaultGetActiveTenantIds below. The tenant feature
-// (bundled-features package) registers a query under this exact name when
-// mounted. framework must not depend on bundled-features (wrong direction —
-// bundled-features already depends on framework), so this is a name-only
-// coupling: createJobRunner looks the handler up in the registry by string
-// instead of importing it.
-const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
-/**
- * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
- * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
- * treated as old repeatables — spawning a new scheduler entry every tick and
- * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
- * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
- */
-export function schedulerIdForJobName(jobName) {
-    return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
-}
-// Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
-// a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
-export function bootJobIdForJobName(jobName) {
-    return `boot-${jobName.replace(/[.:]/g, "-")}`;
-}
-const SCHEDULER_PAGE_SIZE = 100;
-// Rolling deploys run old and new pods against the same Redis for a while; an
-// old runner restarting in that window must not delete the scheduler of a
-// cron job the new version just added.
-export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
-function schedulerSeenKey(queue) {
-    return queue.toKey("kumiko-scheduler-seen");
-}
-// "Seen" has to mean a live runner holds the scheduler, so runners re-stamp
-// their own ids periodically; a single stamp at start() would let a runner
-// restarting during a mixed-version window longer than the grace period delete
-// a scheduler a healthy newer pod still owns.
-async function markSchedulersSeen(queue, schedulerIds) {
-    // skip: no cron scheduler on this lane, nothing to stamp
-    if (schedulerIds.size === 0)
-        return;
-    const stamp = String(Date.now());
-    const entries = Object.fromEntries([...schedulerIds].map((id) => [id, stamp]));
-    await (await queue.client).hset(schedulerSeenKey(queue), entries);
-}
-// A cron job dropped from the app (or an older scheduler-id format) leaves its
-// scheduler in Redis, where it keeps firing jobs the worker no longer knows
-// ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
-async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lane, log) {
-    try {
-        // Collect first: removing while paging would shift the indices.
-        const existingKeys = [];
-        for (let start = 0;; start += SCHEDULER_PAGE_SIZE) {
-            const page = await queue.getJobSchedulers(start, start + SCHEDULER_PAGE_SIZE - 1);
-            for (const scheduler of page)
-                existingKeys.push(scheduler.key);
-            if (page.length < SCHEDULER_PAGE_SIZE)
-                break;
-        }
-        const client = await queue.client;
-        const seenAt = await client.hgetall(schedulerSeenKey(queue));
-        const now = Date.now();
-        for (const key of existingKeys) {
-            if (expectedSchedulerIds.has(key))
-                continue;
-            const lastSeen = Number(seenAt[key]);
-            if (key in seenAt && now - lastSeen < ORPHAN_SCHEDULER_GRACE_MS)
-                continue;
-            await queue.removeJobScheduler(key);
-            await client.hdel(schedulerSeenKey(queue), key);
-            log.info("removed orphan job scheduler", { schedulerId: key, lane });
-        }
-    }
-    catch (err) {
-        log.warn("orphan job scheduler cleanup failed", {
-            lane,
-            error: err instanceof Error ? err.message : String(err),
-        });
-    }
-}
-// Deterministic per-tenant child job id, derived from the perTenant
-// wrapper's own BullMQ job id. A cron tick's wrapper id is
-// `repeat:<schedulerId>:<millis>` (stable for that tick, including across
-// BullMQ's own retry/stalled-job redelivery of the *same* job); a manual
-// dispatch's wrapper id is BullMQ's auto-generated id, equally stable
-// across retries of that job. Reusing it means a wrapper that runs twice
-// for the same trigger (retry after failure, a Redis drop, an instance
-// restarting mid-run) re-derives the *same* child ids, and BullMQ's
-// existing-jobId add() no-ops the second batch instead of creating
-// duplicates — safe only as long as the child job hash outlives the
-// wrapper's retry window, enforced at construction against
-// COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
-// createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
-// (fw#1603/#1604) — the wrapper id already contains ":", so strip
-// separators from both halves before joining instead of interpolating raw.
-function perTenantChildJobId(wrapperJobId, tenantId) {
-    const sanitize = (value) => value.replace(/[.:]/g, "-");
-    return `child-${sanitize(wrapperJobId)}-${sanitize(tenantId)}`;
-}
-function createJobLogger(logs, baseContext = {}) {
-    function push(level, msg, data) {
-        const merged = { ...baseContext, ...data };
-        const message = Object.keys(merged).length > 0 ? `${msg} ${JSON.stringify(merged)}` : msg;
-        logs.push({ level, message, timestamp: Temporal.Now.instant() });
-    }
-    const logger = {
-        info(msg, data) {
-            push("info", msg, data);
-        },
-        warn(msg, data) {
-            push("warn", msg, data);
-        },
-        error(msg, data) {
-            push("error", msg, data);
-        },
-        // No-op by design, not missing coverage: JobLogEntry.level (and the
-        // persisted run-completed/-failed event schema in bundled-features/
-        // jobs/events.ts) only has info|warn|error — a job's debug() calls have
-        // nowhere durable to land.
-        debug() { },
-        child(context) {
-            return createJobLogger(logs, { ...baseContext, ...context });
-        },
-    };
-    return logger;
-}
-// Single source of the stored subject string: the writer (jobSubjectKey) and
-// the tenant failures query both go through it. Matching is exact — field names
-// and JSON types must equal what the writer stored (`"1"` is not `1`).
-export function serializeJobSubject(subject) {
-    return JSON.stringify(Object.keys(subject)
-        .sort()
-        .map((field) => [field, subject[field] ?? null]));
-}
-// Stable identity for the declared payload fields: sorted keys, so two runs
-// with the same subject values produce the same string and the later one
-// replaces the earlier record. Non-primitives would serialize into something
-// no caller can reconstruct for a lookup, so they fail the run loudly.
-function jobSubjectKey(jobName, payload, fields, declaration) {
-    if (fields === undefined || fields.length === 0)
-        return null;
-    const subject = {};
-    for (const field of fields) {
-        const value = payload[field] ?? null;
-        if (value !== null && typeof value === "object") {
-            throw new Error(`Job "${jobName}": ${declaration}.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
-        }
-        subject[field] = value;
-    }
-    return serializeJobSubject(subject);
-}
-async function resolveJobTz(config, context) {
-    const tenantTz = config ? await config(TENANT_TIMEZONE_CONFIG_KEY) : undefined;
-    return createTzContext({
-        ...(context.geoTzProvider !== undefined && { geoTz: context.geoTzProvider }),
-        ...(context.clock !== undefined && { clock: context.clock }),
-        tenant: typeof tenantTz === "string" && isValidIanaTimeZone(tenantTz) ? tenantTz : "UTC",
-    });
-}
-// Jobs that never touch ctx.db run fine on a runner built without one; a job
-// that does gets a named error instead of a bare TypeError on undefined.
-function missingDbProxy(jobName) {
-    // @cast-boundary db-operator — every access throws, so no TenantDb member is ever returned
-    return new Proxy({}, {
-        get() {
-            throw new InternalError({
-                message: `[jobs] job runner constructed without a db connection — ctx.db is unavailable for "${jobName}"`,
-            });
-        },
-    });
-}
-function createCrossTenantReads(systemModeDb, reason, report) {
-    return {
-        selectMany(table, where, options) {
-            report("cross-tenant-read", reason);
-            return systemModeDb.selectMany(table, where, options);
-        },
-        fetchOne(table, where) {
-            report("cross-tenant-read", reason);
-            return systemModeDb.fetchOne(table, where);
-        },
-        count(table, where) {
-            report("cross-tenant-read", reason);
-            return systemModeDb.count(table, where);
-        },
-    };
-}
-function priorAttemptsOf(data) {
-    const value = data["_priorAttempts"];
-    return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
-}
-// Only a translation key ever travels to the tenant: the thrown error's own
-// i18nKey when it carries one, otherwise the key declared at the job. The
-// error message itself can echo provider or payload content and stays on the
-// run row and in the run log.
-function tenantFailureMessageKey(err, declaration) {
-    // skip: job did not opt in — nothing tenant-visible to record
-    if (!declaration)
-        return null;
-    if (isKumikoError(err) && err.i18nKey.length > 0)
-        return err.i18nKey;
-    return declaration.messageKey;
-}
-// Serialized trace context lives under this key in the BullMQ job data.
-// Leading underscore matches the existing internal-meta convention
-// (_triggeredById, _tenantId, _payload).
-const TRACE_CONTEXT_KEY = "_traceContext";
-const DEFER_COUNT_KEY = "_deferCount";
-// Separate from the maxPerTenant counter so tenant-limit deferrals do not eat into maxDefers.
-const HANDLER_DEFER_COUNT_KEY = "_handlerDeferCount";
-const DEFAULT_MAX_DEFERS = 100;
-const DEFER_DELAY_CAP_MS = 30_000;
-const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
-function readTraceContext(data) {
-    const raw = data[TRACE_CONTEXT_KEY];
-    if (!raw || typeof raw !== "object")
-        return undefined;
-    const ctx = raw; // @cast-boundary engine-payload
-    if (!ctx.traceId || !ctx.spanId)
-        return undefined;
-    return { traceId: ctx.traceId, spanId: ctx.spanId };
-}
-function captureTraceContext(tracer) {
-    const span = tracer.getActiveSpan();
-    if (!span?.traceId)
-        return undefined;
-    return { traceId: span.traceId, spanId: span.spanId };
-}
-const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
-const QUEUE_DEPTH_WARN_INTERVAL_MS = 5 * 60_000;
-// kumiko_job_queue_depth: only the lane's own consumer polls — the count is
-// a global Redis-backed value (BullMQ, not per-process), so one reporter per
-// lane is enough. Fires once immediately (metrics exist before the first
-// poll tick) then on an interval; the caller stores + clears the handle.
-async function startQueueDepthPolling(queue, lane, meter) {
-    let lastWarnAt;
-    const pollQueueDepth = async () => {
-        try {
-            const counts = await queue.getJobCounts("waiting", "active", "delayed", "failed", "paused");
-            emitJobQueueDepth(meter, lane, counts);
-        }
-        catch (err) {
-            // Next poll retries; the warn is throttled so a Redis outage logs once per interval, not once per poll.
-            const nowMs = Date.now();
-            if (lastWarnAt === undefined || nowMs - lastWarnAt >= QUEUE_DEPTH_WARN_INTERVAL_MS) {
-                lastWarnAt = nowMs;
-                createFallbackLogger("job-runner").warn("queue depth poll failed", {
-                    lane,
-                    error: scrubLogText(err instanceof Error ? err.message : String(err)),
-                });
-            }
-        }
-    };
-    await pollQueueDepth();
-    return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
-}
-function parseRedisOpts(url) {
-    const parsed = new URL(url);
-    const result = {
-        host: parsed.hostname,
-        port: Number(parsed.port) || 6379,
-    };
-    if (parsed.pathname.length > 1) {
-        result.db = Number(parsed.pathname.slice(1));
-    }
-    return result;
-}
-// redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
-// would otherwise hang start() forever with no health endpoint to notice.
-const BOOT_REDIS_TIMEOUT_MS = 10_000;
-// Shutdown must fit a k8s grace period even when Redis is down, so the
-// readiness wait in stop() is capped well below the boot timeout.
-const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
-// A gate without an explicit `timeout` may legitimately run long (migration-style
-// checks), so it is never rejected; a hang just must not stay silent.
-const BOOT_GATE_SLOW_WARN_MS = 60_000;
-function timeoutReject(ms, message) {
-    let timer;
-    const promise = new Promise((_, reject) => {
-        timer = setTimeout(() => reject(new Error(message)), ms);
-    });
-    return {
-        promise,
-        cancel: () => {
-            if (timer !== undefined)
-                clearTimeout(timer);
-        },
-    };
-}
-// Default base delay when a job opts into backoff without an explicit
-// delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
-// NaN/undefined, which is falsy — the job retries immediately instead of
-// waiting.
-const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
-// Shared by every enqueue path (dispatch(), handleEvent(), cron, runOnBoot,
-// perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
-// set must retry the same way regardless of how it got enqueued, or it
-// fails for good on the very first error on whichever path skips this.
-function buildRetryBullOpts(jobDef, priorAttempts = 0) {
-    const opts = {};
-    if (jobDef.retries !== undefined)
-        opts.attempts = Math.max(1, jobDef.retries + 1 - priorAttempts);
-    if (jobDef.backoff) {
-        opts.backoff =
-            typeof jobDef.backoff === "string"
-                ? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
-                : {
-                    type: jobDef.backoff.type,
-                    delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
-                };
-    }
-    return opts;
-}
-// Equality filter on top-level payload fields for a job's trigger.where —
-// lets N jobs share one broad event QN, partitioned by a payload
-// discriminant instead of needing N distinct event types.
-function payloadMatchesTriggerFilter(where, payload) {
-    if (!where)
-        return true;
-    return Object.entries(where).every(([key, value]) => payload[key] === value);
-}
-// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
-// OTHER jobs in the same completed/failed zset (moveToFinished lua,
-// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
-// the sweep only runs lazily on a later finish into that set. Per-job
-// retention is therefore meaningless on a shared queue — only queue-wide,
-// age-only retention (no count) is safe: a count would evict perTenant
-// children still inside their wrapper's retry window, and boot jobs. The
-// completed age must stay above every perTenant wrapper's own worst-case
-// retry window (see maxRetryWindowMs and the invariant check in
-// createJobRunner below), because child dedup (perTenantChildJobId) relies
-// on the children still existing in Redis when a wrapper retry re-derives
-// their ids. The margin here is generous — queue wait time isn't bounded by
-// backoff — to cover realistic windows. Audit trail lives in read_job_runs,
-// not BullMQ (fw#3199).
-const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
-const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
-// Total wall-clock time BullMQ can hold a perTenant wrapper across all its
-// retries, consistent with buildRetryBullOpts's own backoff computation.
-function maxRetryWindowMs(jobDef) {
-    if (!jobDef.backoff || jobDef.retries === undefined)
-        return 0;
-    const delayMs = (typeof jobDef.backoff === "string" ? undefined : jobDef.backoff.delayMs) ??
-        DEFAULT_JOB_BACKOFF_DELAY_MS;
-    const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
-    return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
-}
+export { serializeJobSubject } from "./job-runner-helpers.js";
+export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.js";
 export function createJobRunner(options) {
     const { registry, context, redisUrl, consumerLane } = options;
     const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;

package/dist/pipeline/dispatch-shared.js

@@ -576,8 +576,8 @@ readOnlyForTenant = false) {
             await appendDomainEvent(ctx, args, user, tx, registry.getHandlerFeature(type));
         },
         // Savepoint-scoped append: catches a losing writer's VersionConflictError
-        // without poisoning the rest of the handler's transaction. Bun.SQL/
-        // postgres.js abort the WHOLE begin() on an uncaught statement error
+        // without poisoning the rest of the handler's transaction. postgres.js
+        // aborts the WHOLE begin() on an uncaught statement error
         // (SQLSTATE 25P02) even if the JS error is caught — runInSavepoint
         // wraps the append in a real SAVEPOINT so only that nested scope rolls
         // back on conflict, and subsequent statements in the outer tx (e.g. a

package/dist/pipeline/event-dispatcher.js

@@ -21,7 +21,6 @@ const CONNECTION_LOSS_CODES = new Set([
     "CONNECT_TIMEOUT",
     "ECONNREFUSED",
     "ECONNRESET",
-    "ERR_POSTGRES_CONNECTION_CLOSED",
     "57P01",
 ]);
 function isConnectionLossError(e) {
@@ -506,7 +505,7 @@ export function createEventDispatcher(options) {
                         // Fires on initial connect and again when the driver's own
                         // re-LISTEN lands after a drop; during an outage that re-LISTEN
                         // waits in its connection backoff (postgres-js: measured 10-30s
-                        // after the DB returns, Bun.SQL: up to 32s), polling covers
+                        // after the DB returns), polling covers
                         // delivery meanwhile. Never unlisten()
                         // + listen() by hand: that leaves duplicate listeners.
                         // ponytail: no own LISTEN reconnect, the backoff gap is polled;
@@ -534,7 +533,7 @@ export function createEventDispatcher(options) {
                 timer = null;
             }
             if (pgSubscription) {
-                // Called on the subscription: Bun.SQL's unlisten is a prototype method.
+                // Called on the subscription, not detached: unlisten may rely on `this`.
                 await pgSubscription.unlisten().catch(() => {
                     // skip: unlisten failure only matters during shutdown — the
                     // subscription is being torn down anyway.

package/dist/schema-cli.js

@@ -38,6 +38,9 @@ function withFileStoragePresence(env) {
 function isStringSet(value) {
     return value instanceof Set && [...value].every((entry) => typeof entry === "string");
 }
+function isNonEmptyStringArray(value) {
+    return Array.isArray(value) && value.every((entry) => typeof entry === "string" && entry !== "");
+}
 function isBoolean(value) {
     return typeof value === "boolean";
 }
@@ -64,7 +67,9 @@ function parseValidateBootOptions(raw) {
     const navAllowlist = readOptionalOption(raw, "navAllowlist", isStringSet, "a Set<string>");
     const navAllowlistExempt = readOptionalOption(raw, "navAllowlistExempt", isStringSet, "a Set<string>");
     const warnOnUniqueAccessRoles = readOptionalOption(raw, "warnOnUniqueAccessRoles", isBoolean, "a boolean");
+    const requiredLocales = readOptionalOption(raw, "requiredLocales", isNonEmptyStringArray, "an array of non-empty strings");
     return {
+        ...(requiredLocales !== undefined && { requiredLocales }),
         ...(navAllowlist !== undefined && { navAllowlist }),
         ...(navAllowlistExempt !== undefined && { navAllowlistExempt }),
         ...(warnOnUniqueAccessRoles !== undefined && { warnOnUniqueAccessRoles }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-framework",
-  "version": "0.384.0",
+  "version": "0.385.0",
   "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -199,7 +199,7 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-types": "0.384.0",
+    "@cosmicdrift/kumiko-types": "0.385.0",
     "bullmq": "^5.76.7",
     "bun-types": "^1.3.13",
     "hono": "^4.13.1",
@@ -215,7 +215,7 @@
     "zod": "^4.4.3"
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-testing": "0.384.0",
+    "@cosmicdrift/kumiko-testing": "0.385.0",
     "bun-types": "^1.3.13",
     "pino-pretty": "^13.1.3"
   },

package/src/changes.json

@@ -1,4 +1,17 @@
 [
+  {
+    "version": "0.385.0",
+    "type": "improvement",
+    "title": "requiredLocales boot option",
+    "detail": "`validateBootOptions.requiredLocales` (also `VALIDATE_BOOT_OPTIONS` in kumiko/schema.ts) makes boot fail when any translation key of the mounted features lacks one of the listed locales. Locales are combined across features, so locale packages close the gaps of the features they translate. Without the option nothing changes. This replaces the locale list the i18n-keys guard read from `src/i18n-guard-locales.ts` until 0.384.0; without that file it checked `[\"de\", \"en\"]`.",
+    "migration": "Define the option once, e.g. `{ requiredLocales: [\"de\", \"en\"] }`, pass it as `validateBootOptions` wherever the app boots (`runProdApp`, `runDevApp`, workers) and re-export it as `VALIDATE_BOOT_OPTIONS` from `kumiko/schema.ts` so `kumiko-schema validate` checks it in CI. The check covers framework and bundled keys too; a key missing a locale fails boot with the feature and key named."
+  },
+  {
+    "version": "0.385.0",
+    "type": "improvement",
+    "title": "selectMany distinctOn for newest-row-per-key reads",
+    "detail": "SelectOptions.distinctOn takes field names and emits SELECT DISTINCT ON over them, keeping the first row per key by orderBy, so newest-row-per-key reads no longer need unsafeRaw. orderBy has to start with the distinctOn fields in the same order, otherwise selectMany throws; without orderBy it defaults to them ascending. Tenant scoping applies as for every selectMany. Error classification and driver detection drop the branches for the removed Bun.SQL driver (ERR_POSTGRES_* codes, SQLSTATE in errno, millisecond timeouts), which postgres-js never produces."
+  },
   {
     "version": "0.384.0",
     "type": "breaking",