@cosmicdrift/kumiko-framework 0.384.0 → 0.386.0

package/dist/jobs/job-runner-types.d.ts

@@ -0,0 +1,65 @@
+/// <reference types="temporal-polyfill/global" preserve="true" />
+import type { AppContext, DispatchWriteRef, JobRunIn, JobTriggerEvent, Registry, SessionUser, TenantId } from "../engine/types/index.ts";
+export type JobLogEntry = {
+    level: "info" | "warn" | "error";
+    message: string;
+    timestamp: Temporal.Instant;
+};
+export type JobMeta = {
+    triggeredById?: string | undefined;
+    payload?: string | undefined;
+    attempt?: number | undefined;
+    priority?: number | undefined;
+    jobId?: string | undefined;
+    tenantVisibleRun?: {
+        readonly tenantId: string;
+        readonly subject: string | null;
+    } | undefined;
+};
+export type JobQueuedMeta = {
+    readonly tenantId: string;
+    readonly subject: string | null;
+};
+export type JobDroppedMeta = {
+    readonly tenantId: string;
+};
+export type JobOutcomeMeta = {
+    readonly tenantId: string;
+    readonly finalAttempt: boolean;
+    readonly tenantVisible?: {
+        readonly subject: string | null;
+        readonly messageKey: string | null;
+    } | undefined;
+    readonly tenantVisibleRun?: {
+        readonly subject: string | null;
+    } | undefined;
+};
+export type JobSubjectValue = string | number | boolean | null;
+export type JobRunner = {
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    dispatch(jobName: string, payload?: Record<string, unknown>, meta?: JobMeta): Promise<string>;
+    handleEvent(eventName: string, payload: Record<string, unknown>, user?: SessionUser, triggerEvent?: JobTriggerEvent): Promise<void>;
+    attachDispatcher(ref: DispatchWriteRef): void;
+    countPendingJobs(): Promise<number>;
+};
+export type JobRunnerOptions = {
+    registry: Registry;
+    context: AppContext;
+    redisUrl: string;
+    consumerLane?: JobRunIn | undefined;
+    queueNamePrefix?: string | undefined;
+    bootRedisTimeoutMs?: number | undefined;
+    schedulerSeenRefreshMs?: number | undefined;
+    jobRetention?: {
+        completedAgeSec?: number | undefined;
+        failedAgeSec?: number | undefined;
+    } | undefined;
+    getActiveTenantIds?: () => Promise<TenantId[]>;
+    onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
+    onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
+    onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
+    onJobDeferred?: (jobName: string, jobId: string, meta: JobMeta) => void;
+    onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
+    onJobDropped?: (jobName: string, jobId: string, meta: JobDroppedMeta) => void;
+};

package/dist/jobs/job-runner-types.js

@@ -0,0 +1,2 @@
+/// <reference types="temporal-polyfill/global" preserve="true" />
+export {};

package/dist/jobs/job-runner.d.ts

@@ -1,77 +1,6 @@
 /// <reference types="temporal-polyfill/global" preserve="true" />
-import { type AppContext, type DispatchWriteRef, type JobRunIn, type JobTriggerEvent, type Registry, type SessionUser, type TenantId } from "../engine/types/index.ts";
-/**
- * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
- * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
- * treated as old repeatables — spawning a new scheduler entry every tick and
- * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
- * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
- */
-export declare function schedulerIdForJobName(jobName: string): string;
-export declare function bootJobIdForJobName(jobName: string): string;
-export declare const ORPHAN_SCHEDULER_GRACE_MS: number;
-export type JobLogEntry = {
-    level: "info" | "warn" | "error";
-    message: string;
-    timestamp: Temporal.Instant;
-};
-export type JobMeta = {
-    triggeredById?: string | undefined;
-    payload?: string | undefined;
-    attempt?: number | undefined;
-    priority?: number | undefined;
-    jobId?: string | undefined;
-    tenantVisibleRun?: {
-        readonly tenantId: string;
-        readonly subject: string | null;
-    } | undefined;
-};
-export type JobQueuedMeta = {
-    readonly tenantId: string;
-    readonly subject: string | null;
-};
-export type JobDroppedMeta = {
-    readonly tenantId: string;
-};
-export type JobOutcomeMeta = {
-    readonly tenantId: string;
-    readonly finalAttempt: boolean;
-    readonly tenantVisible?: {
-        readonly subject: string | null;
-        readonly messageKey: string | null;
-    } | undefined;
-    readonly tenantVisibleRun?: {
-        readonly subject: string | null;
-    } | undefined;
-};
-export type JobSubjectValue = string | number | boolean | null;
-export declare function serializeJobSubject(subject: Readonly<Record<string, JobSubjectValue>>): string;
-export type JobRunner = {
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    dispatch(jobName: string, payload?: Record<string, unknown>, meta?: JobMeta): Promise<string>;
-    handleEvent(eventName: string, payload: Record<string, unknown>, user?: SessionUser, triggerEvent?: JobTriggerEvent): Promise<void>;
-    attachDispatcher(ref: DispatchWriteRef): void;
-    countPendingJobs(): Promise<number>;
-};
-export type JobRunnerOptions = {
-    registry: Registry;
-    context: AppContext;
-    redisUrl: string;
-    consumerLane?: JobRunIn | undefined;
-    queueNamePrefix?: string | undefined;
-    bootRedisTimeoutMs?: number | undefined;
-    schedulerSeenRefreshMs?: number | undefined;
-    jobRetention?: {
-        completedAgeSec?: number | undefined;
-        failedAgeSec?: number | undefined;
-    } | undefined;
-    getActiveTenantIds?: () => Promise<TenantId[]>;
-    onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
-    onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
-    onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
-    onJobDeferred?: (jobName: string, jobId: string, meta: JobMeta) => void;
-    onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
-    onJobDropped?: (jobName: string, jobId: string, meta: JobDroppedMeta) => void;
-};
+import type { JobRunner, JobRunnerOptions } from "./job-runner-types.ts";
+export { serializeJobSubject } from "./job-runner-helpers.ts";
+export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.ts";
+export type { JobDroppedMeta, JobLogEntry, JobMeta, JobOutcomeMeta, JobQueuedMeta, JobRunner, JobRunnerOptions, JobSubjectValue, } from "./job-runner-types.ts";
 export declare function createJobRunner(options: JobRunnerOptions): JobRunner;

package/dist/jobs/job-runner.js

@@ -1,406 +1,30 @@
 /// <reference types="temporal-polyfill/global" preserve="true" />
-import { escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
-import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
+import { crossTenantReadsGrantFor, escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
 import { DelayedError, Queue, UnrecoverableError, Worker } from "bullmq";
 import { Redis } from "ioredis";
 import { requestContext } from "../api/request-context.js";
+import { createCrossTenantReads } from "../db/cross-tenant-reads.js";
 import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
 import { createDerivativesContext } from "../derivatives/derivatives-context.js";
 import { bridgeStub } from "../engine/bridge-stub.js";
 import { qnScope } from "../engine/qualified-name.js";
 import { createSystemUser } from "../engine/system-user.js";
-import { SYSTEM_TENANT_ID, } from "../engine/types/index.js";
+import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
 import { InternalError } from "../errors/index.js";
-import { isKumikoError } from "../errors/kumiko-error.js";
 import { createTenantFileContext } from "../files/file-handle.js";
 import { createFallbackLogger, scrubLogText } from "../logging/index.js";
-import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
+import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
 import { configAccessorFactoryOf } from "../pipeline/config-accessor-factory.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
 import { RedisKeys } from "../pipeline/redis-keys.js";
 import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js";
-import { TENANT_TIMEZONE_CONFIG_KEY } from "../pipeline/tenant-timezone-config-key.js";
-import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
-import { createTzContext, isValidIanaTimeZone } from "../time/index.js";
+import { buildPersonalDataGate, parseWriteOrigin } from "../pipeline/write-origin.js";
 import { cronRepeatOptions } from "./cron-repeat-options.js";
+import { BOOT_GATE_SLOW_WARN_MS, BOOT_REDIS_TIMEOUT_MS, buildRetryBullOpts, COMPLETED_JOB_RETENTION_AGE_SEC, captureTraceContext, createJobLogger, DEFAULT_MAX_DEFERS, DEFER_BACKOFF_MAX_DOUBLINGS, DEFER_COUNT_KEY, DEFER_DELAY_CAP_MS, FAILED_JOB_RETENTION_AGE_SEC, HANDLER_DEFER_COUNT_KEY, isJobTriggerEvent, jobSubjectKey, maxRetryWindowMs, missingDbProxy, parseRedisOpts, payloadMatchesTriggerFilter, priorAttemptsOf, readTraceContext, resolveJobTz, STOP_QUEUE_READY_TIMEOUT_MS, stampGatedWriteOrigin, startQueueDepthPolling, TRACE_CONTEXT_KEY, tenantFailureMessageKey, timeoutReject, } from "./job-runner-helpers.js";
+import { ACTIVE_TENANT_IDS_QUERY_NAME, bootJobIdForJobName, DEFAULT_QUEUE_NAME_PREFIX, markSchedulersSeen, ORPHAN_SCHEDULER_GRACE_MS, perTenantChildJobId, queueNameFor, removeOrphanJobSchedulers, schedulerIdForJobName, } from "./job-runner-schedulers.js";
 import { resolveMaxPerTenant } from "./max-per-tenant.js";
-// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
-// (a security-relevant escape hatch that must fail the run closed), a
-// malformed `_triggerEvent` only costs an idempotency-key convenience — the
-// run proceeds without it rather than failing over what would be a framework
-// bug, not untrusted input.
-function isJobTriggerEvent(value) {
-    if (!isPlainObject(value))
-        return false;
-    if (typeof value["id"] !== "string")
-        return false;
-    const headers = value["headers"];
-    if (typeof headers !== "object" || headers === null)
-        return false;
-    return Object.values(headers).every((v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean");
-}
-// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
-function stampGatedWriteOrigin(data) {
-    delete data["_writeOrigin"];
-    const origin = requestContext.get()?.writeOrigin;
-    if (origin && isPersonalDataGated(origin)) {
-        data["_writeOrigin"] = origin;
-    }
-}
-// Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
-// ("kumiko-jobs") — it must match between enqueuers and consumers, and an
-// accidental drift would silently drop jobs. Tests override via
-// `queueNamePrefix` for per-run isolation (stale jobs from a prior run
-// don't leak into a new test because the queue name includes a timestamp).
-const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
-function queueNameFor(prefix, lane) {
-    return `${prefix}-${lane}`;
-}
-// perTenant jobs need a source of truth for "which tenants are active" —
-// see createJobRunner's defaultGetActiveTenantIds below. The tenant feature
-// (bundled-features package) registers a query under this exact name when
-// mounted. framework must not depend on bundled-features (wrong direction —
-// bundled-features already depends on framework), so this is a name-only
-// coupling: createJobRunner looks the handler up in the registry by string
-// instead of importing it.
-const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
-/**
- * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
- * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
- * treated as old repeatables — spawning a new scheduler entry every tick and
- * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
- * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
- */
-export function schedulerIdForJobName(jobName) {
-    return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
-}
-// Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
-// a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
-export function bootJobIdForJobName(jobName) {
-    return `boot-${jobName.replace(/[.:]/g, "-")}`;
-}
-const SCHEDULER_PAGE_SIZE = 100;
-// Rolling deploys run old and new pods against the same Redis for a while; an
-// old runner restarting in that window must not delete the scheduler of a
-// cron job the new version just added.
-export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
-function schedulerSeenKey(queue) {
-    return queue.toKey("kumiko-scheduler-seen");
-}
-// "Seen" has to mean a live runner holds the scheduler, so runners re-stamp
-// their own ids periodically; a single stamp at start() would let a runner
-// restarting during a mixed-version window longer than the grace period delete
-// a scheduler a healthy newer pod still owns.
-async function markSchedulersSeen(queue, schedulerIds) {
-    // skip: no cron scheduler on this lane, nothing to stamp
-    if (schedulerIds.size === 0)
-        return;
-    const stamp = String(Date.now());
-    const entries = Object.fromEntries([...schedulerIds].map((id) => [id, stamp]));
-    await (await queue.client).hset(schedulerSeenKey(queue), entries);
-}
-// A cron job dropped from the app (or an older scheduler-id format) leaves its
-// scheduler in Redis, where it keeps firing jobs the worker no longer knows
-// ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
-async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lane, log) {
-    try {
-        // Collect first: removing while paging would shift the indices.
-        const existingKeys = [];
-        for (let start = 0;; start += SCHEDULER_PAGE_SIZE) {
-            const page = await queue.getJobSchedulers(start, start + SCHEDULER_PAGE_SIZE - 1);
-            for (const scheduler of page)
-                existingKeys.push(scheduler.key);
-            if (page.length < SCHEDULER_PAGE_SIZE)
-                break;
-        }
-        const client = await queue.client;
-        const seenAt = await client.hgetall(schedulerSeenKey(queue));
-        const now = Date.now();
-        for (const key of existingKeys) {
-            if (expectedSchedulerIds.has(key))
-                continue;
-            const lastSeen = Number(seenAt[key]);
-            if (key in seenAt && now - lastSeen < ORPHAN_SCHEDULER_GRACE_MS)
-                continue;
-            await queue.removeJobScheduler(key);
-            await client.hdel(schedulerSeenKey(queue), key);
-            log.info("removed orphan job scheduler", { schedulerId: key, lane });
-        }
-    }
-    catch (err) {
-        log.warn("orphan job scheduler cleanup failed", {
-            lane,
-            error: err instanceof Error ? err.message : String(err),
-        });
-    }
-}
-// Deterministic per-tenant child job id, derived from the perTenant
-// wrapper's own BullMQ job id. A cron tick's wrapper id is
-// `repeat:<schedulerId>:<millis>` (stable for that tick, including across
-// BullMQ's own retry/stalled-job redelivery of the *same* job); a manual
-// dispatch's wrapper id is BullMQ's auto-generated id, equally stable
-// across retries of that job. Reusing it means a wrapper that runs twice
-// for the same trigger (retry after failure, a Redis drop, an instance
-// restarting mid-run) re-derives the *same* child ids, and BullMQ's
-// existing-jobId add() no-ops the second batch instead of creating
-// duplicates — safe only as long as the child job hash outlives the
-// wrapper's retry window, enforced at construction against
-// COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
-// createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
-// (fw#1603/#1604) — the wrapper id already contains ":", so strip
-// separators from both halves before joining instead of interpolating raw.
-function perTenantChildJobId(wrapperJobId, tenantId) {
-    const sanitize = (value) => value.replace(/[.:]/g, "-");
-    return `child-${sanitize(wrapperJobId)}-${sanitize(tenantId)}`;
-}
-function createJobLogger(logs, baseContext = {}) {
-    function push(level, msg, data) {
-        const merged = { ...baseContext, ...data };
-        const message = Object.keys(merged).length > 0 ? `${msg} ${JSON.stringify(merged)}` : msg;
-        logs.push({ level, message, timestamp: Temporal.Now.instant() });
-    }
-    const logger = {
-        info(msg, data) {
-            push("info", msg, data);
-        },
-        warn(msg, data) {
-            push("warn", msg, data);
-        },
-        error(msg, data) {
-            push("error", msg, data);
-        },
-        // No-op by design, not missing coverage: JobLogEntry.level (and the
-        // persisted run-completed/-failed event schema in bundled-features/
-        // jobs/events.ts) only has info|warn|error — a job's debug() calls have
-        // nowhere durable to land.
-        debug() { },
-        child(context) {
-            return createJobLogger(logs, { ...baseContext, ...context });
-        },
-    };
-    return logger;
-}
-// Single source of the stored subject string: the writer (jobSubjectKey) and
-// the tenant failures query both go through it. Matching is exact — field names
-// and JSON types must equal what the writer stored (`"1"` is not `1`).
-export function serializeJobSubject(subject) {
-    return JSON.stringify(Object.keys(subject)
-        .sort()
-        .map((field) => [field, subject[field] ?? null]));
-}
-// Stable identity for the declared payload fields: sorted keys, so two runs
-// with the same subject values produce the same string and the later one
-// replaces the earlier record. Non-primitives would serialize into something
-// no caller can reconstruct for a lookup, so they fail the run loudly.
-function jobSubjectKey(jobName, payload, fields, declaration) {
-    if (fields === undefined || fields.length === 0)
-        return null;
-    const subject = {};
-    for (const field of fields) {
-        const value = payload[field] ?? null;
-        if (value !== null && typeof value === "object") {
-            throw new Error(`Job "${jobName}": ${declaration}.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
-        }
-        subject[field] = value;
-    }
-    return serializeJobSubject(subject);
-}
-async function resolveJobTz(config, context) {
-    const tenantTz = config ? await config(TENANT_TIMEZONE_CONFIG_KEY) : undefined;
-    return createTzContext({
-        ...(context.geoTzProvider !== undefined && { geoTz: context.geoTzProvider }),
-        ...(context.clock !== undefined && { clock: context.clock }),
-        tenant: typeof tenantTz === "string" && isValidIanaTimeZone(tenantTz) ? tenantTz : "UTC",
-    });
-}
-// Jobs that never touch ctx.db run fine on a runner built without one; a job
-// that does gets a named error instead of a bare TypeError on undefined.
-function missingDbProxy(jobName) {
-    // @cast-boundary db-operator — every access throws, so no TenantDb member is ever returned
-    return new Proxy({}, {
-        get() {
-            throw new InternalError({
-                message: `[jobs] job runner constructed without a db connection — ctx.db is unavailable for "${jobName}"`,
-            });
-        },
-    });
-}
-function createCrossTenantReads(systemModeDb, reason, report) {
-    return {
-        selectMany(table, where, options) {
-            report("cross-tenant-read", reason);
-            return systemModeDb.selectMany(table, where, options);
-        },
-        fetchOne(table, where) {
-            report("cross-tenant-read", reason);
-            return systemModeDb.fetchOne(table, where);
-        },
-        count(table, where) {
-            report("cross-tenant-read", reason);
-            return systemModeDb.count(table, where);
-        },
-    };
-}
-function priorAttemptsOf(data) {
-    const value = data["_priorAttempts"];
-    return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
-}
-// Only a translation key ever travels to the tenant: the thrown error's own
-// i18nKey when it carries one, otherwise the key declared at the job. The
-// error message itself can echo provider or payload content and stays on the
-// run row and in the run log.
-function tenantFailureMessageKey(err, declaration) {
-    // skip: job did not opt in — nothing tenant-visible to record
-    if (!declaration)
-        return null;
-    if (isKumikoError(err) && err.i18nKey.length > 0)
-        return err.i18nKey;
-    return declaration.messageKey;
-}
-// Serialized trace context lives under this key in the BullMQ job data.
-// Leading underscore matches the existing internal-meta convention
-// (_triggeredById, _tenantId, _payload).
-const TRACE_CONTEXT_KEY = "_traceContext";
-const DEFER_COUNT_KEY = "_deferCount";
-// Separate from the maxPerTenant counter so tenant-limit deferrals do not eat into maxDefers.
-const HANDLER_DEFER_COUNT_KEY = "_handlerDeferCount";
-const DEFAULT_MAX_DEFERS = 100;
-const DEFER_DELAY_CAP_MS = 30_000;
-const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
-function readTraceContext(data) {
-    const raw = data[TRACE_CONTEXT_KEY];
-    if (!raw || typeof raw !== "object")
-        return undefined;
-    const ctx = raw; // @cast-boundary engine-payload
-    if (!ctx.traceId || !ctx.spanId)
-        return undefined;
-    return { traceId: ctx.traceId, spanId: ctx.spanId };
-}
-function captureTraceContext(tracer) {
-    const span = tracer.getActiveSpan();
-    if (!span?.traceId)
-        return undefined;
-    return { traceId: span.traceId, spanId: span.spanId };
-}
-const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
-const QUEUE_DEPTH_WARN_INTERVAL_MS = 5 * 60_000;
-// kumiko_job_queue_depth: only the lane's own consumer polls — the count is
-// a global Redis-backed value (BullMQ, not per-process), so one reporter per
-// lane is enough. Fires once immediately (metrics exist before the first
-// poll tick) then on an interval; the caller stores + clears the handle.
-async function startQueueDepthPolling(queue, lane, meter) {
-    let lastWarnAt;
-    const pollQueueDepth = async () => {
-        try {
-            const counts = await queue.getJobCounts("waiting", "active", "delayed", "failed", "paused");
-            emitJobQueueDepth(meter, lane, counts);
-        }
-        catch (err) {
-            // Next poll retries; the warn is throttled so a Redis outage logs once per interval, not once per poll.
-            const nowMs = Date.now();
-            if (lastWarnAt === undefined || nowMs - lastWarnAt >= QUEUE_DEPTH_WARN_INTERVAL_MS) {
-                lastWarnAt = nowMs;
-                createFallbackLogger("job-runner").warn("queue depth poll failed", {
-                    lane,
-                    error: scrubLogText(err instanceof Error ? err.message : String(err)),
-                });
-            }
-        }
-    };
-    await pollQueueDepth();
-    return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
-}
-function parseRedisOpts(url) {
-    const parsed = new URL(url);
-    const result = {
-        host: parsed.hostname,
-        port: Number(parsed.port) || 6379,
-    };
-    if (parsed.pathname.length > 1) {
-        result.db = Number(parsed.pathname.slice(1));
-    }
-    return result;
-}
-// redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
-// would otherwise hang start() forever with no health endpoint to notice.
-const BOOT_REDIS_TIMEOUT_MS = 10_000;
-// Shutdown must fit a k8s grace period even when Redis is down, so the
-// readiness wait in stop() is capped well below the boot timeout.
-const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
-// A gate without an explicit `timeout` may legitimately run long (migration-style
-// checks), so it is never rejected; a hang just must not stay silent.
-const BOOT_GATE_SLOW_WARN_MS = 60_000;
-function timeoutReject(ms, message) {
-    let timer;
-    const promise = new Promise((_, reject) => {
-        timer = setTimeout(() => reject(new Error(message)), ms);
-    });
-    return {
-        promise,
-        cancel: () => {
-            if (timer !== undefined)
-                clearTimeout(timer);
-        },
-    };
-}
-// Default base delay when a job opts into backoff without an explicit
-// delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
-// NaN/undefined, which is falsy — the job retries immediately instead of
-// waiting.
-const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
-// Shared by every enqueue path (dispatch(), handleEvent(), cron, runOnBoot,
-// perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
-// set must retry the same way regardless of how it got enqueued, or it
-// fails for good on the very first error on whichever path skips this.
-function buildRetryBullOpts(jobDef, priorAttempts = 0) {
-    const opts = {};
-    if (jobDef.retries !== undefined)
-        opts.attempts = Math.max(1, jobDef.retries + 1 - priorAttempts);
-    if (jobDef.backoff) {
-        opts.backoff =
-            typeof jobDef.backoff === "string"
-                ? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
-                : {
-                    type: jobDef.backoff.type,
-                    delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
-                };
-    }
-    return opts;
-}
-// Equality filter on top-level payload fields for a job's trigger.where —
-// lets N jobs share one broad event QN, partitioned by a payload
-// discriminant instead of needing N distinct event types.
-function payloadMatchesTriggerFilter(where, payload) {
-    if (!where)
-        return true;
-    return Object.entries(where).every(([key, value]) => payload[key] === value);
-}
-// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
-// OTHER jobs in the same completed/failed zset (moveToFinished lua,
-// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
-// the sweep only runs lazily on a later finish into that set. Per-job
-// retention is therefore meaningless on a shared queue — only queue-wide,
-// age-only retention (no count) is safe: a count would evict perTenant
-// children still inside their wrapper's retry window, and boot jobs. The
-// completed age must stay above every perTenant wrapper's own worst-case
-// retry window (see maxRetryWindowMs and the invariant check in
-// createJobRunner below), because child dedup (perTenantChildJobId) relies
-// on the children still existing in Redis when a wrapper retry re-derives
-// their ids. The margin here is generous — queue wait time isn't bounded by
-// backoff — to cover realistic windows. Audit trail lives in read_job_runs,
-// not BullMQ (fw#3199).
-const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
-const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
-// Total wall-clock time BullMQ can hold a perTenant wrapper across all its
-// retries, consistent with buildRetryBullOpts's own backoff computation.
-function maxRetryWindowMs(jobDef) {
-    if (!jobDef.backoff || jobDef.retries === undefined)
-        return 0;
-    const delayMs = (typeof jobDef.backoff === "string" ? undefined : jobDef.backoff.delayMs) ??
-        DEFAULT_JOB_BACKOFF_DELAY_MS;
-    const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
-    return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
-}
+export { serializeJobSubject } from "./job-runner-helpers.js";
+export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.js";
 export function createJobRunner(options) {
     const { registry, context, redisUrl, consumerLane } = options;
     const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;
@@ -988,7 +612,7 @@ export function createJobRunner(options) {
                 },
             })
             : undefined;
-        const crossTenantReadsReason = escapeHatchFor(jobDef.escapeHatch, "unsafeRaw")?.reason;
+        const crossTenantReadsReason = crossTenantReadsGrantFor(jobDef.escapeHatch)?.reason;
         const crossTenantReads = tenantScopedDb && crossTenantReadsReason !== undefined
             ? createCrossTenantReads(tenantScopedDb, crossTenantReadsReason, reportEscapeHatch)
             : undefined;

package/dist/pipeline/dispatch-shared.js

@@ -1,5 +1,6 @@
-import { escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
-import { requestContext, runWithOrigin } from "../api/request-context.js";
+import { crossTenantReadsGrantFor, escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
+import { isAgentToolCallScope, requestContext, runWithOrigin } from "../api/request-context.js";
+import { createHandlerCrossTenantReads } from "../db/cross-tenant-reads.js";
 import { runInSavepoint, selectMany } from "../db/query.js";
 import { createTenantDb, createUncheckedSystemDb, hasTenantColumn, } from "../db/tenant-db.js";
 import { createDerivativesContext } from "../derivatives/derivatives-context.js";
@@ -186,6 +187,7 @@ export function applyMemberResolutionReadOnly(handlerContext) {
         // A read as a member gets no system-scope DB, preSave pipeline or file storage.
         runPreSave: undefined,
         systemDb: undefined,
+        crossTenantReads: undefined,
         files: undefined,
         derivatives: undefined,
         ...(handlerContext.jobRunner && { jobRunner: denyingJobRunnerProxy() }),
@@ -326,6 +328,28 @@ readOnlyForTenant = false) {
     const systemDb = isSystem && db
         ? createUncheckedSystemDb(db, rawDbOutsideTransaction, reportEscapeHatch)
         : undefined;
+    // An agent tool call runs with the user's session but feeds the LLM context, so
+    // cross-tenant rows must never reach it, even when unsafeRaw implies the grant.
+    const crossTenantReadsReason = isAgentToolCallScope()
+        ? undefined
+        : crossTenantReadsGrantFor(handlerEscapeHatch)?.reason;
+    const crossTenantReads = dbSource && crossTenantReadsReason !== undefined
+        ? createHandlerCrossTenantReads({
+            db: dbSource,
+            tenantId: user.tenantId,
+            tracer: context.tracer,
+            meter: context.meter,
+            signal: reqCtx?.signal,
+            reason: crossTenantReadsReason,
+            report: reportEscapeHatch,
+            grants: {
+                report: reportEscapeHatch,
+                memberReadOnly: isMemberResolutionPrincipal(user),
+                personalDataGate: buildPersonalDataGate(registry, origin),
+                projectionRegistry: registry,
+            },
+        })
+        : undefined;
     // Exposed as ctx.db below — the internal `db` above stays the real,
     // working TenantDb for this function's own use (config/derivatives
     // accessors, systemDb construction).
@@ -576,8 +600,8 @@ readOnlyForTenant = false) {
             await appendDomainEvent(ctx, args, user, tx, registry.getHandlerFeature(type));
         },
         // Savepoint-scoped append: catches a losing writer's VersionConflictError
-        // without poisoning the rest of the handler's transaction. Bun.SQL/
-        // postgres.js abort the WHOLE begin() on an uncaught statement error
+        // without poisoning the rest of the handler's transaction. postgres.js
+        // aborts the WHOLE begin() on an uncaught statement error
         // (SQLSTATE 25P02) even if the JS error is caught — runInSavepoint
         // wraps the append in a real SAVEPOINT so only that nested scope rolls
         // back on conflict, and subsequent statements in the outer tx (e.g. a
@@ -855,6 +879,7 @@ readOnlyForTenant = false) {
         db: exposedDb,
         dbOutsideTransaction,
         ...(systemDb && { systemDb }),
+        ...(crossTenantReads && { crossTenantReads }),
         log,
         notify,
         ...(config && { config }),

package/dist/pipeline/event-dispatcher.js

@@ -21,7 +21,6 @@ const CONNECTION_LOSS_CODES = new Set([
     "CONNECT_TIMEOUT",
     "ECONNREFUSED",
     "ECONNRESET",
-    "ERR_POSTGRES_CONNECTION_CLOSED",
     "57P01",
 ]);
 function isConnectionLossError(e) {
@@ -506,7 +505,7 @@ export function createEventDispatcher(options) {
                         // Fires on initial connect and again when the driver's own
                         // re-LISTEN lands after a drop; during an outage that re-LISTEN
                         // waits in its connection backoff (postgres-js: measured 10-30s
-                        // after the DB returns, Bun.SQL: up to 32s), polling covers
+                        // after the DB returns), polling covers
                         // delivery meanwhile. Never unlisten()
                         // + listen() by hand: that leaves duplicate listeners.
                         // ponytail: no own LISTEN reconnect, the backoff gap is polled;
@@ -534,7 +533,7 @@ export function createEventDispatcher(options) {
                 timer = null;
             }
             if (pgSubscription) {
-                // Called on the subscription: Bun.SQL's unlisten is a prototype method.
+                // Called on the subscription, not detached: unlisten may rely on `this`.
                 await pgSubscription.unlisten().catch(() => {
                     // skip: unlisten failure only matters during shutdown — the
                     // subscription is being torn down anyway.