@cosmicdrift/kumiko-framework 0.386.0 → 0.388.0

package/dist/jobs/job-runner-gates.js

@@ -0,0 +1,198 @@
+/// <reference types="temporal-polyfill/global" preserve="true" />
+import { prepareJobRun } from "./job-runner-context.js";
+import { attemptNumberOf, buildRetryBullOpts, DEFER_COUNT_KEY, deferDelayMs, isFinalAttempt, priorAttemptsOf, priorDeferCountOf, } from "./job-runner-helpers.js";
+import { ACTIVE_TENANT_IDS_QUERY_NAME, perTenantChildJobId } from "./job-runner-schedulers.js";
+import { announceDropped, announceQueued, isOverPerTenantLimit, laneForJob, logFailedAttempt, } from "./job-runner-scope.js";
+import { resolveMaxPerTenant } from "./max-per-tenant.js";
+// Default lock-TTL for sequential jobs that didn't declare a timeout.
+// 5 minutes matches BullMQ's default stalledInterval — long enough for
+// any reasonable handler, short enough that a crashed worker recovers
+// without manual intervention.
+const SEQUENTIAL_DEFAULT_TTL_SEC = 305;
+// How long to wait before re-trying a busy sequential lock. Short enough
+// to feel responsive, long enough that we don't hammer Redis.
+const SEQUENTIAL_RETRY_DELAY_MS = 200;
+export const PER_TENANT_PREFIX = "_perTenant:";
+export function perTenantTargetOf(jobName) {
+    return jobName.startsWith(PER_TENANT_PREFIX)
+        ? jobName.slice(PER_TENANT_PREFIX.length)
+        : undefined;
+}
+function missingTenantResolverError(scope, actualName) {
+    if (scope.registry.getQueryHandler(ACTIVE_TENANT_IDS_QUERY_NAME)) {
+        return new Error(`perTenant job "${actualName}": the tenant feature is mounted but the job runner has no ` +
+            "`context.db`: pass a DbConnection in the runner context or supply options.getActiveTenantIds");
+    }
+    return new Error(`perTenant job "${actualName}" requires either options.getActiveTenantIds or the ` +
+        `tenant feature mounted (it registers "${ACTIVE_TENANT_IDS_QUERY_NAME}", which the ` +
+        "framework uses to resolve active tenants on its own)");
+}
+async function enqueuePerTenantChildren(scope, bullJob, actualName) {
+    const { getActiveTenantIds } = scope;
+    if (!getActiveTenantIds) {
+        throw missingTenantResolverError(scope, actualName);
+    }
+    const actualDef = scope.allJobs.get(actualName);
+    if (!actualDef) {
+        throw new Error(`Unknown job: ${actualName}`);
+    }
+    const tenantIds = await getActiveTenantIds();
+    const targetQueue = scope.queues[laneForJob(actualDef)];
+    // wrapperJobId is only absent for a Job BullMQ somehow never assigned
+    // an id to (not observed in practice) — falling back to a constant
+    // there would collide every such run's children onto the same ids
+    // and silently swallow later runs as "duplicates", which is worse
+    // than the duplicate this is meant to prevent. Skip id-based dedup
+    // for that edge case instead: enqueue plainly, same as before.
+    const wrapperJobId = bullJob.id;
+    for (const tenantId of tenantIds) {
+        const childData = { ...bullJob.data, _tenantId: tenantId };
+        const child = await targetQueue.add(actualName, childData, {
+            ...buildRetryBullOpts(actualDef),
+            // Dedup over wrapper retries only holds as long as the children
+            // stay in Redis for the wrapper's whole retry window — see the
+            // COMPLETED_JOB_RETENTION_AGE_SEC invariant check in createJobRunner.
+            ...(wrapperJobId !== undefined ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) } : {}),
+        });
+        await announceQueued(scope, actualName, actualDef, child, childData);
+    }
+}
+// Handle perTenant dispatch jobs — fan out to one job per tenant. The
+// fan-out re-enqueues into the lane the actual job is assigned to;
+// the _perTenant wrapper itself always lives in the consumer-lane
+// (it's picked up by this runner's own worker).
+export async function fanOutPerTenantJob(scope, bullJob, actualName) {
+    // These throws happen before runInSpan, so its onJobFailed never sees them
+    // and a drained test stack would stay green over a fan-out that never ran.
+    try {
+        await enqueuePerTenantChildren(scope, bullJob, actualName);
+    }
+    catch (err) {
+        // A retry may still succeed, and a succeeding wrapper reports no completion
+        // that could clear an earlier failure, so only the final attempt is reported.
+        const retries = scope.allJobs.get(actualName)?.retries ?? 0;
+        const finalAttempt = bullJob.attemptsMade + 1 >= retries + 1;
+        logFailedAttempt(scope, {
+            job: bullJob.name,
+            jobId: bullJob.id ?? "unknown",
+            attempt: bullJob.attemptsMade + 1,
+            finalAttempt,
+            err,
+        });
+        if (finalAttempt) {
+            await reportPreRunFailure(scope, bullJob.name, bullJob, err);
+        }
+        throw err;
+    }
+}
+async function reportPreRunFailure(scope, jobName, bullJob, err) {
+    const message = err instanceof Error ? err.message : String(err);
+    await scope.options.onJobFailed?.(jobName, bullJob.id ?? "unknown", message, [
+        { level: "error", message, timestamp: Temporal.Now.instant() },
+    ]);
+}
+// prepareJobRun and onJobStart throw before the handler span exists, so the
+// span's own failure recording never sees them.
+export async function startJobRun(scope, jobDef, bullJob, token) {
+    try {
+        const run = prepareJobRun(scope, jobDef, bullJob, token);
+        await scope.options.onJobStart?.(run.jobName, run.jobId, run.meta);
+        return run;
+    }
+    catch (err) {
+        const finalAttempt = isFinalAttempt(bullJob, jobDef);
+        logFailedAttempt(scope, {
+            job: bullJob.name,
+            jobId: bullJob.id ?? "unknown",
+            attempt: attemptNumberOf(bullJob),
+            finalAttempt,
+            err,
+        });
+        if (finalAttempt) {
+            await reportPreRunFailure(scope, bullJob.name, bullJob, err);
+        }
+        throw err;
+    }
+}
+export async function resolveJobDefinitionOrFail(scope, bullJob) {
+    const jobName = bullJob.name;
+    const jobDef = scope.allJobs.get(jobName);
+    if (jobDef)
+        return jobDef;
+    const err = new Error(`Unknown job: ${jobName}`);
+    // Final regardless of attempts left: a retry cannot make the job exist.
+    logFailedAttempt(scope, {
+        job: jobName,
+        jobId: bullJob.id ?? "unknown",
+        attempt: bullJob.attemptsMade + 1,
+        finalAttempt: true,
+        err,
+    });
+    await reportPreRunFailure(scope, jobName, bullJob, err);
+    throw err;
+}
+// maxPerTenant defer: checked before anything is recorded for this run, so a
+// deferred pickup leaves no span, run-log entry or failure trace and does not
+// consume an attempt (moveToDelayed + DelayedError, not a retry).
+export async function deferOverTenantLimit(scope, jobDef, bullJob, token) {
+    const maxPerTenant = resolveMaxPerTenant(jobDef);
+    const deferTenantId = bullJob.data["_tenantId"];
+    if (maxPerTenant?.onLimit === "defer" &&
+        typeof deferTenantId === "string" &&
+        token !== undefined &&
+        bullJob.moveToDelayed &&
+        (await isOverPerTenantLimit(scope, bullJob.name, deferTenantId, maxPerTenant.limit, {
+            activeOnly: true,
+            self: { id: bullJob.id, processedOn: bullJob.processedOn },
+        }))) {
+        const deferCount = priorDeferCountOf(bullJob.data, DEFER_COUNT_KEY);
+        await bullJob.updateData?.({ ...bullJob.data, [DEFER_COUNT_KEY]: deferCount + 1 });
+        await bullJob.moveToDelayed(Date.now() + deferDelayMs(maxPerTenant.deferMs, deferCount), token);
+        return "deferred";
+    }
+    return "run";
+}
+async function requeueBehindSequentialLock(scope, jobDef, bullJob) {
+    // Re-enqueue onto the job's own lane-queue. In practice that's the
+    // same queue the worker just picked from (since only the consuming
+    // lane runs handleJob at all), but route explicitly — no implicit
+    // coupling to "whichever queue the caller happened to be on".
+    // Attempts already spent travel in _priorAttempts so a lock conflict
+    // on a retry cannot hand out a fresh budget and exceed `retries`.
+    const priorAttempts = priorAttemptsOf(bullJob.data) + bullJob.attemptsMade;
+    const requeuedData = { ...bullJob.data, _priorAttempts: priorAttempts };
+    const requeued = await scope.queues[laneForJob(jobDef)].add(bullJob.name, requeuedData, {
+        delay: SEQUENTIAL_RETRY_DELAY_MS,
+        ...buildRetryBullOpts(jobDef, priorAttempts),
+    });
+    if (bullJob.id !== undefined && bullJob.id !== requeued.id) {
+        await announceDropped(scope, bullJob.name, jobDef, bullJob.id, bullJob.data);
+    }
+    await announceQueued(scope, bullJob.name, jobDef, requeued, requeuedData);
+}
+// Sequential gate: try to claim the per-name lock. If another worker
+// (or this worker on a different bullJob) holds it, re-enqueue with a
+// small delay and exit *successfully* — using throw would burn the
+// job's retry budget and pollute failure metrics, but a re-enqueue
+// looks like an ordinary handoff to BullMQ.
+export async function claimSequentialSlot(scope, jobDef, bullJob) {
+    const { sequentialLock } = scope;
+    if (jobDef.concurrency !== "sequential" || !sequentialLock) {
+        return { kind: "run", lockToken: null };
+    }
+    const ttlSec = jobDef.timeout ? Math.ceil(jobDef.timeout / 1000) + 5 : SEQUENTIAL_DEFAULT_TTL_SEC;
+    const lockToken = await sequentialLock.acquire(bullJob.name, { ttlSeconds: ttlSec });
+    if (lockToken)
+        return { kind: "run", lockToken };
+    await requeueBehindSequentialLock(scope, jobDef, bullJob);
+    return { kind: "requeued" };
+}
+// Release the sequential lock value-matched (Lua compare-and-delete
+// inside DistributedLock). A TTL-expired lock that's been claimed by
+// a different owner stays put — releasing it would break sequencing
+// for the new owner.
+export async function releaseSequentialSlot(scope, jobName, lockToken) {
+    if (lockToken && scope.sequentialLock) {
+        await scope.sequentialLock.release(jobName, lockToken);
+    }
+}

package/dist/jobs/job-runner-helpers.d.ts

@@ -3,18 +3,20 @@ import type { ConfigAccessor } from "@cosmicdrift/kumiko-types/config";
 import type { TzContext } from "@cosmicdrift/kumiko-types/tz-context";
 import type { JobsOptions, Queue } from "bullmq";
 import type { TenantDb } from "../db/tenant-db.ts";
-import type { AppContext, JobDefinition, JobRunIn, JobTriggerEvent } from "../engine/types/index.ts";
+import type { AppContext, JobDeferral, JobDefinition, JobRunIn, JobTriggerEvent } from "../engine/types/index.ts";
 import type { Logger } from "../logging/types.ts";
 import type { Meter, SerializedTraceContext, Tracer } from "../observability/index.ts";
-import type { JobLogEntry, JobSubjectValue } from "./job-runner-types.ts";
+import type { JobInvocation, JobLogEntry, JobSubjectValue } from "./job-runner-types.ts";
 export declare function isJobTriggerEvent(value: unknown): value is JobTriggerEvent;
 export declare function stampGatedWriteOrigin(data: Record<string, unknown>): void;
 export declare function createJobLogger(logs: JobLogEntry[], baseContext?: Record<string, unknown>): Logger;
 export declare function serializeJobSubject(subject: Readonly<Record<string, JobSubjectValue>>): string;
 export declare function jobSubjectKey(jobName: string, payload: Record<string, unknown>, fields: readonly string[] | undefined, declaration: "tenantVisibleFailure" | "tenantVisibleRun"): string | null;
-export declare function resolveJobTz(config: ConfigAccessor | undefined, context: Pick<AppContext, "geoTzProvider" | "clock">): Promise<TzContext>;
+export declare function resolveJobTz(config: ConfigAccessor | undefined, context: Pick<AppContext, "geoTzProvider" | "clock">, logger: Pick<Logger, "warn">): Promise<TzContext>;
 export declare function missingDbProxy(jobName: string): TenantDb;
 export declare function priorAttemptsOf(data: Record<string, unknown>): number;
+export declare function attemptNumberOf(bullJob: JobInvocation): number;
+export declare function isFinalAttempt(bullJob: JobInvocation, jobDef: JobDefinition): boolean;
 export declare function tenantFailureMessageKey(err: unknown, declaration: JobDefinition["tenantVisibleFailure"]): string | null;
 export declare const TRACE_CONTEXT_KEY = "_traceContext";
 export declare const DEFER_COUNT_KEY = "_deferCount";
@@ -42,3 +44,15 @@ export declare function payloadMatchesTriggerFilter(where: Readonly<Record<strin
 export declare const COMPLETED_JOB_RETENTION_AGE_SEC = 86400;
 export declare const FAILED_JOB_RETENTION_AGE_SEC = 604800;
 export declare function maxRetryWindowMs(jobDef: JobDefinition): number;
+export declare function isJobDeferral(value: unknown): value is JobDeferral;
+export declare function priorDeferCountOf(data: Record<string, unknown>, key: string): number;
+export declare function handlerPayloadOf(data: Record<string, unknown>): Record<string, unknown>;
+export declare function tenantIdOfJobData(data: Record<string, unknown>): string;
+export declare function deferDelayMs(deferMs: number, deferCount: number): number;
+export declare function isActiveBefore(other: {
+    readonly id?: string | undefined;
+    readonly processedOn?: number | undefined;
+}, self: {
+    readonly id: string | undefined;
+    readonly processedOn: number | undefined;
+}): boolean;

package/dist/jobs/job-runner-helpers.js

@@ -1,6 +1,7 @@
 /// <reference types="temporal-polyfill/global" preserve="true" />
 import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
 import { requestContext } from "../api/request-context.js";
+import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
 import { InternalError } from "../errors/index.js";
 import { isKumikoError } from "../errors/kumiko-error.js";
 import { createFallbackLogger, scrubLogText } from "../logging/index.js";
@@ -66,6 +67,12 @@ export function serializeJobSubject(subject) {
         .sort()
         .map((field) => [field, subject[field] ?? null]));
 }
+function isJobSubjectValue(value) {
+    return (value === null ||
+        typeof value === "string" ||
+        typeof value === "number" ||
+        typeof value === "boolean");
+}
 // Stable identity for the declared payload fields: sorted keys, so two runs
 // with the same subject values produce the same string and the later one
 // replaces the earlier record. Non-primitives would serialize into something
@@ -76,15 +83,29 @@ export function jobSubjectKey(jobName, payload, fields, declaration) {
     const subject = {};
     for (const field of fields) {
         const value = payload[field] ?? null;
-        if (value !== null && typeof value === "object") {
+        if (!isJobSubjectValue(value)) {
             throw new Error(`Job "${jobName}": ${declaration}.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
         }
         subject[field] = value;
     }
     return serializeJobSubject(subject);
 }
-export async function resolveJobTz(config, context) {
-    const tenantTz = config ? await config(TENANT_TIMEZONE_CONFIG_KEY) : undefined;
+// A job that never reads ctx.tz must not fail because the tenant config read did.
+async function readTenantTimezoneOrUndefined(config, logger) {
+    if (!config)
+        return undefined;
+    try {
+        return await config(TENANT_TIMEZONE_CONFIG_KEY);
+    }
+    catch (err) {
+        logger.warn("tenant timezone config read failed, falling back to UTC", {
+            error: scrubLogText(err instanceof Error ? err.message : String(err)),
+        });
+        return undefined;
+    }
+}
+export async function resolveJobTz(config, context, logger) {
+    const tenantTz = await readTenantTimezoneOrUndefined(config, logger);
     return createTzContext({
         ...(context.geoTzProvider !== undefined && { geoTz: context.geoTzProvider }),
         ...(context.clock !== undefined && { clock: context.clock }),
@@ -107,6 +128,14 @@ export function priorAttemptsOf(data) {
     const value = data["_priorAttempts"];
     return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
 }
+export function attemptNumberOf(bullJob) {
+    return bullJob.attemptsMade + 1 + priorAttemptsOf(bullJob.data);
+}
+// BullMQ stops retrying once attemptsMade reaches the configured attempts
+// (`retries + 1`), so this is the attempt whose failure is final.
+export function isFinalAttempt(bullJob, jobDef) {
+    return bullJob.retryable === false || attemptNumberOf(bullJob) >= (jobDef.retries ?? 0) + 1;
+}
 // Only a translation key ever travels to the tenant: the thrown error's own
 // i18nKey when it carries one, otherwise the key declared at the job. The
 // error message itself can echo provider or payload content and stays on the
@@ -250,7 +279,7 @@ export function payloadMatchesTriggerFilter(where, payload) {
 // on the children still existing in Redis when a wrapper retry re-derives
 // their ids. The margin here is generous — queue wait time isn't bounded by
 // backoff — to cover realistic windows. Audit trail lives in read_job_runs,
-// not BullMQ (fw#3199).
+// not BullMQ.
 export const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
 export const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
 // Total wall-clock time BullMQ can hold a perTenant wrapper across all its
@@ -263,3 +292,39 @@ export function maxRetryWindowMs(jobDef) {
     const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
     return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
 }
+export function isJobDeferral(value) {
+    return (typeof value === "object" && value !== null && "kind" in value && value.kind === "job-deferral");
+}
+export function priorDeferCountOf(data, key) {
+    const prior = data[key];
+    return typeof prior === "number" ? prior : 0;
+}
+export function handlerPayloadOf(data) {
+    const payload = {};
+    for (const [k, v] of Object.entries(data)) {
+        if (!k.startsWith("_"))
+            payload[k] = v;
+    }
+    return payload;
+}
+export function tenantIdOfJobData(data) {
+    return (data["_tenantId"] ?? // @cast-boundary dynamic-key
+        data["tenantId"] ?? // @cast-boundary dynamic-key
+        SYSTEM_TENANT_ID);
+}
+// Growing delay bounds Redis and worker load while one tenant has a large
+// backlog: every deferred job would otherwise be re-checked every deferMs.
+export function deferDelayMs(deferMs, deferCount) {
+    const grown = Math.min(deferMs * 2 ** Math.min(deferCount, DEFER_BACKOFF_MAX_DOUBLINGS), DEFER_DELAY_CAP_MS);
+    return grown + Math.floor(Math.random() * deferMs);
+}
+export function isActiveBefore(other, self) {
+    if (other.id === self.id)
+        return false;
+    if (self.processedOn === undefined)
+        return true;
+    const otherProcessedOn = other.processedOn ?? 0;
+    if (otherProcessedOn !== self.processedOn)
+        return otherProcessedOn < self.processedOn;
+    return (other.id ?? "") < (self.id ?? "");
+}

package/dist/jobs/job-runner-schedulers.d.ts

@@ -7,7 +7,7 @@ export declare const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-
  * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
  * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
  * treated as old repeatables — spawning a new scheduler entry every tick and
- * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
+ * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828,
  * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
  */
 export declare function schedulerIdForJobName(jobName: string): string;

package/dist/jobs/job-runner-schedulers.js

@@ -19,13 +19,13 @@ export const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
  * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
  * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
  * treated as old repeatables — spawning a new scheduler entry every tick and
- * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
+ * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828,
  * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
  */
 export function schedulerIdForJobName(jobName) {
     return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
 }
-// Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
+// Same colon-in-BullMQ-id hazard as schedulerIdForJobName —
 // a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
 export function bootJobIdForJobName(jobName) {
     return `boot-${jobName.replace(/[.:]/g, "-")}`;
@@ -97,8 +97,7 @@ export async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lan
 // duplicates — safe only as long as the child job hash outlives the
 // wrapper's retry window, enforced at construction against
 // COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
-// createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
-// (fw#1603/#1604) — the wrapper id already contains ":", so strip
+// createJobRunner). The wrapper id already contains ":", so strip
 // separators from both halves before joining instead of interpolating raw.
 export function perTenantChildJobId(wrapperJobId, tenantId) {
     const sanitize = (value) => value.replace(/[.:]/g, "-");

package/dist/jobs/job-runner-scope.d.ts

@@ -0,0 +1,44 @@
+import type { Queue } from "bullmq";
+import type { AppContext, DispatchWriteRef, JobDefinition, JobRunIn, Registry, TenantId } from "../engine/types/index.ts";
+import type { createFallbackLogger } from "../logging/index.ts";
+import type { EscapeHatchProcessDedup, EscapeHatchReportWindow, Tracer } from "../observability/index.ts";
+import type { DistributedLock } from "../pipeline/distributed-lock.ts";
+import type { JobRunner, JobRunnerOptions } from "./job-runner-types.ts";
+export type JobRunnerScope = {
+    readonly registry: Registry;
+    readonly context: AppContext;
+    readonly options: JobRunnerOptions;
+    readonly allJobs: ReadonlyMap<string, JobDefinition>;
+    readonly queues: Readonly<Record<JobRunIn, Queue>>;
+    readonly consumerLane: JobRunIn | undefined;
+    readonly tracer: Tracer;
+    readonly errorLogger: ReturnType<typeof createFallbackLogger>;
+    readonly sequentialLock: DistributedLock | null;
+    readonly getActiveTenantIds: (() => Promise<TenantId[]>) | undefined;
+    readonly escapeHatchReportWindow: EscapeHatchReportWindow;
+    readonly systemCronEscapeHatchDedup: EscapeHatchProcessDedup;
+    readonly getSelfRunner: () => JobRunner | undefined;
+    readonly getDispatchWriteRef: () => DispatchWriteRef | undefined;
+};
+export declare function laneForJob(def: {
+    readonly runIn?: JobRunIn | undefined;
+}): JobRunIn;
+export declare function isOverPerTenantLimit(scope: JobRunnerScope, jobName: string, tenantId: string, max: number, countFilter?: {
+    readonly activeOnly?: boolean;
+    readonly self?: {
+        readonly id: string | undefined;
+        readonly processedOn: number | undefined;
+    };
+}): Promise<boolean>;
+export declare function announceQueued(scope: JobRunnerScope, jobName: string, jobDef: JobDefinition, queuedJob: {
+    readonly id?: string | undefined;
+    readonly timestamp: number;
+}, data: Record<string, unknown>): Promise<void>;
+export declare function announceDropped(scope: JobRunnerScope, jobName: string, jobDef: JobDefinition, bullJobId: string, data: Record<string, unknown>): Promise<void>;
+export declare function logFailedAttempt(scope: JobRunnerScope, entry: {
+    readonly job: string;
+    readonly jobId: string;
+    readonly attempt: number;
+    readonly finalAttempt: boolean;
+    readonly err: unknown;
+}): void;

package/dist/jobs/job-runner-scope.js

@@ -0,0 +1,102 @@
+import { UnrecoverableError } from "bullmq";
+import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
+import { scrubLogText } from "../logging/index.js";
+import { handlerPayloadOf, isActiveBefore, jobSubjectKey, tenantIdOfJobData, } from "./job-runner-helpers.js";
+// Resolve the lane for a job — "worker" is the default because that's the
+// sensible prod lane (heavy async off the request path). Jobs that opted
+// into "api" must have been validated at registry boot already.
+export function laneForJob(def) {
+    return def.runIn ?? "worker";
+}
+// Counts active + waiting jobs with this name for this tenant across
+// BOTH lane queues. Jobs with the same name should only live in one
+// lane (jobDef.runIn is static), but walking both is cheap and avoids
+// a subtle bug if someone ever reassigns a job to a different lane
+// between deploys while old queue contents are still draining.
+// `activeOnly` + `self` serve the defer check, which runs inside the job being
+// counted: it must not see waiting jobs or itself, and only counts jobs that
+// became active before it. Without that order, workers that fetch a burst of
+// jobs together all see each other as active and all defer, again and again.
+export async function isOverPerTenantLimit(scope, jobName, tenantId, max, countFilter = {}) {
+    const { queues } = scope;
+    const results = await Promise.all([
+        queues.api.getActive(),
+        queues.worker.getActive(),
+        ...(countFilter.activeOnly ? [] : [queues.api.getWaiting(), queues.worker.getWaiting()]),
+    ]);
+    let count = 0;
+    for (const list of results) {
+        for (const j of list) {
+            if (j.name !== jobName)
+                continue;
+            if (countFilter.self && !isActiveBefore(j, countFilter.self))
+                continue;
+            const t = j.data?._tenantId; // @cast-boundary dynamic-key
+            if (t === tenantId) {
+                count += 1;
+                if (count >= max)
+                    return true;
+            }
+        }
+    }
+    return false;
+}
+// Tenant-visible queued state. A callback or subject problem must never fail
+// the enqueue; a bad subject config surfaces as a failed run instead.
+export async function announceQueued(scope, jobName, jobDef, queuedJob, data) {
+    const { options } = scope;
+    const bullJobId = queuedJob.id;
+    const declaration = jobDef.tenantVisibleRun;
+    // skip: job did not opt in, no logger wired, or BullMQ returned no id
+    if (!declaration || !options.onJobQueued || bullJobId === undefined)
+        return;
+    const tenantId = tenantIdOfJobData(data);
+    // skip: tenant-less run — no tenant-scoped query could ever read it
+    if (tenantId === SYSTEM_TENANT_ID)
+        return;
+    try {
+        const subject = jobSubjectKey(jobName, handlerPayloadOf(data), declaration.subjectFields, "tenantVisibleRun");
+        await options.onJobQueued(jobName, bullJobId, {
+            tenantId,
+            subject,
+            jobCreatedAtMs: queuedJob.timestamp,
+        });
+    }
+    catch (err) {
+        scope.errorLogger.warn("tenant-visible queued state not recorded", {
+            job: jobName,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+}
+export async function announceDropped(scope, jobName, jobDef, bullJobId, data) {
+    const { options } = scope;
+    // skip: job did not opt in or no logger wired
+    if (!jobDef.tenantVisibleRun || !options.onJobDropped)
+        return;
+    const tenantId = tenantIdOfJobData(data);
+    // skip: tenant-less run — announceQueued never recorded a row for it
+    if (tenantId === SYSTEM_TENANT_ID)
+        return;
+    try {
+        await options.onJobDropped(jobName, bullJobId, { tenantId });
+    }
+    catch (err) {
+        scope.errorLogger.warn("tenant-visible queued state not cleared", {
+            job: jobName,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+}
+// Only err.message, never payload or the logs array: job payloads can carry PII.
+export function logFailedAttempt(scope, entry) {
+    scope.errorLogger.error("job failed", {
+        job: entry.job,
+        jobId: entry.jobId,
+        attempt: entry.attempt,
+        // BullMQ never retries an UnrecoverableError, whatever attempts remain.
+        final: entry.finalAttempt || entry.err instanceof UnrecoverableError,
+        error: scrubLogText(entry.err instanceof Error ? entry.err.message : String(entry.err)),
+        ...(entry.err instanceof Error && entry.err.stack && { stack: scrubLogText(entry.err.stack) }),
+    });
+}

package/dist/jobs/job-runner-types.d.ts

@@ -19,6 +19,7 @@ export type JobMeta = {
 export type JobQueuedMeta = {
     readonly tenantId: string;
     readonly subject: string | null;
+    readonly jobCreatedAtMs: number;
 };
 export type JobDroppedMeta = {
     readonly tenantId: string;
@@ -63,3 +64,14 @@ export type JobRunnerOptions = {
     onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
     onJobDropped?: (jobName: string, jobId: string, meta: JobDroppedMeta) => void;
 };
+export type JobInvocation = {
+    readonly id?: string | undefined;
+    readonly name: string;
+    readonly data: Record<string, unknown>;
+    readonly attemptsMade: number;
+    /** False for inline invocations (boot gates): nothing retries a throw. */
+    readonly retryable?: boolean;
+    readonly processedOn?: number | undefined;
+    moveToDelayed?(timestamp: number, token?: string): Promise<void>;
+    updateData?(data: Record<string, unknown>): Promise<void>;
+};

package/dist/jobs/job-runner.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="temporal-polyfill/global" preserve="true" />
 import type { JobRunner, JobRunnerOptions } from "./job-runner-types.ts";
 export { serializeJobSubject } from "./job-runner-helpers.ts";
 export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.ts";