@cosmicdrift/kumiko-framework 0.384.0 → 0.386.0

package/dist/engine/feature-ast/pattern-change-schema.d.ts

@@ -233,6 +233,7 @@ export declare const PATTERN_SCHEMAS_BY_KIND: {
         escapeHatch: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
             reason: z.ZodString;
             grants: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                crossTenantReads: "crossTenantReads";
                 globalWrites: "globalWrites";
                 systemIdentity: "systemIdentity";
                 unsafeRaw: "unsafeRaw";
@@ -314,6 +315,7 @@ export declare const PATTERN_SCHEMAS_BY_KIND: {
         escapeHatch: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
             reason: z.ZodString;
             grants: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                crossTenantReads: "crossTenantReads";
                 globalWrites: "globalWrites";
                 systemIdentity: "systemIdentity";
                 unsafeRaw: "unsafeRaw";
@@ -384,6 +386,7 @@ export declare const PATTERN_SCHEMAS_BY_KIND: {
         escapeHatch: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
             reason: z.ZodString;
             grants: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                crossTenantReads: "crossTenantReads";
                 globalWrites: "globalWrites";
                 systemIdentity: "systemIdentity";
                 unsafeRaw: "unsafeRaw";
@@ -420,6 +423,7 @@ export declare const PATTERN_SCHEMAS_BY_KIND: {
         escapeHatch: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
             reason: z.ZodString;
             grants: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                crossTenantReads: "crossTenantReads";
                 globalWrites: "globalWrites";
                 systemIdentity: "systemIdentity";
                 unsafeRaw: "unsafeRaw";
@@ -828,6 +832,7 @@ export declare const patternSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     escapeHatch: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
         reason: z.ZodString;
         grants: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            crossTenantReads: "crossTenantReads";
             globalWrites: "globalWrites";
             systemIdentity: "systemIdentity";
             unsafeRaw: "unsafeRaw";
@@ -908,6 +913,7 @@ export declare const patternSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     escapeHatch: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
         reason: z.ZodString;
         grants: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            crossTenantReads: "crossTenantReads";
             globalWrites: "globalWrites";
             systemIdentity: "systemIdentity";
             unsafeRaw: "unsafeRaw";
@@ -977,6 +983,7 @@ export declare const patternSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     escapeHatch: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
         reason: z.ZodString;
         grants: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            crossTenantReads: "crossTenantReads";
             globalWrites: "globalWrites";
             systemIdentity: "systemIdentity";
             unsafeRaw: "unsafeRaw";
@@ -1012,6 +1019,7 @@ export declare const patternSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     escapeHatch: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
         reason: z.ZodString;
         grants: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            crossTenantReads: "crossTenantReads";
             globalWrites: "globalWrites";
             systemIdentity: "systemIdentity";
             unsafeRaw: "unsafeRaw";

package/dist/engine/steps/read-find-one.js

@@ -1,6 +1,6 @@
 // r.step.read.findOne — load a single row from a projection table.
 //
-// Thin wrapper on selectMany(db, table, where, { limit: 1 }) (bun-db).
+// Thin wrapper on selectMany(db, table, where, { limit: 1 }) (db/query.ts).
 // Resolves to the first row or null. Tenant-filtered like ctx.db
 // method-form reads (own tenant + SYSTEM_TENANT_ID reference rows); a
 // foreign `where.tenantId` is narrowed to the caller's own scope.

package/dist/event-store/provenance-append.js

@@ -32,7 +32,7 @@ export async function appendProvenanceEvent(db, event) {
             message: `appendProvenanceEvent: event tenant "${event.tenantId}" does not match the TenantDb's tenant "${db.tenantId}".`,
         });
     }
-    // Bun.SQL/postgres.js abort the whole surrounding begin() on a statement
+    // postgres-js aborts the whole surrounding begin() on a statement
     // error (25P02) even when the caller's catch swallows it — the savepoint
     // confines that to this scope. Pool connections have no ambient tx to poison.
     await runInSavepointIfSupported(runner, async (sp) => {

package/dist/files/file-routes.js

@@ -447,7 +447,7 @@ export function createFileRoutes(options) {
     });
     async function loadFileForTenant(id, tenantId) {
         // fileRefsTable.id is a UUID column — a malformed id must fail here, not
-        // at Postgres (22P02 poisons the pooled Bun.SQL connection). 404, not
+        // at Postgres (22P02 poisons the pooled connection). 404, not
         // 400: an authenticated caller sending a bad id must see the same
         // response as a nonexistent one, no existence-oracle.
         if (!isUuid(id))

package/dist/i18n/required-surface-keys.d.ts

@@ -37,3 +37,19 @@ export type TranslationLocaleGap = {
  * consistent by definition (locale packages, en-only framework features).
  */
 export declare function findTranslationLocaleGaps(features: readonly FeatureDefinition[]): readonly TranslationLocaleGap[];
+export type RequiredLocaleGap = {
+    readonly featureName: string;
+    readonly key: string;
+    readonly missingLocales: readonly string[];
+};
+/**
+ * Per translation key, the union of locales over ALL features must cover
+ * `requiredLocales`. Locale packages translate other features' keys under the
+ * full `feature:key` name, so the lookup unions by effective name. Raw names
+ * only come from locale packages and already-prefixed keys. Entries that only translate another
+ * mounted feature's key are not reported separately; the defining feature is.
+ * Locale packages (`locale-*`) only contribute locales: their keys often belong to
+ * client bundles (e.g. `auth.errors.*`) that no feature defines, so they are never
+ * reported as gap owners.
+ */
+export declare function findRequiredLocaleGaps(features: readonly FeatureDefinition[], requiredLocales: readonly string[]): readonly RequiredLocaleGap[];

package/dist/i18n/required-surface-keys.js

@@ -549,3 +549,65 @@ export function findTranslationLocaleGaps(features) {
     }
     return gaps;
 }
+const LOCALE_PACKAGE_PREFIX = "locale-";
+function collectLocalesByName(features) {
+    // A plain key of a regular feature is only its own; registering it raw would let two
+    // features with the same plain key cover each other's missing locales.
+    const namesOf = (featureName, key) => featureName.startsWith(LOCALE_PACKAGE_PREFIX) || key.includes(":")
+        ? [`${featureName}:${key}`, key]
+        : [`${featureName}:${key}`];
+    const localesByName = new Map();
+    for (const feature of features) {
+        for (const [key, byLocale] of Object.entries(feature.translations ?? {})) {
+            const present = Object.entries(byLocale)
+                .filter(([, value]) => hasLocaleValue(value))
+                .map(([locale]) => locale);
+            for (const name of namesOf(feature.name, key)) {
+                const locales = localesByName.get(name) ?? new Set();
+                for (const locale of present)
+                    locales.add(locale);
+                localesByName.set(name, locales);
+            }
+        }
+    }
+    return localesByName;
+}
+/**
+ * Per translation key, the union of locales over ALL features must cover
+ * `requiredLocales`. Locale packages translate other features' keys under the
+ * full `feature:key` name, so the lookup unions by effective name. Raw names
+ * only come from locale packages and already-prefixed keys. Entries that only translate another
+ * mounted feature's key are not reported separately; the defining feature is.
+ * Locale packages (`locale-*`) only contribute locales: their keys often belong to
+ * client bundles (e.g. `auth.errors.*`) that no feature defines, so they are never
+ * reported as gap owners.
+ */
+export function findRequiredLocaleGaps(features, requiredLocales) {
+    const featureNames = new Set(features.map((feature) => feature.name));
+    const localesByName = collectLocalesByName(features);
+    const gaps = [];
+    const reported = new Set();
+    for (const feature of features) {
+        if (feature.name.startsWith(LOCALE_PACKAGE_PREFIX))
+            continue;
+        for (const key of Object.keys(feature.translations ?? {})) {
+            const prefix = key.includes(":") ? key.slice(0, key.indexOf(":")) : undefined;
+            const translatesOtherFeature = prefix !== undefined && prefix !== feature.name && featureNames.has(prefix);
+            if (translatesOtherFeature)
+                continue;
+            const effective = key.includes(":") ? key : `${feature.name}:${key}`;
+            if (reported.has(effective))
+                continue;
+            // Locale packages spell plurals as flat `.one`/`.other` keys; `other` is the form every locale has.
+            const present = new Set([effective, key, `${effective}.other`, `${key}.other`].flatMap((name) => [
+                ...(localesByName.get(name) ?? []),
+            ]));
+            const missingLocales = requiredLocales.filter((locale) => !present.has(locale));
+            if (missingLocales.length > 0) {
+                reported.add(effective);
+                gaps.push({ featureName: feature.name, key, missingLocales });
+            }
+        }
+    }
+    return gaps;
+}

package/dist/internal/stack-support.d.ts

@@ -1,5 +1,5 @@
 export { createDbConnection } from "../db/connection.ts";
-export { createDatabase, databaseExists, dropDatabaseIfExists, truncateTablesRestartIdentity, } from "../db/queries/test-stack.ts";
+export { acquireSessionAdvisoryLock, createDatabase, databaseExists, dropDatabaseIfExists, listDatabaseNamesWithPrefix, renameDatabase, truncateTablesRestartIdentity, } from "../db/queries/test-stack.ts";
 export { type AnyDb, isClosedConnectionError } from "../db/query.ts";
 export { columnNamesOf } from "../db/schema-inspection.ts";
 export { schemaTableName } from "../db/table-introspection.ts";

package/dist/internal/stack-support.js

@@ -1,6 +1,6 @@
 // @runtime dev
 export { createDbConnection } from "../db/connection.js";
-export { createDatabase, databaseExists, dropDatabaseIfExists, truncateTablesRestartIdentity, } from "../db/queries/test-stack.js";
+export { acquireSessionAdvisoryLock, createDatabase, databaseExists, dropDatabaseIfExists, listDatabaseNamesWithPrefix, renameDatabase, truncateTablesRestartIdentity, } from "../db/queries/test-stack.js";
 export { isClosedConnectionError } from "../db/query.js";
 export { columnNamesOf } from "../db/schema-inspection.js";
 export { schemaTableName } from "../db/table-introspection.js";

package/dist/jobs/job-runner-helpers.d.ts

@@ -0,0 +1,44 @@
+/// <reference types="temporal-polyfill/global" preserve="true" />
+import type { ConfigAccessor } from "@cosmicdrift/kumiko-types/config";
+import type { TzContext } from "@cosmicdrift/kumiko-types/tz-context";
+import type { JobsOptions, Queue } from "bullmq";
+import type { TenantDb } from "../db/tenant-db.ts";
+import type { AppContext, JobDefinition, JobRunIn, JobTriggerEvent } from "../engine/types/index.ts";
+import type { Logger } from "../logging/types.ts";
+import type { Meter, SerializedTraceContext, Tracer } from "../observability/index.ts";
+import type { JobLogEntry, JobSubjectValue } from "./job-runner-types.ts";
+export declare function isJobTriggerEvent(value: unknown): value is JobTriggerEvent;
+export declare function stampGatedWriteOrigin(data: Record<string, unknown>): void;
+export declare function createJobLogger(logs: JobLogEntry[], baseContext?: Record<string, unknown>): Logger;
+export declare function serializeJobSubject(subject: Readonly<Record<string, JobSubjectValue>>): string;
+export declare function jobSubjectKey(jobName: string, payload: Record<string, unknown>, fields: readonly string[] | undefined, declaration: "tenantVisibleFailure" | "tenantVisibleRun"): string | null;
+export declare function resolveJobTz(config: ConfigAccessor | undefined, context: Pick<AppContext, "geoTzProvider" | "clock">): Promise<TzContext>;
+export declare function missingDbProxy(jobName: string): TenantDb;
+export declare function priorAttemptsOf(data: Record<string, unknown>): number;
+export declare function tenantFailureMessageKey(err: unknown, declaration: JobDefinition["tenantVisibleFailure"]): string | null;
+export declare const TRACE_CONTEXT_KEY = "_traceContext";
+export declare const DEFER_COUNT_KEY = "_deferCount";
+export declare const HANDLER_DEFER_COUNT_KEY = "_handlerDeferCount";
+export declare const DEFAULT_MAX_DEFERS = 100;
+export declare const DEFER_DELAY_CAP_MS = 30000;
+export declare const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
+export declare function readTraceContext(data: Record<string, unknown>): SerializedTraceContext | undefined;
+export declare function captureTraceContext(tracer: Tracer): SerializedTraceContext | undefined;
+export declare function startQueueDepthPolling(queue: Queue, lane: JobRunIn, meter: Meter): Promise<ReturnType<typeof setInterval>>;
+export declare function parseRedisOpts(url: string): {
+    host: string;
+    port: number;
+    db?: number | undefined;
+};
+export declare const BOOT_REDIS_TIMEOUT_MS = 10000;
+export declare const STOP_QUEUE_READY_TIMEOUT_MS = 1000;
+export declare const BOOT_GATE_SLOW_WARN_MS = 60000;
+export declare function timeoutReject(ms: number, message: string): {
+    promise: Promise<never>;
+    cancel: () => void;
+};
+export declare function buildRetryBullOpts(jobDef: JobDefinition, priorAttempts?: number): Pick<JobsOptions, "attempts" | "backoff">;
+export declare function payloadMatchesTriggerFilter(where: Readonly<Record<string, string | number | boolean>> | undefined, payload: Record<string, unknown>): boolean;
+export declare const COMPLETED_JOB_RETENTION_AGE_SEC = 86400;
+export declare const FAILED_JOB_RETENTION_AGE_SEC = 604800;
+export declare function maxRetryWindowMs(jobDef: JobDefinition): number;

package/dist/jobs/job-runner-helpers.js

@@ -0,0 +1,265 @@
+/// <reference types="temporal-polyfill/global" preserve="true" />
+import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
+import { requestContext } from "../api/request-context.js";
+import { InternalError } from "../errors/index.js";
+import { isKumikoError } from "../errors/kumiko-error.js";
+import { createFallbackLogger, scrubLogText } from "../logging/index.js";
+import { emitJobQueueDepth } from "../observability/index.js";
+import { TENANT_TIMEZONE_CONFIG_KEY } from "../pipeline/tenant-timezone-config-key.js";
+import { isPersonalDataGated } from "../pipeline/write-origin.js";
+import { createTzContext, isValidIanaTimeZone } from "../time/index.js";
+// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
+// (a security-relevant escape hatch that must fail the run closed), a
+// malformed `_triggerEvent` only costs an idempotency-key convenience — the
+// run proceeds without it rather than failing over what would be a framework
+// bug, not untrusted input.
+export function isJobTriggerEvent(value) {
+    if (!isPlainObject(value))
+        return false;
+    if (typeof value["id"] !== "string")
+        return false;
+    const headers = value["headers"];
+    if (typeof headers !== "object" || headers === null)
+        return false;
+    return Object.values(headers).every((v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean");
+}
+// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
+export function stampGatedWriteOrigin(data) {
+    delete data["_writeOrigin"];
+    const origin = requestContext.get()?.writeOrigin;
+    if (origin && isPersonalDataGated(origin)) {
+        data["_writeOrigin"] = origin;
+    }
+}
+export function createJobLogger(logs, baseContext = {}) {
+    function push(level, msg, data) {
+        const merged = { ...baseContext, ...data };
+        const message = Object.keys(merged).length > 0 ? `${msg} ${JSON.stringify(merged)}` : msg;
+        logs.push({ level, message, timestamp: Temporal.Now.instant() });
+    }
+    const logger = {
+        info(msg, data) {
+            push("info", msg, data);
+        },
+        warn(msg, data) {
+            push("warn", msg, data);
+        },
+        error(msg, data) {
+            push("error", msg, data);
+        },
+        // No-op by design, not missing coverage: JobLogEntry.level (and the
+        // persisted run-completed/-failed event schema in bundled-features/
+        // jobs/events.ts) only has info|warn|error — a job's debug() calls have
+        // nowhere durable to land.
+        debug() { },
+        child(context) {
+            return createJobLogger(logs, { ...baseContext, ...context });
+        },
+    };
+    return logger;
+}
+// Single source of the stored subject string: the writer (jobSubjectKey) and
+// the tenant failures query both go through it. Matching is exact — field names
+// and JSON types must equal what the writer stored (`"1"` is not `1`).
+export function serializeJobSubject(subject) {
+    return JSON.stringify(Object.keys(subject)
+        .sort()
+        .map((field) => [field, subject[field] ?? null]));
+}
+// Stable identity for the declared payload fields: sorted keys, so two runs
+// with the same subject values produce the same string and the later one
+// replaces the earlier record. Non-primitives would serialize into something
+// no caller can reconstruct for a lookup, so they fail the run loudly.
+export function jobSubjectKey(jobName, payload, fields, declaration) {
+    if (fields === undefined || fields.length === 0)
+        return null;
+    const subject = {};
+    for (const field of fields) {
+        const value = payload[field] ?? null;
+        if (value !== null && typeof value === "object") {
+            throw new Error(`Job "${jobName}": ${declaration}.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
+        }
+        subject[field] = value;
+    }
+    return serializeJobSubject(subject);
+}
+export async function resolveJobTz(config, context) {
+    const tenantTz = config ? await config(TENANT_TIMEZONE_CONFIG_KEY) : undefined;
+    return createTzContext({
+        ...(context.geoTzProvider !== undefined && { geoTz: context.geoTzProvider }),
+        ...(context.clock !== undefined && { clock: context.clock }),
+        tenant: typeof tenantTz === "string" && isValidIanaTimeZone(tenantTz) ? tenantTz : "UTC",
+    });
+}
+// Jobs that never touch ctx.db run fine on a runner built without one; a job
+// that does gets a named error instead of a bare TypeError on undefined.
+export function missingDbProxy(jobName) {
+    // @cast-boundary db-operator — every access throws, so no TenantDb member is ever returned
+    return new Proxy({}, {
+        get() {
+            throw new InternalError({
+                message: `[jobs] job runner constructed without a db connection — ctx.db is unavailable for "${jobName}"`,
+            });
+        },
+    });
+}
+export function priorAttemptsOf(data) {
+    const value = data["_priorAttempts"];
+    return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
+}
+// Only a translation key ever travels to the tenant: the thrown error's own
+// i18nKey when it carries one, otherwise the key declared at the job. The
+// error message itself can echo provider or payload content and stays on the
+// run row and in the run log.
+export function tenantFailureMessageKey(err, declaration) {
+    // skip: job did not opt in — nothing tenant-visible to record
+    if (!declaration)
+        return null;
+    if (isKumikoError(err) && err.i18nKey.length > 0)
+        return err.i18nKey;
+    return declaration.messageKey;
+}
+// Serialized trace context lives under this key in the BullMQ job data.
+// Leading underscore matches the existing internal-meta convention
+// (_triggeredById, _tenantId, _payload).
+export const TRACE_CONTEXT_KEY = "_traceContext";
+export const DEFER_COUNT_KEY = "_deferCount";
+// Separate from the maxPerTenant counter so tenant-limit deferrals do not eat into maxDefers.
+export const HANDLER_DEFER_COUNT_KEY = "_handlerDeferCount";
+export const DEFAULT_MAX_DEFERS = 100;
+export const DEFER_DELAY_CAP_MS = 30_000;
+export const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
+export function readTraceContext(data) {
+    const raw = data[TRACE_CONTEXT_KEY];
+    if (!raw || typeof raw !== "object")
+        return undefined;
+    const ctx = raw; // @cast-boundary engine-payload
+    if (!ctx.traceId || !ctx.spanId)
+        return undefined;
+    return { traceId: ctx.traceId, spanId: ctx.spanId };
+}
+export function captureTraceContext(tracer) {
+    const span = tracer.getActiveSpan();
+    if (!span?.traceId)
+        return undefined;
+    return { traceId: span.traceId, spanId: span.spanId };
+}
+const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
+const QUEUE_DEPTH_WARN_INTERVAL_MS = 5 * 60_000;
+// kumiko_job_queue_depth: only the lane's own consumer polls — the count is
+// a global Redis-backed value (BullMQ, not per-process), so one reporter per
+// lane is enough. Fires once immediately (metrics exist before the first
+// poll tick) then on an interval; the caller stores + clears the handle.
+export async function startQueueDepthPolling(queue, lane, meter) {
+    let lastWarnAt;
+    const pollQueueDepth = async () => {
+        try {
+            const counts = await queue.getJobCounts("waiting", "active", "delayed", "failed", "paused");
+            emitJobQueueDepth(meter, lane, counts);
+        }
+        catch (err) {
+            // Next poll retries; the warn is throttled so a Redis outage logs once per interval, not once per poll.
+            const nowMs = Date.now();
+            if (lastWarnAt === undefined || nowMs - lastWarnAt >= QUEUE_DEPTH_WARN_INTERVAL_MS) {
+                lastWarnAt = nowMs;
+                createFallbackLogger("job-runner").warn("queue depth poll failed", {
+                    lane,
+                    error: scrubLogText(err instanceof Error ? err.message : String(err)),
+                });
+            }
+        }
+    };
+    await pollQueueDepth();
+    return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
+}
+export function parseRedisOpts(url) {
+    const parsed = new URL(url);
+    const result = {
+        host: parsed.hostname,
+        port: Number(parsed.port) || 6379,
+    };
+    if (parsed.pathname.length > 1) {
+        result.db = Number(parsed.pathname.slice(1));
+    }
+    return result;
+}
+// redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
+// would otherwise hang start() forever with no health endpoint to notice.
+export const BOOT_REDIS_TIMEOUT_MS = 10_000;
+// Shutdown must fit a k8s grace period even when Redis is down, so the
+// readiness wait in stop() is capped well below the boot timeout.
+export const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
+// A gate without an explicit `timeout` may legitimately run long (migration-style
+// checks), so it is never rejected; a hang just must not stay silent.
+export const BOOT_GATE_SLOW_WARN_MS = 60_000;
+export function timeoutReject(ms, message) {
+    let timer;
+    const promise = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(message)), ms);
+    });
+    return {
+        promise,
+        cancel: () => {
+            if (timer !== undefined)
+                clearTimeout(timer);
+        },
+    };
+}
+// Default base delay when a job opts into backoff without an explicit
+// delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
+// NaN/undefined, which is falsy — the job retries immediately instead of
+// waiting.
+const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
+// Shared by every enqueue path (dispatch(), handleEvent(), cron, runOnBoot,
+// perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
+// set must retry the same way regardless of how it got enqueued, or it
+// fails for good on the very first error on whichever path skips this.
+export function buildRetryBullOpts(jobDef, priorAttempts = 0) {
+    const opts = {};
+    if (jobDef.retries !== undefined)
+        opts.attempts = Math.max(1, jobDef.retries + 1 - priorAttempts);
+    if (jobDef.backoff) {
+        opts.backoff =
+            typeof jobDef.backoff === "string"
+                ? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
+                : {
+                    type: jobDef.backoff.type,
+                    delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
+                };
+    }
+    return opts;
+}
+// Equality filter on top-level payload fields for a job's trigger.where —
+// lets N jobs share one broad event QN, partitioned by a payload
+// discriminant instead of needing N distinct event types.
+export function payloadMatchesTriggerFilter(where, payload) {
+    if (!where)
+        return true;
+    return Object.entries(where).every(([key, value]) => payload[key] === value);
+}
+// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
+// OTHER jobs in the same completed/failed zset (moveToFinished lua,
+// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
+// the sweep only runs lazily on a later finish into that set. Per-job
+// retention is therefore meaningless on a shared queue — only queue-wide,
+// age-only retention (no count) is safe: a count would evict perTenant
+// children still inside their wrapper's retry window, and boot jobs. The
+// completed age must stay above every perTenant wrapper's own worst-case
+// retry window (see maxRetryWindowMs and the invariant check in
+// createJobRunner below), because child dedup (perTenantChildJobId) relies
+// on the children still existing in Redis when a wrapper retry re-derives
+// their ids. The margin here is generous — queue wait time isn't bounded by
+// backoff — to cover realistic windows. Audit trail lives in read_job_runs,
+// not BullMQ (fw#3199).
+export const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
+export const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
+// Total wall-clock time BullMQ can hold a perTenant wrapper across all its
+// retries, consistent with buildRetryBullOpts's own backoff computation.
+export function maxRetryWindowMs(jobDef) {
+    if (!jobDef.backoff || jobDef.retries === undefined)
+        return 0;
+    const delayMs = (typeof jobDef.backoff === "string" ? undefined : jobDef.backoff.delayMs) ??
+        DEFAULT_JOB_BACKOFF_DELAY_MS;
+    const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
+    return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
+}

package/dist/jobs/job-runner-schedulers.d.ts

@@ -0,0 +1,23 @@
+import type { Queue } from "bullmq";
+import type { JobRunIn } from "../engine/types/index.ts";
+export declare const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
+export declare function queueNameFor(prefix: string, lane: JobRunIn): string;
+export declare const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
+/**
+ * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
+ * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
+ * treated as old repeatables — spawning a new scheduler entry every tick and
+ * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
+ * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
+ */
+export declare function schedulerIdForJobName(jobName: string): string;
+export declare function bootJobIdForJobName(jobName: string): string;
+export declare const ORPHAN_SCHEDULER_GRACE_MS: number;
+export declare function markSchedulersSeen(queue: Queue, schedulerIds: ReadonlySet<string>): Promise<void>;
+type OrphanSchedulerLog = {
+    info(msg: string, data?: Record<string, unknown>): void;
+    warn(msg: string, data?: Record<string, unknown>): void;
+};
+export declare function removeOrphanJobSchedulers(queue: Queue, expectedSchedulerIds: ReadonlySet<string>, lane: JobRunIn, log: OrphanSchedulerLog): Promise<void>;
+export declare function perTenantChildJobId(wrapperJobId: string, tenantId: string): string;
+export {};

package/dist/jobs/job-runner-schedulers.js

@@ -0,0 +1,106 @@
+// Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
+// ("kumiko-jobs") — it must match between enqueuers and consumers, and an
+// accidental drift would silently drop jobs. Tests override via
+// `queueNamePrefix` for per-run isolation (stale jobs from a prior run
+// don't leak into a new test because the queue name includes a timestamp).
+export const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
+export function queueNameFor(prefix, lane) {
+    return `${prefix}-${lane}`;
+}
+// perTenant jobs need a source of truth for "which tenants are active" —
+// see createJobRunner's defaultGetActiveTenantIds below. The tenant feature
+// (bundled-features package) registers a query under this exact name when
+// mounted. framework must not depend on bundled-features (wrong direction —
+// bundled-features already depends on framework), so this is a name-only
+// coupling: createJobRunner looks the handler up in the registry by string
+// instead of importing it.
+export const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
+/**
+ * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
+ * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
+ * treated as old repeatables — spawning a new scheduler entry every tick and
+ * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
+ * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
+ */
+export function schedulerIdForJobName(jobName) {
+    return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
+}
+// Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
+// a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
+export function bootJobIdForJobName(jobName) {
+    return `boot-${jobName.replace(/[.:]/g, "-")}`;
+}
+const SCHEDULER_PAGE_SIZE = 100;
+// Rolling deploys run old and new pods against the same Redis for a while; an
+// old runner restarting in that window must not delete the scheduler of a
+// cron job the new version just added.
+export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
+function schedulerSeenKey(queue) {
+    return queue.toKey("kumiko-scheduler-seen");
+}
+// "Seen" has to mean a live runner holds the scheduler, so runners re-stamp
+// their own ids periodically; a single stamp at start() would let a runner
+// restarting during a mixed-version window longer than the grace period delete
+// a scheduler a healthy newer pod still owns.
+export async function markSchedulersSeen(queue, schedulerIds) {
+    // skip: no cron scheduler on this lane, nothing to stamp
+    if (schedulerIds.size === 0)
+        return;
+    const stamp = String(Date.now());
+    const entries = Object.fromEntries([...schedulerIds].map((id) => [id, stamp]));
+    await (await queue.client).hset(schedulerSeenKey(queue), entries);
+}
+// A cron job dropped from the app (or an older scheduler-id format) leaves its
+// scheduler in Redis, where it keeps firing jobs the worker no longer knows
+// ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
+export async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lane, log) {
+    try {
+        // Collect first: removing while paging would shift the indices.
+        const existingKeys = [];
+        for (let start = 0;; start += SCHEDULER_PAGE_SIZE) {
+            const page = await queue.getJobSchedulers(start, start + SCHEDULER_PAGE_SIZE - 1);
+            for (const scheduler of page)
+                existingKeys.push(scheduler.key);
+            if (page.length < SCHEDULER_PAGE_SIZE)
+                break;
+        }
+        const client = await queue.client;
+        const seenAt = await client.hgetall(schedulerSeenKey(queue));
+        const now = Date.now();
+        for (const key of existingKeys) {
+            if (expectedSchedulerIds.has(key))
+                continue;
+            const lastSeen = Number(seenAt[key]);
+            if (key in seenAt && now - lastSeen < ORPHAN_SCHEDULER_GRACE_MS)
+                continue;
+            await queue.removeJobScheduler(key);
+            await client.hdel(schedulerSeenKey(queue), key);
+            log.info("removed orphan job scheduler", { schedulerId: key, lane });
+        }
+    }
+    catch (err) {
+        log.warn("orphan job scheduler cleanup failed", {
+            lane,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+}
+// Deterministic per-tenant child job id, derived from the perTenant
+// wrapper's own BullMQ job id. A cron tick's wrapper id is
+// `repeat:<schedulerId>:<millis>` (stable for that tick, including across
+// BullMQ's own retry/stalled-job redelivery of the *same* job); a manual
+// dispatch's wrapper id is BullMQ's auto-generated id, equally stable
+// across retries of that job. Reusing it means a wrapper that runs twice
+// for the same trigger (retry after failure, a Redis drop, an instance
+// restarting mid-run) re-derives the *same* child ids, and BullMQ's
+// existing-jobId add() no-ops the second batch instead of creating
+// duplicates — safe only as long as the child job hash outlives the
+// wrapper's retry window, enforced at construction against
+// COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
+// createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
+// (fw#1603/#1604) — the wrapper id already contains ":", so strip
+// separators from both halves before joining instead of interpolating raw.
+export function perTenantChildJobId(wrapperJobId, tenantId) {
+    const sanitize = (value) => value.replace(/[.:]/g, "-");
+    return `child-${sanitize(wrapperJobId)}-${sanitize(tenantId)}`;
+}