@cosmicdrift/kumiko-framework 0.363.0 → 0.365.0

package/dist/files/purge-expired-file-ref-bytes.js

@@ -1,34 +1,59 @@
 import { deleteMany, selectMany } from "../db/query.js";
-import { deleteStoredFileAndDerivatives } from "../derivatives/delete-stored-file.js";
+import { deleteFileRefBytes } from "../derivatives/delete-stored-file.js";
 export const EXPIRED_FILE_REF_PURGE_BATCH = 500;
+const PAGE_ORDER = [
+    { col: "deletedAt", direction: "asc" },
+    { col: "id", direction: "asc" },
+];
+// Keyset page over (deletedAt, id): rows at the cursor's instant with a larger id first, then
+// strictly later instants. `where` already bounds deletedAt by the cutoff; the later-instants
+// query repeats that bound next to the cursor.
+async function selectNextPage(args) {
+    const { db, table, where, expiredBefore, cursor, pageSize } = args;
+    const options = { orderBy: PAGE_ORDER, limit: pageSize };
+    if (cursor === undefined)
+        return selectMany(db, table, where, options);
+    const sameInstant = await selectMany(db, table, { ...where, deletedAt: cursor.deletedAt, id: { gt: cursor.id } }, options);
+    if (sameInstant.length >= pageSize)
+        return sameInstant;
+    const later = await selectMany(db, table, { ...where, deletedAt: { gt: cursor.deletedAt, lt: expiredBefore } }, { ...options, limit: pageSize - sameInstant.length });
+    return [...sameInstant, ...later];
+}
 // Deletes the bytes (original + derivatives) of expired soft-deleted fileRef rows, then the
 // rows. Bytes first and fail-closed: once the row is gone nobody knows the storageKey, so a
-// row whose bytes could not be removed stays and the next run retries it.
+// row whose bytes could not be removed stays and the next run retries it. A run walks the
+// expired rows page by page along (deletedAt, id), so rows that keep failing cannot hide the
+// rows behind them; the run ends on the first empty page.
 export async function purgeExpiredFileRefsWithBytes(args) {
-    const { db, table, where, files, log } = args;
+    const { db, table, where, expiredBefore, files, log } = args;
+    const pageSize = args.pageSize ?? EXPIRED_FILE_REF_PURGE_BATCH;
     if (!files) {
         log?.warn("[soft-delete-cleanup] no file storage available, expired fileRef rows kept");
         return;
     }
-    const rows = await selectMany(db, table, where, {
-        orderBy: [{ col: "deletedAt", direction: "asc" }],
-        limit: EXPIRED_FILE_REF_PURGE_BATCH,
-    });
-    const purgedIds = [];
-    const store = { list: files.list, delete: (key) => files.ref(key).delete() };
-    for (const row of rows) {
-        try {
-            const failedKeys = await deleteStoredFileAndDerivatives(row.storageKey, store, "soft-delete-cleanup:fileRef");
-            if (failedKeys.length === 0) {
-                purgedIds.push(row.id);
-                continue;
+    let cursor;
+    for (;;) {
+        const rows = await selectNextPage({ db, table, where, expiredBefore, cursor, pageSize });
+        const lastRow = rows[rows.length - 1];
+        // skip: an empty page means every expired row has been visited
+        if (lastRow === undefined)
+            return;
+        cursor = lastRow;
+        const purgedIds = [];
+        for (const row of rows) {
+            try {
+                const failedKeys = await deleteFileRefBytes(files, row, "soft-delete-cleanup:fileRef");
+                if (failedKeys.length === 0) {
+                    purgedIds.push(row.id);
+                    continue;
+                }
+                log?.warn(`[soft-delete-cleanup] bytes of fileRef ${row.id} not fully deleted, row kept`);
+            }
+            catch (err) {
+                log?.warn(`[soft-delete-cleanup] bytes of fileRef ${row.id} not deleted, row kept: ${err instanceof Error ? err.message : String(err)}`);
             }
-            log?.warn(`[soft-delete-cleanup] bytes of fileRef ${row.id} not fully deleted, row kept`);
-        }
-        catch (err) {
-            log?.warn(`[soft-delete-cleanup] bytes of fileRef ${row.id} not deleted, row kept: ${err instanceof Error ? err.message : String(err)}`);
         }
+        if (purgedIds.length > 0)
+            await deleteMany(db, table, { ...where, id: { in: purgedIds } });
     }
-    if (purgedIds.length > 0)
-        await deleteMany(db, table, { ...where, id: { in: purgedIds } });
 }

package/dist/files/resolve-content-type.js

@@ -13,6 +13,41 @@ const UNAMBIGUOUS_SNIFFED_TYPES = new Set([
 ]);
 const ZIP_SIGNATURE = [0x50, 0x4b, 0x03, 0x04];
 const XML_PROBE_BYTES = 512;
+// Root elements of markup that starts with `<` but is not a document the XML
+// ingest providers should receive: HTML (documents and fragments), SVG.
+const NON_XML_ROOT_ELEMENTS = new Set([
+    "html",
+    "head",
+    "body",
+    "script",
+    "style",
+    "meta",
+    "link",
+    "title",
+    "svg",
+    "div",
+    "span",
+    "p",
+    "a",
+    "img",
+    "br",
+    "hr",
+    "table",
+    "ul",
+    "ol",
+    "li",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "section",
+    "article",
+    "font",
+    "center",
+]);
+const ROOT_ELEMENT_PATTERN = /^<(?:[A-Za-z_][\w.-]*:)?([A-Za-z_][\w.-]*)/;
 const EXTENSION_FALLBACK = {
     xml: "application/xml",
     pdf: "application/pdf",
@@ -38,8 +73,8 @@ function hasPrefix(bytes, prefix) {
     return prefix.every((byte, i) => bytes[i] === byte);
 }
 // Textual start only: a NUL in the probe means binary (or UTF-16, which is
-// deliberately not detected). HTML also starts with `<` and must not be
-// relabelled as XML.
+// deliberately not detected). Without an XML declaration the root element
+// decides: HTML/SVG roots (also namespace-prefixed) are not relabelled as XML.
 function looksLikeXml(bytes) {
     const probe = bytes.subarray(0, XML_PROBE_BYTES);
     if (probe.includes(0))
@@ -47,9 +82,10 @@ function looksLikeXml(bytes) {
     const text = new TextDecoder().decode(probe).replace(/^/, "").trimStart();
     if (text.startsWith("<?xml"))
         return true;
-    if (!/^<[A-Za-z_]/.test(text))
+    const root = ROOT_ELEMENT_PATTERN.exec(text)?.[1];
+    if (root === undefined)
         return false;
-    return !/^<(html|head|body|script)\b/i.test(text);
+    return !NON_XML_ROOT_ELEMENTS.has(root.toLowerCase());
 }
 function extensionOf(filename) {
     const dot = filename.lastIndexOf(".");

package/dist/files/storage-tracking.js

@@ -12,6 +12,7 @@
 // into createApp / setupTestStack alongside their domain features.
 import { bigint, instant, integer, table as pgTable, sql, uuid } from "../db/dialect.js";
 import { entityEventName, executeRawQueryRead } from "../db/index.js";
+import { toPostgresArrayLiteral } from "../db/queries/postgres-array-literal.js";
 import { incrementCounter } from "../db/query.js";
 import { defineFeature, qn, toKebab } from "../engine/index.js";
 import { SHARED_INSTANCE_SENTINEL, selectConsumerCursorForUpdate, } from "../pipeline/index.js";
@@ -141,12 +142,12 @@ export async function transferTenantStorageUsage(args) {
     if (!cursor)
         return;
     const rows = await executeRawQueryRead(db, `SELECT "id"::text AS id, "type", "payload"::text AS payload FROM kumiko_events
-     WHERE "tenant_id" = $1 AND "aggregate_type" = $2 AND "aggregate_id" = ANY($3)
-       AND "type" = ANY($4) AND "id" <= $5`, [
+     WHERE "tenant_id" = $1 AND "aggregate_type" = $2 AND "aggregate_id" = ANY($3::uuid[])
+       AND "type" = ANY($4::text[]) AND "id" <= $5`, [
         sourceTenantId,
         FILE_REF_AGGREGATE_TYPE,
-        fileRefIds,
-        TRACKED_FILE_REF_EVENT_TYPES,
+        toPostgresArrayLiteral(fileRefIds),
+        toPostgresArrayLiteral(TRACKED_FILE_REF_EVENT_TYPES),
         cursor.lastProcessedEventId,
     ]);
     let bytes = 0;

package/dist/files/types.js

@@ -1,3 +1,4 @@
+import { readZipEntryNames } from "./zip-entry-names.js";
 // Lives here (not in file-ref-entity.ts) so engine code can name the entity without importing
 // a module that calls createEntity at load time.
 export const FILE_REF_ENTITY_NAME = "fileRef";
@@ -111,13 +112,9 @@ const MAGIC_BYTE_SIGNATURES = [
 // so a byte search finds the docx part directory without unzipping. The main
 // part is resolved via _rels/.rels and may be named e.g. word/document2.xml.
 const DOCX_PART_PREFIX = "word/";
-function containsAscii(bytes, ascii) {
-    const first = ascii.charCodeAt(0);
-    for (let i = bytes.indexOf(first); i !== -1; i = bytes.indexOf(first, i + 1)) {
-        if (startsWithAscii(bytes, ascii, i))
-            return true;
-    }
-    return false;
+function hasDocxPart(zipBytes) {
+    const names = readZipEntryNames(zipBytes);
+    return names?.some((name) => name.startsWith(DOCX_PART_PREFIX)) ?? false;
 }
 function startsWithBytes(bytes, signature) {
     if (bytes.length < signature.length)
@@ -237,7 +234,7 @@ export function validateFileContent(fileName, content, accept) {
         return { kind: "ok" };
     const sniffed = sniffMimeType(content);
     // sniffMimeType only sees the ZIP container; any .xlsx/.pptx/.zip shares it.
-    if (sniffed === DOCX_MIME_TYPE && !containsAscii(content, DOCX_PART_PREFIX)) {
+    if (sniffed === DOCX_MIME_TYPE && !hasDocxPart(content)) {
         return {
             kind: "rejected",
             error: `content_mismatch: ".${ext}" upload bytes are a ZIP archive without ${DOCX_PART_PREFIX} entries, not a docx`,

package/dist/files/zip-entry-names.d.ts

@@ -0,0 +1 @@
+export declare function readZipEntryNames(bytes: Uint8Array): readonly string[] | null;

package/dist/files/zip-entry-names.js

@@ -0,0 +1,46 @@
+const EOCD_SIGNATURE = 0x06054b50;
+const CENTRAL_HEADER_SIGNATURE = 0x02014b50;
+const EOCD_MIN_LENGTH = 22;
+const EOCD_MAX_COMMENT_LENGTH = 0xffff;
+const CENTRAL_HEADER_FIXED_LENGTH = 46;
+// Entry names from the central directory, which (unlike local headers) always
+// carries sizes and names even for streamed ZIPs written with data descriptors.
+// Returns null when the container is not a readable (non-ZIP64) ZIP.
+export function readZipEntryNames(bytes) {
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const eocdOffset = findEndOfCentralDirectory(view);
+    if (eocdOffset === null)
+        return null;
+    const entryCount = view.getUint16(eocdOffset + 10, true);
+    let offset = view.getUint32(eocdOffset + 16, true);
+    const decoder = new TextDecoder();
+    const names = [];
+    for (let i = 0; i < entryCount; i++) {
+        if (offset + CENTRAL_HEADER_FIXED_LENGTH > view.byteLength)
+            return null;
+        if (view.getUint32(offset, true) !== CENTRAL_HEADER_SIGNATURE)
+            return null;
+        const nameLength = view.getUint16(offset + 28, true);
+        const extraLength = view.getUint16(offset + 30, true);
+        const commentLength = view.getUint16(offset + 32, true);
+        const nameStart = offset + CENTRAL_HEADER_FIXED_LENGTH;
+        if (nameStart + nameLength > view.byteLength)
+            return null;
+        names.push(decoder.decode(bytes.subarray(nameStart, nameStart + nameLength)));
+        offset = nameStart + nameLength + extraLength + commentLength;
+    }
+    return names;
+}
+function findEndOfCentralDirectory(view) {
+    const lowest = Math.max(0, view.byteLength - EOCD_MIN_LENGTH - EOCD_MAX_COMMENT_LENGTH);
+    for (let offset = view.byteLength - EOCD_MIN_LENGTH; offset >= lowest; offset--) {
+        if (view.getUint32(offset, true) !== EOCD_SIGNATURE)
+            continue;
+        // A signature inside the archive comment is skipped: the real record's
+        // comment length reaches exactly to the end of the buffer.
+        const commentLength = view.getUint16(offset + 20, true);
+        if (offset + EOCD_MIN_LENGTH + commentLength === view.byteLength)
+            return offset;
+    }
+    return null;
+}

package/dist/jobs/index.d.ts

@@ -1,2 +1,2 @@
-export type { JobLogEntry, JobMeta, JobOutcomeMeta, JobQueuedMeta, JobRunner, JobRunnerOptions, JobSubjectValue, } from "./job-runner.js";
+export type { JobDroppedMeta, JobLogEntry, JobMeta, JobOutcomeMeta, JobQueuedMeta, JobRunner, JobRunnerOptions, JobSubjectValue, } from "./job-runner.js";
 export { createJobRunner, serializeJobSubject } from "./job-runner.js";

package/dist/jobs/job-runner.d.ts

@@ -29,6 +29,9 @@ export type JobQueuedMeta = {
     readonly tenantId: string;
     readonly subject: string | null;
 };
+export type JobDroppedMeta = {
+    readonly tenantId: string;
+};
 export type JobOutcomeMeta = {
     readonly tenantId: string;
     readonly finalAttempt: boolean;
@@ -57,6 +60,7 @@ export type JobRunnerOptions = {
     consumerLane?: JobRunIn | undefined;
     queueNamePrefix?: string | undefined;
     bootRedisTimeoutMs?: number | undefined;
+    schedulerSeenRefreshMs?: number | undefined;
     jobRetention?: {
         completedAgeSec?: number | undefined;
         failedAgeSec?: number | undefined;
@@ -66,6 +70,6 @@ export type JobRunnerOptions = {
     onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
     onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
     onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
-    onJobDropped?: (jobName: string, jobId: string) => void;
+    onJobDropped?: (jobName: string, jobId: string, meta: JobDroppedMeta) => void;
 };
 export declare function createJobRunner(options: JobRunnerOptions): JobRunner;

package/dist/jobs/job-runner.js

@@ -12,7 +12,7 @@ import { InternalError } from "../errors/index.js";
 import { isKumikoError } from "../errors/kumiko-error.js";
 import { createFileContext } from "../files/file-handle.js";
 import { createFallbackLogger } from "../logging/index.js";
-import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
+import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
 import { RedisKeys } from "../pipeline/redis-keys.js";
 import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js";
@@ -82,6 +82,18 @@ export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
 function schedulerSeenKey(queue) {
     return queue.toKey("kumiko-scheduler-seen");
 }
+// "Seen" has to mean a live runner holds the scheduler, so runners re-stamp
+// their own ids periodically; a single stamp at start() would let a runner
+// restarting during a mixed-version window longer than the grace period delete
+// a scheduler a healthy newer pod still owns.
+async function markSchedulersSeen(queue, schedulerIds) {
+    // skip: no cron scheduler on this lane, nothing to stamp
+    if (schedulerIds.size === 0)
+        return;
+    const stamp = String(Date.now());
+    const entries = Object.fromEntries([...schedulerIds].map((id) => [id, stamp]));
+    await (await queue.client).hset(schedulerSeenKey(queue), entries);
+}
 // A cron job dropped from the app (or an older scheduler-id format) leaves its
 // scheduler in Redis, where it keeps firing jobs the worker no longer knows
 // ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
@@ -364,6 +376,7 @@ export function createJobRunner(options) {
     const { registry, context, redisUrl, consumerLane } = options;
     const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;
     const bootRedisTimeoutMs = options.bootRedisTimeoutMs ?? BOOT_REDIS_TIMEOUT_MS;
+    const schedulerSeenRefreshMs = options.schedulerSeenRefreshMs ?? ORPHAN_SCHEDULER_GRACE_MS / 4;
     const redisOpts = parseRedisOpts(redisUrl);
     // Use the context's tracer when present (observability-provider injected at
     // boot); otherwise noop so dispatch/handleJob stay zero-cost without config.
@@ -524,6 +537,11 @@ export function createJobRunner(options) {
     }
     let worker = null;
     let queueDepthTimer = null;
+    let schedulerSeenTimer = null;
+    let schedulerStampPending = false;
+    // A timed-out gate is abandoned, not aborted (the job context carries no
+    // AbortSignal); stop() waits for it so it cannot write after the shutdown.
+    const abandonedBootGates = new Set();
     // Forward reference to the runner's own API, exposed on the job-handler ctx
     // so a handler can dispatch a follow-up job (job→job chaining, e.g.
     // delivery.render → delivery.send). Assigned just before return; reads happen
@@ -611,12 +629,16 @@ export function createJobRunner(options) {
             });
         }
     }
-    async function announceDropped(jobName, jobDef, bullJobId) {
+    async function announceDropped(jobName, jobDef, bullJobId, data) {
         // skip: job did not opt in or no logger wired
         if (!jobDef.tenantVisibleRun || !options.onJobDropped)
             return;
+        const tenantId = tenantIdOfJobData(data);
+        // skip: tenant-less run — announceQueued never recorded a row for it
+        if (tenantId === SYSTEM_TENANT_ID)
+            return;
         try {
-            await options.onJobDropped(jobName, bullJobId);
+            await options.onJobDropped(jobName, bullJobId, { tenantId });
         }
         catch (err) {
             errorLogger.warn("tenant-visible queued state not cleared", {
@@ -732,7 +754,7 @@ export function createJobRunner(options) {
                     ...buildRetryBullOpts(jobDef, priorAttempts),
                 });
                 if (bullJob.id !== undefined && bullJob.id !== requeued.id) {
-                    await announceDropped(jobName, jobDef, bullJob.id);
+                    await announceDropped(jobName, jobDef, bullJob.id, bullJob.data);
                 }
                 await announceQueued(jobName, jobDef, requeued.id, requeuedData);
                 // skip: lock taken, work re-enqueued with delay, current invocation done
@@ -939,6 +961,7 @@ export function createJobRunner(options) {
             attempt,
             finalAttempt,
             log: createJobLogger(logs),
+            metricsFor: createMetricsForFactory(context.meter),
             ...(triggerName !== undefined && { triggerName }),
             ...(triggerEvent !== undefined && { triggerEvent }),
             write: (qn, payload) => {
@@ -1074,17 +1097,18 @@ export function createJobRunner(options) {
                     });
                 }, BOOT_GATE_SLOW_WARN_MS)
                 : undefined;
+            const gateRun = handleJob({
+                id: bootJobIdForJobName(name),
+                name,
+                data: {},
+                attemptsMade: 0,
+                retryable: false,
+            });
+            const gateSettled = gateRun.then(() => undefined, () => undefined);
+            abandonedBootGates.add(gateSettled);
+            void gateSettled.then(() => abandonedBootGates.delete(gateSettled));
             try {
-                await Promise.race([
-                    handleJob({
-                        id: bootJobIdForJobName(name),
-                        name,
-                        data: {},
-                        attemptsMade: 0,
-                        retryable: false,
-                    }),
-                    ...(gateTimeout ? [gateTimeout.promise] : []),
-                ]);
+                await Promise.race([gateRun, ...(gateTimeout ? [gateTimeout.promise] : [])]);
             }
             catch (err) {
                 // Leaving the worker's Redis connections open would keep the event loop alive and turn the aborted boot into
@@ -1100,6 +1124,74 @@ export function createJobRunner(options) {
             }
         }
     }
+    function cronSchedulerIdsForLane(lane) {
+        const ids = new Set();
+        for (const [name, jobDef] of allJobs) {
+            if (laneForJob(jobDef) === lane && "cron" in jobDef.trigger) {
+                ids.add(schedulerIdForJobName(name));
+            }
+        }
+        return ids;
+    }
+    async function upsertCronSchedulers(queue, lane) {
+        for (const [name, jobDef] of allJobs) {
+            if (laneForJob(jobDef) !== lane)
+                continue;
+            if ("cron" in jobDef.trigger) {
+                await queue.upsertJobScheduler(schedulerIdForJobName(name), cronRepeatOptions(jobDef.trigger), {
+                    name: jobDef.perTenant ? `_perTenant:${name}` : name,
+                    data: {},
+                    // Queue-level defaultJobOptions (jobRetentionOpts) covers
+                    // retention; a count here would sweep queue-wide again and
+                    // evict perTenant children and boot jobs.
+                    opts: {
+                        ...buildRetryBullOpts(jobDef),
+                    },
+                });
+            }
+        }
+    }
+    // After a failed initial stamp an older runner may have removed our
+    // schedulers as unseen, so the next tick stamps and re-upserts before the
+    // flag clears.
+    async function refreshSchedulerSeen(queue, lane, schedulerIds) {
+        await markSchedulersSeen(queue, schedulerIds);
+        if (schedulerStampPending) {
+            await upsertCronSchedulers(queue, lane);
+            schedulerStampPending = false;
+        }
+    }
+    function startSchedulerSeenRefresh(queue, lane, schedulerIds) {
+        if (schedulerIds.size === 0)
+            return null;
+        const timer = setInterval(() => {
+            refreshSchedulerSeen(queue, lane, schedulerIds).catch((err) => {
+                errorLogger.warn("scheduler seen refresh failed", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            });
+        }, schedulerSeenRefreshMs);
+        timer.unref();
+        return timer;
+    }
+    // Bounded: a gate that never returns must not block shutdown as well.
+    async function settleAbandonedBootGates(maxWaitMs) {
+        // skip: no timed-out gate left behind
+        if (abandonedBootGates.size === 0)
+            return;
+        const wait = timeoutReject(maxWaitMs, "job-runner: abandoned boot gate still running at stop()");
+        try {
+            await Promise.race([Promise.all(abandonedBootGates), wait.promise]);
+        }
+        catch (err) {
+            errorLogger.warn("abandoned boot gate did not finish before stop()", {
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+        finally {
+            wait.cancel();
+        }
+    }
     const runnerApi = {
         async start() {
             // skip: enqueuer-only runner — no BullMQ worker, no cron schedules,
@@ -1148,28 +1240,18 @@ export function createJobRunner(options) {
             // Only schedule cron + boot for jobs that belong to this lane. Jobs
             // assigned to the other lane get their cron/boot wiring from the
             // runner running on that lane. Running both here would double-fire.
-            const expectedSchedulerIds = new Set();
-            for (const [name, jobDef] of allJobs) {
-                if (laneForJob(jobDef) !== consumerLane)
-                    continue;
-                if ("cron" in jobDef.trigger) {
-                    const schedulerId = schedulerIdForJobName(name);
-                    expectedSchedulerIds.add(schedulerId);
-                    await consumerQueue.upsertJobScheduler(schedulerId, cronRepeatOptions(jobDef.trigger), {
-                        name: jobDef.perTenant ? `_perTenant:${name}` : name,
-                        data: {},
-                        // Queue-level defaultJobOptions (jobRetentionOpts) covers
-                        // retention; a count here would sweep queue-wide again and
-                        // evict perTenant children and boot jobs.
-                        opts: {
-                            ...buildRetryBullOpts(jobDef),
-                        },
-                    });
-                    await (await consumerQueue.client).hset(schedulerSeenKey(consumerQueue), {
-                        [schedulerId]: String(Date.now()),
-                    });
-                }
-            }
+            const expectedSchedulerIds = cronSchedulerIdsForLane(consumerLane);
+            // Before the upserts: a seen entry without a scheduler is harmless, a
+            // scheduler a concurrent older runner finds without one gets deleted.
+            // Best-effort like the periodic refresh: a failed stamp must not stop the start.
+            await markSchedulersSeen(consumerQueue, expectedSchedulerIds).catch((err) => {
+                schedulerStampPending = true;
+                errorLogger.warn("scheduler seen stamp failed", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            });
+            await upsertCronSchedulers(consumerQueue, consumerLane);
+            schedulerSeenTimer = startSchedulerSeenRefresh(consumerQueue, consumerLane, expectedSchedulerIds);
             await removeOrphanJobSchedulers(consumerQueue, expectedSchedulerIds, consumerLane, {
                 info: (msg, data) => context.log ? context.log.info(msg, data) : errorLogger.debug(msg, data),
                 warn: errorLogger.warn,
@@ -1217,6 +1299,10 @@ export function createJobRunner(options) {
                 clearInterval(queueDepthTimer);
                 queueDepthTimer = null;
             }
+            if (schedulerSeenTimer) {
+                clearInterval(schedulerSeenTimer);
+                schedulerSeenTimer = null;
+            }
             if (worker) {
                 await worker.close();
                 worker = null;
@@ -1226,6 +1312,7 @@ export function createJobRunner(options) {
             // without listeners -> unhandled rejection (fw#1805). Wait for readiness
             // first; a failed/timed-out wait must not block the close itself.
             const stopReadyTimeoutMs = Math.min(bootRedisTimeoutMs, STOP_QUEUE_READY_TIMEOUT_MS);
+            await settleAbandonedBootGates(stopReadyTimeoutMs);
             const readinessTimeout = timeoutReject(stopReadyTimeoutMs, `job-runner: Queue not ready within ${stopReadyTimeoutMs}ms during stop()`);
             try {
                 await Promise.race([
@@ -1298,7 +1385,7 @@ export function createJobRunner(options) {
                     for (const j of waiting) {
                         if (j.name === jobName && j.id) {
                             await j.remove();
-                            await announceDropped(jobName, jobDef, j.id);
+                            await announceDropped(jobName, jobDef, j.id, j.data);
                         }
                     }
                     break;

package/dist/observability/escape-hatch-report.d.ts

@@ -11,10 +11,11 @@ export declare function createEscapeHatchReportWindow(opts?: {
 }): EscapeHatchReportWindow;
 export type EscapeHatchProcessDedup = {
     readonly kinds: ReadonlySet<EscapeHatchKind>;
-    readonly seen: Set<string>;
+    readonly lastAuditedAt: Map<string, number>;
 };
 export declare function createEscapeHatchProcessDedup(kinds: readonly EscapeHatchKind[]): EscapeHatchProcessDedup;
 export declare function withEscapeHatchAuditScope<T>(run: () => Promise<T>): Promise<T>;
+export declare function withEscapeHatchAuditScopeStream<T>(run: () => AsyncGenerator<T>): AsyncGenerator<T>;
 export declare function flushEscapeHatchAudits(): Promise<void>;
 export declare function reportEscapeHatchUse(event: EscapeHatchUseEvent, deps: {
     readonly sink?: EscapeHatchAuditSink;

package/dist/observability/escape-hatch-report.js

@@ -27,8 +27,11 @@ export function createEscapeHatchReportWindow(opts) {
         },
     };
 }
+// A long-lived worker must leave an audit event inside every retention window
+// (90 days), or its crons vanish from the log once the first event is pruned.
+const PROCESS_REAUDIT_INTERVAL_MS = 24 * 60 * 60 * 1000;
 export function createEscapeHatchProcessDedup(kinds) {
-    return { kinds: new Set(kinds), seen: new Set() };
+    return { kinds: new Set(kinds), lastAuditedAt: new Map() };
 }
 // Default window for createEscapeHatchReporter callers that don't pass their own.
 const fallbackReportWindow = createEscapeHatchReportWindow();
@@ -37,9 +40,14 @@ const processPendingAudits = new Set();
 // The sink writes of the dispatch or job run currently executing.
 const scopePendingAudits = new AsyncLocalStorage();
 function trackPendingAudit(write) {
+    const scope = scopePendingAudits.getStore();
     processPendingAudits.add(write);
-    scopePendingAudits.getStore()?.add(write);
-    void write.finally(() => processPendingAudits.delete(write));
+    scope?.add(write);
+    // Settled writes leave the scope so a long-lived stream does not accumulate them.
+    void write.finally(() => {
+        processPendingAudits.delete(write);
+        scope?.delete(write);
+    });
 }
 // Awaits the audit writes started inside `run` once it has returned, so callers place it around
 // the transaction and the write is durable when the dispatch or run ends. Nested scopes belong
@@ -56,6 +64,30 @@ export async function withEscapeHatchAuditScope(run) {
         await Promise.allSettled([...pending]);
     }
 }
+// Stream variant: the scope has to be entered on every pull because an async generator body
+// runs in the caller's async context, and the wait happens once the stream ends or is closed.
+export async function* withEscapeHatchAuditScopeStream(run) {
+    if (scopePendingAudits.getStore() !== undefined) {
+        yield* run();
+        // skip: an enclosing scope already waits for the pending audits
+        return;
+    }
+    const pending = new Set();
+    const source = run();
+    try {
+        while (true) {
+            const next = await scopePendingAudits.run(pending, () => source.next());
+            // skip: the source is exhausted
+            if (next.done)
+                return;
+            yield next.value;
+        }
+    }
+    finally {
+        await scopePendingAudits.run(pending, () => source.return(undefined));
+        await Promise.allSettled([...pending]);
+    }
+}
 export async function flushEscapeHatchAudits() {
     while (processPendingAudits.size > 0) {
         await Promise.allSettled([...processPendingAudits]);
@@ -105,11 +137,20 @@ export function createEscapeHatchReporter(opts) {
             emitEscapeHatchUse(opts.meter, opts.handler, kind);
         const { processDedup } = opts;
         if (processDedup?.kinds.has(kind)) {
-            const processKey = JSON.stringify([opts.handler, kind, opts.tenantId]);
-            // skip: this (handler, kind, tenant) was already audited by this process
-            if (processDedup.seen.has(processKey))
+            const processKey = JSON.stringify([
+                opts.handler,
+                kind,
+                opts.tenantId,
+                target?.id ?? null,
+                target?.tenantId ?? null,
+            ]);
+            const nowMs = now();
+            const lastAuditedAt = processDedup.lastAuditedAt.get(processKey);
+            // skip: audited by this process within the re-audit interval
+            if (lastAuditedAt !== undefined && nowMs - lastAuditedAt < PROCESS_REAUDIT_INTERVAL_MS) {
                 return;
-            processDedup.seen.add(processKey);
+            }
+            processDedup.lastAuditedAt.set(processKey, nowMs);
             reportEscapeHatchUse(eventFor(kind, reason, target), deps);
             // skip: audited above, the window dedup below only serves the other kinds
             return;

package/dist/observability/index.d.ts

@@ -4,7 +4,7 @@ export { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeH
 export { getFallbackMeter, getFallbackProvider, getFallbackTracer } from "./fallback.js";
 export { generateSpanId, generateTraceId } from "./ids.js";
 export { buildMetricName, validateLabelKey, validateMetricName, } from "./metric-validator.js";
-export { createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
+export { createMetricsForFactory, createMetricsHandle, createNoopMetricsHandle, createSafeMetricsHandle, createUnboundMetricsHandle, } from "./metrics-handle.js";
 export { type ObservabilityWiring, prometheusMetricsEnvSchema, resolveObservabilityWiring, } from "./metrics-wiring.js";
 export { createNoopProvider } from "./noop-provider.js";
 export { createPrometheusMeter, type PrometheusMeter, type PrometheusMeterSnapshot, serializeOpenMetrics, } from "./prometheus-meter.js";