@qlever-llc/trellis 0.10.22 → 0.11.0-rc.1

package/script/server/internal_jobs/job-manager.js

@@ -10,17 +10,20 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
     if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
     return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
 };
-var _JobManager_instances, _JobManager_context, _JobManager_meta, _JobManager_getQueueBinding, _JobManager_publishJobEvent;
+var _JobManager_instances, _JobManager_context, _JobManager_meta, _JobManager_getQueueBinding, _JobManager_publishJobEvent, _JobManager_submit, _JobManager_submitPrepared, _JobManager_admitKeyedCreate, _JobManager_removeQueuedKeyedReservation, _JobManager_restoreReplacedKeyedReservation, _JobManager_publishSkipped, _JobManager_keyedHeartbeat, _JobManager_releaseKeyedSlot, _JobManager_releaseKeyedSlotAfterPublishFailure, _JobManager_publishStaleCompletionIgnored;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.JobCancellationToken = exports.ActiveJobRuntimeError = exports.ActiveJob = exports.JobManager = exports.JobProcessError = void 0;
+exports.createJobContext = createJobContext;
 const nats_core_1 = require("@nats-io/nats-core");
 const ulid_1 = require("ulid");
 const carrier_js_1 = require("../../telemetry/carrier.js");
 const mod_js_1 = require("../../telemetry/mod.js");
+const jobs_js_1 = require("../../jobs.js");
 const active_job_js_1 = require("./active-job.js");
 Object.defineProperty(exports, "ActiveJob", { enumerable: true, get: function () { return active_job_js_1.ActiveJob; } });
 Object.defineProperty(exports, "ActiveJobRuntimeError", { enumerable: true, get: function () { return active_job_js_1.ActiveJobRuntimeError; } });
 Object.defineProperty(exports, "JobCancellationToken", { enumerable: true, get: function () { return active_job_js_1.JobCancellationToken; } });
+const key_coordinator_js_1 = require("./key-coordinator.js");
 class JobProcessError extends Error {
     constructor(kind, message) {
         super(message);
@@ -48,41 +51,83 @@ class JobManager {
         __classPrivateFieldSet(this, _JobManager_context, context, "f");
     }
     async create(type, payload) {
-        const binding = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, type);
-        const meta = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this);
-        const now = meta.nowIso();
-        const id = meta.nextJobId();
-        const context = createJobContext();
-        const namespace = __classPrivateFieldGet(this, _JobManager_context, "f").jobs.namespace;
-        const deadline = computeDeadline(now, binding.defaultDeadlineMs);
-        const job = {
-            id,
-            service: namespace,
-            type,
-            state: "pending",
-            context,
-            payload,
-            createdAt: now,
-            updatedAt: now,
-            tries: 0,
-            maxTries: binding.maxDeliver,
-            ...(deadline ? { deadline } : {}),
-        };
-        const event = {
-            jobId: id,
-            service: job.service,
-            jobType: type,
-            eventType: "created",
-            state: "pending",
-            context,
-            tries: 0,
-            maxTries: binding.maxDeliver,
-            payload,
-            ...(deadline ? { deadline } : {}),
-            timestamp: now,
-        };
-        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, type, id, event);
-        return job;
+        const outcome = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_submit).call(this, type, payload, true);
+        if (outcome.kind === "accepted") {
+            return outcome.job;
+        }
+        if (outcome.kind === "coalesced") {
+            throw new jobs_js_1.JobNotEnqueuedError({
+                reason: "coalesced",
+                key: outcome.key,
+                active: 0,
+                queued: 1,
+                limit: 0,
+                existingJobId: outcome.existing.id,
+            });
+        }
+        if (outcome.kind === "replaced") {
+            throw new jobs_js_1.JobNotEnqueuedError({
+                reason: "queue-depth",
+                key: outcome.key,
+                active: 0,
+                queued: 1,
+                limit: 0,
+                existingJobId: outcome.replaced.id,
+            });
+        }
+        throw new jobs_js_1.JobNotEnqueuedError({
+            reason: outcome.reason,
+            key: outcome.key,
+            active: outcome.active,
+            queued: outcome.queued,
+            limit: outcome.limit,
+        });
+    }
+    async submit(type, payload) {
+        return await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_submit).call(this, type, payload, false);
+    }
+    async createPrepared(submission) {
+        const outcome = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_submitPrepared).call(this, submission, true, true);
+        if (outcome.kind === "accepted") {
+            return outcome.job;
+        }
+        if (outcome.kind === "coalesced") {
+            throw new jobs_js_1.JobNotEnqueuedError({
+                reason: "coalesced",
+                key: outcome.key,
+                active: 0,
+                queued: 1,
+                limit: 0,
+                existingJobId: outcome.existing.id,
+            });
+        }
+        if (outcome.kind === "replaced") {
+            throw new jobs_js_1.JobNotEnqueuedError({
+                reason: "queue-depth",
+                key: outcome.key,
+                active: 0,
+                queued: 1,
+                limit: 0,
+                existingJobId: outcome.replaced.id,
+            });
+        }
+        throw new jobs_js_1.JobNotEnqueuedError({
+            reason: outcome.reason,
+            key: outcome.key,
+            active: outcome.active,
+            queued: outcome.queued,
+            limit: outcome.limit,
+        });
+    }
+    async submitPrepared(submission) {
+        return await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_submitPrepared).call(this, submission, false, true);
+    }
+    async cleanupQueuedKeyedJob(job) {
+        const queue = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, job.type);
+        const keyedPolicy = getKeyPolicy(queue);
+        if (!keyedPolicy)
+            return;
+        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_removeQueuedKeyedReservation).call(this, job, keyedPolicy);
     }
     process(job, cancellation, handler, metadata = {}, validation = {}) {
         return this.processWithHeartbeat(job, cancellation, () => {
@@ -90,25 +135,78 @@ class JobManager {
         }, handler, metadata, validation);
     }
     async processWithHeartbeat(job, cancellation, heartbeat, handler, metadata = {}, validation = {}) {
-        __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, job.type);
         const tries = job.tries + 1;
-        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
-            jobId: job.id,
-            service: job.service,
-            jobType: job.type,
-            eventType: "started",
-            state: "active",
-            previousState: job.state,
-            context: job.context,
-            tries,
-            timestamp: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
-        });
+        const queue = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, job.type);
+        const keyedPolicy = getKeyPolicy(queue);
+        let lease;
+        let staleSlots = [];
+        if (keyedPolicy) {
+            const coordinator = __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator;
+            if (!coordinator) {
+                throw new Error(`Keyed jobs queue '${job.type}' requires the JOBS_KEYS coordinator`);
+            }
+            const acquired = await coordinator.acquireActiveSlot({
+                service: job.service,
+                jobType: job.type,
+                jobId: job.id,
+                payload: job.payload,
+                context: job.context,
+                lifecycleState: metadata.latestState ?? job.state,
+                tries,
+                instanceId: metadata.instanceId ?? "unknown",
+                now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+                policy: keyedPolicy,
+            });
+            if (acquired.kind === "blocked") {
+                return { outcome: "deferred", tries, reason: acquired.reason };
+            }
+            lease = {
+                key: acquired.key,
+                keyHash: acquired.keyHash,
+                slotToken: acquired.slotToken,
+                policy: keyedPolicy,
+            };
+            staleSlots = acquired.stale;
+        }
+        try {
+            for (const stale of staleSlots) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, stale.jobId, {
+                    jobId: stale.jobId,
+                    service: job.service,
+                    jobType: job.type,
+                    eventType: "stale",
+                    state: "stale",
+                    previousState: "active",
+                    context: stale.context,
+                    tries: stale.tries,
+                    error: "keyed job lease expired",
+                    timestamp: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+                });
+            }
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
+                jobId: job.id,
+                service: job.service,
+                jobType: job.type,
+                eventType: "started",
+                state: "active",
+                previousState: job.state,
+                context: job.context,
+                tries,
+                timestamp: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+            });
+        }
+        catch (error) {
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlotAfterPublishFailure).call(this, job, lease, error);
+            throw error;
+        }
         try {
-            const result = await this.withActiveJobAndHeartbeat({ ...job, state: "active", tries }, cancellation, heartbeat, handler, metadata);
+            const result = await this.withActiveJobAndHeartbeat({ ...job, state: "active", tries }, cancellation, lease ? __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_keyedHeartbeat).call(this, job, lease, heartbeat) : heartbeat, handler, metadata, lease);
             if (cancellation.isHostShutdown()) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
                 return { outcome: "interrupted", tries };
             }
             if (cancellation.isCancelled()) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
                 return { outcome: "cancelled", tries };
             }
             try {
@@ -121,6 +219,11 @@ class JobManager {
             catch (error) {
                 throw JobProcessError.failed(error instanceof Error ? error.message : String(error));
             }
+            const release = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
+            if (release === "staleCompletion") {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishStaleCompletionIgnored).call(this, job, tries);
+                return { outcome: "stale_completion_ignored", tries };
+            }
             await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
                 jobId: job.id,
                 service: job.service,
@@ -137,9 +240,11 @@ class JobManager {
         }
         catch (error) {
             if (cancellation.isHostShutdown()) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
                 return { outcome: "interrupted", tries };
             }
             if (cancellation.isCancelled()) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
                 return { outcome: "cancelled", tries };
             }
             if (error instanceof JobProcessError) {
@@ -151,6 +256,11 @@ class JobManager {
                         operation: job.type,
                         phase: "handler_result",
                     });
+                    const release = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
+                    if (release === "staleCompletion") {
+                        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishStaleCompletionIgnored).call(this, job, tries);
+                        return { outcome: "stale_completion_ignored", tries };
+                    }
                     await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
                         jobId: job.id,
                         service: job.service,
@@ -171,6 +281,11 @@ class JobManager {
                     operation: job.type,
                     phase: "handler_result",
                 });
+                const release = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
+                if (release === "staleCompletion") {
+                    await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishStaleCompletionIgnored).call(this, job, tries);
+                    return { outcome: "stale_completion_ignored", tries };
+                }
                 await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
                     jobId: job.id,
                     service: job.service,
@@ -241,7 +356,10 @@ class JobManager {
             throw new active_job_js_1.ActiveJobRuntimeError("worker heartbeat unavailable");
         }, f, metadata);
     }
-    async withActiveJobAndHeartbeat(job, cancellation, heartbeat, f, metadata = {}) {
+    async withActiveJobAndHeartbeat(job, cancellation, heartbeat, f, metadata = {}, lease) {
+        const stopAutoHeartbeat = lease
+            ? startAutoHeartbeat(heartbeat, lease.policy.heartbeatIntervalMs)
+            : () => { };
         const activeJob = new active_job_js_1.ActiveJob(job, cancellation, {
             updateProgress: (progress) => this.emitProgress(job, progress),
             log: (entry) => this.emitLog(job, entry),
@@ -259,7 +377,12 @@ class JobManager {
         }, {
             redeliveryCount: metadata.redeliveryCount ?? 0,
         });
-        return await f(activeJob);
+        try {
+            return await f(activeJob);
+        }
+        finally {
+            stopAutoHeartbeat();
+        }
     }
 }
 exports.JobManager = JobManager;
@@ -274,9 +397,9 @@ _JobManager_context = new WeakMap(), _JobManager_instances = new WeakSet(), _Job
         throw new Error(`Missing jobs binding for queue '${type}'`);
     }
     return binding;
-}, _JobManager_publishJobEvent = async function _JobManager_publishJobEvent(type, jobId, event) {
+}, _JobManager_publishJobEvent = async function _JobManager_publishJobEvent(type, jobId, event, stableMessageId) {
     const binding = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, type);
-    const headers = headersFromJobContext(event.context);
+    const headers = headersFromJobContext(event.context, stableMessageId);
     try {
         await __classPrivateFieldGet(this, _JobManager_context, "f").nc.publish(`${binding.publishPrefix}.${jobId}.${event.eventType}`, new TextEncoder().encode(JSON.stringify(event)), { headers });
     }
@@ -290,7 +413,256 @@ _JobManager_context = new WeakMap(), _JobManager_instances = new WeakSet(), _Job
         });
         throw error;
     }
+}, _JobManager_submit = async function _JobManager_submit(type, payload, strictCreate) {
+    const meta = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this);
+    const now = meta.nowIso();
+    const id = meta.nextJobId();
+    const context = createJobContext();
+    const namespace = __classPrivateFieldGet(this, _JobManager_context, "f").jobs.namespace;
+    const submission = {
+        submissionId: id,
+        mode: strictCreate ? "create" : "submit",
+        service: namespace,
+        queue: type,
+        jobId: id,
+        payload,
+        createdAt: now,
+        context,
+    };
+    return await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_submitPrepared).call(this, submission, strictCreate);
+}, _JobManager_submitPrepared = async function _JobManager_submitPrepared(submission, strictCreate, stableMessageIds) {
+    const { queue: type, jobId: id, createdAt: now, context, service, payload, } = submission;
+    const binding = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, type);
+    const deadline = computeDeadline(now, binding.defaultDeadlineMs);
+    const job = {
+        id,
+        service,
+        type,
+        state: "pending",
+        context,
+        payload,
+        createdAt: now,
+        updatedAt: now,
+        tries: 0,
+        maxTries: binding.maxDeliver,
+        ...(deadline ? { deadline } : {}),
+    };
+    const event = {
+        jobId: id,
+        service,
+        jobType: type,
+        eventType: "created",
+        state: "pending",
+        context,
+        tries: 0,
+        maxTries: binding.maxDeliver,
+        payload,
+        ...(deadline ? { deadline } : {}),
+        timestamp: now,
+    };
+    const keyedPolicy = getKeyPolicy(binding);
+    if (keyedPolicy) {
+        const admission = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_admitKeyedCreate).call(this, {
+            type,
+            job,
+            event,
+            policy: keyedPolicy,
+            strictCreate,
+            submissionId: submission.submissionId,
+        });
+        if (admission.kind !== "accepted" && admission.kind !== "replaced") {
+            return admission;
+        }
+        if (strictCreate && admission.kind === "replaced") {
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_restoreReplacedKeyedReservation).call(this, job, admission.replaced, keyedPolicy, submission.submissionId);
+            return {
+                kind: "rejected",
+                key: admission.key,
+                reason: "queue-depth",
+                active: admission.state.active.length,
+                queued: admission.state.queued.length,
+                limit: keyedPolicy.queue.maxQueuedPerKey,
+            };
+        }
+        if (admission.kind === "replaced") {
+            try {
+                const skippedMsgId = stableMessageIds
+                    ? `trellis-job-skipped:${submission.submissionId}:${admission.replaced.id}`
+                    : undefined;
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishSkipped).call(this, type, admission.replaced, now, skippedMsgId);
+            }
+            catch (error) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_restoreReplacedKeyedReservation).call(this, job, admission.replaced, keyedPolicy, submission.submissionId);
+                throw error;
+            }
+        }
+        try {
+            const createdMsgId = stableMessageIds
+                ? `trellis-job-created:${submission.submissionId}`
+                : undefined;
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, type, id, event, createdMsgId);
+        }
+        catch (error) {
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_removeQueuedKeyedReservation).call(this, job, keyedPolicy);
+            throw error;
+        }
+        if (admission.kind === "replaced") {
+            return {
+                kind: "replaced",
+                key: admission.key,
+                replaced: {
+                    service: admission.replaced.service,
+                    jobType: admission.replaced.jobType,
+                    id: admission.replaced.id,
+                },
+                job,
+            };
+        }
+        return { kind: "accepted", job, key: admission.key };
+    }
+    const createdMsgId = stableMessageIds
+        ? `trellis-job-created:${submission.submissionId}`
+        : undefined;
+    await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, type, id, event, createdMsgId);
+    return { kind: "accepted", job };
+}, _JobManager_admitKeyedCreate = async function _JobManager_admitKeyedCreate(args) {
+    const coordinator = __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator;
+    if (!coordinator) {
+        throw new Error(`Keyed jobs queue '${args.type}' requires the JOBS_KEYS coordinator`);
+    }
+    return await coordinator.admitCreate({
+        service: args.job.service,
+        jobType: args.type,
+        jobId: args.job.id,
+        payload: args.job.payload,
+        context: args.event.context,
+        createdAt: args.event.timestamp,
+        policy: args.policy,
+        strictCreate: args.strictCreate,
+        ...(args.submissionId ? { submissionId: args.submissionId } : {}),
+    });
+}, _JobManager_removeQueuedKeyedReservation = async function _JobManager_removeQueuedKeyedReservation(job, policy) {
+    const coordinator = __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator;
+    if (!coordinator)
+        return;
+    await coordinator.removeQueuedJob({
+        service: job.service,
+        jobType: job.type,
+        jobId: job.id,
+        payload: job.payload,
+        now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+        policy,
+    });
+}, _JobManager_restoreReplacedKeyedReservation = async function _JobManager_restoreReplacedKeyedReservation(job, replaced, policy, submissionId) {
+    const coordinator = __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator;
+    if (!coordinator)
+        return;
+    await coordinator.restoreReplacedQueuedJob({
+        service: job.service,
+        jobType: job.type,
+        replacementJobId: job.id,
+        replaced,
+        payload: job.payload,
+        now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+        policy,
+        ...(submissionId ? { submissionId } : {}),
+    });
+}, _JobManager_publishSkipped = async function _JobManager_publishSkipped(type, replaced, timestamp, stableMessageId) {
+    await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, type, replaced.id, {
+        jobId: replaced.id,
+        service: __classPrivateFieldGet(this, _JobManager_context, "f").jobs.namespace,
+        jobType: type,
+        eventType: "skipped",
+        state: "skipped",
+        previousState: "pending",
+        context: replaced.context,
+        tries: 0,
+        error: "replaced by newer keyed job",
+        timestamp,
+    }, stableMessageId);
+}, _JobManager_keyedHeartbeat = function _JobManager_keyedHeartbeat(job, lease, heartbeat) {
+    return async () => {
+        await heartbeat();
+        const renewed = await __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator.renewHeartbeat({
+            service: job.service,
+            jobType: job.type,
+            jobId: job.id,
+            lease,
+            now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+        });
+        if (renewed.kind === "lost") {
+            (0, mod_js_1.recordTrellisError)(new active_job_js_1.ActiveJobRuntimeError("keyed job slot lease was lost"), {
+                surface: "job",
+                direction: "worker",
+                operation: job.type,
+                phase: "heartbeat",
+            });
+        }
+    };
+}, _JobManager_releaseKeyedSlot = async function _JobManager_releaseKeyedSlot(job, lease) {
+    if (!lease)
+        return undefined;
+    const released = await __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator.releaseActiveSlot({
+        service: job.service,
+        jobType: job.type,
+        jobId: job.id,
+        lease,
+        now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+    });
+    return released.kind === "released" ? "released" : "staleCompletion";
+}, _JobManager_releaseKeyedSlotAfterPublishFailure = async function _JobManager_releaseKeyedSlotAfterPublishFailure(job, lease, publishError) {
+    try {
+        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
+    }
+    catch (cleanupError) {
+        (0, mod_js_1.recordTrellisError)(cleanupError, {
+            surface: "job",
+            direction: "worker",
+            operation: job.type,
+            phase: "keyed_slot_cleanup",
+        });
+        (0, mod_js_1.recordTrellisError)(publishError, {
+            surface: "job",
+            direction: "worker",
+            operation: job.type,
+            phase: "publish",
+        });
+    }
+}, _JobManager_publishStaleCompletionIgnored = async function _JobManager_publishStaleCompletionIgnored(job, tries) {
+    await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
+        jobId: job.id,
+        service: job.service,
+        jobType: job.type,
+        eventType: "staleCompletionIgnored",
+        state: "stale",
+        previousState: "active",
+        context: job.context,
+        tries,
+        error: "keyed job slot lease was lost before completion",
+        timestamp: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+    });
 };
+function getKeyPolicy(binding) {
+    if (!binding.keyConcurrency) {
+        return undefined;
+    }
+    return (0, key_coordinator_js_1.normalizeJobKeyPolicy)({
+        keyConcurrency: binding.keyConcurrency,
+        queue: binding.queue,
+    });
+}
+function startAutoHeartbeat(heartbeat, intervalMs) {
+    const timer = setInterval(() => {
+        void heartbeat().catch((error) => {
+            (0, mod_js_1.recordTrellisError)(error, {
+                surface: "job",
+                direction: "worker",
+                phase: "heartbeat",
+            });
+        });
+    }, intervalMs);
+    return () => clearInterval(timer);
+}
 function computeDeadline(now, deadlineMs) {
     if (deadlineMs === undefined) {
         return undefined;
@@ -314,13 +686,16 @@ function createJobContext() {
         ...(tracestate ? { tracestate } : {}),
     };
 }
-function headersFromJobContext(context) {
+function headersFromJobContext(context, stableMessageId) {
     const headers = (0, nats_core_1.headers)();
     headers.set("request-id", context.requestId);
     headers.set("traceparent", context.traceparent);
     if (context.tracestate) {
         headers.set("tracestate", context.tracestate);
     }
+    if (stableMessageId) {
+        headers.set("Nats-Msg-Id", stableMessageId);
+    }
     return headers;
 }
 function isValidTraceparent(value) {