@qlever-llc/trellis 0.10.17 → 0.19.0-rc.1

package/esm/server/internal_jobs/job-manager.js

@@ -9,12 +9,14 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
     if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
     return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
 };
-var _JobManager_instances, _JobManager_context, _JobManager_meta, _JobManager_getQueueBinding, _JobManager_publishJobEvent;
+var _JobManager_instances, _JobManager_context, _JobManager_meta, _JobManager_getQueueBinding, _JobManager_publishJobEvent, _JobManager_submit, _JobManager_admitKeyedCreate, _JobManager_removeQueuedKeyedReservation, _JobManager_restoreReplacedKeyedReservation, _JobManager_publishSkipped, _JobManager_keyedHeartbeat, _JobManager_releaseKeyedSlot, _JobManager_releaseKeyedSlotAfterPublishFailure, _JobManager_publishStaleCompletionIgnored;
 import { headers as natsHeaders } from "@nats-io/nats-core";
 import { ulid } from "ulid";
 import { createMapCarrier, injectTraceContext, } from "../../telemetry/carrier.js";
 import { recordTrellisError } from "../../telemetry/mod.js";
+import { JobNotEnqueuedError } from "../../jobs.js";
 import { ActiveJob, ActiveJobRuntimeError, JobCancellationToken, } from "./active-job.js";
+import { normalizeJobKeyPolicy } from "./key-coordinator.js";
 export class JobProcessError extends Error {
     constructor(kind, message) {
         super(message);
@@ -41,41 +43,47 @@ export class JobManager {
         __classPrivateFieldSet(this, _JobManager_context, context, "f");
     }
     async create(type, payload) {
-        const binding = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, type);
-        const meta = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this);
-        const now = meta.nowIso();
-        const id = meta.nextJobId();
-        const context = createJobContext();
-        const namespace = __classPrivateFieldGet(this, _JobManager_context, "f").jobs.namespace;
-        const deadline = computeDeadline(now, binding.defaultDeadlineMs);
-        const job = {
-            id,
-            service: namespace,
-            type,
-            state: "pending",
-            context,
-            payload,
-            createdAt: now,
-            updatedAt: now,
-            tries: 0,
-            maxTries: binding.maxDeliver,
-            ...(deadline ? { deadline } : {}),
-        };
-        const event = {
-            jobId: id,
-            service: job.service,
-            jobType: type,
-            eventType: "created",
-            state: "pending",
-            context,
-            tries: 0,
-            maxTries: binding.maxDeliver,
-            payload,
-            ...(deadline ? { deadline } : {}),
-            timestamp: now,
-        };
-        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, type, id, event);
-        return job;
+        const outcome = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_submit).call(this, type, payload, true);
+        if (outcome.kind === "accepted") {
+            return outcome.job;
+        }
+        if (outcome.kind === "coalesced") {
+            throw new JobNotEnqueuedError({
+                reason: "coalesced",
+                key: outcome.key,
+                active: 0,
+                queued: 1,
+                limit: 0,
+                existingJobId: outcome.existing.id,
+            });
+        }
+        if (outcome.kind === "replaced") {
+            throw new JobNotEnqueuedError({
+                reason: "queue-depth",
+                key: outcome.key,
+                active: 0,
+                queued: 1,
+                limit: 0,
+                existingJobId: outcome.replaced.id,
+            });
+        }
+        throw new JobNotEnqueuedError({
+            reason: outcome.reason,
+            key: outcome.key,
+            active: outcome.active,
+            queued: outcome.queued,
+            limit: outcome.limit,
+        });
+    }
+    async submit(type, payload) {
+        return await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_submit).call(this, type, payload, false);
+    }
+    async cleanupQueuedKeyedJob(job) {
+        const queue = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, job.type);
+        const keyedPolicy = getKeyPolicy(queue);
+        if (!keyedPolicy)
+            return;
+        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_removeQueuedKeyedReservation).call(this, job, keyedPolicy);
     }
     process(job, cancellation, handler, metadata = {}, validation = {}) {
         return this.processWithHeartbeat(job, cancellation, () => {
@@ -83,25 +91,78 @@ export class JobManager {
         }, handler, metadata, validation);
     }
     async processWithHeartbeat(job, cancellation, heartbeat, handler, metadata = {}, validation = {}) {
-        __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, job.type);
         const tries = job.tries + 1;
-        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
-            jobId: job.id,
-            service: job.service,
-            jobType: job.type,
-            eventType: "started",
-            state: "active",
-            previousState: job.state,
-            context: job.context,
-            tries,
-            timestamp: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
-        });
+        const queue = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, job.type);
+        const keyedPolicy = getKeyPolicy(queue);
+        let lease;
+        let staleSlots = [];
+        if (keyedPolicy) {
+            const coordinator = __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator;
+            if (!coordinator) {
+                throw new Error(`Keyed jobs queue '${job.type}' requires the JOBS_KEYS coordinator`);
+            }
+            const acquired = await coordinator.acquireActiveSlot({
+                service: job.service,
+                jobType: job.type,
+                jobId: job.id,
+                payload: job.payload,
+                context: job.context,
+                lifecycleState: metadata.latestState ?? job.state,
+                tries,
+                instanceId: metadata.instanceId ?? "unknown",
+                now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+                policy: keyedPolicy,
+            });
+            if (acquired.kind === "blocked") {
+                return { outcome: "deferred", tries, reason: acquired.reason };
+            }
+            lease = {
+                key: acquired.key,
+                keyHash: acquired.keyHash,
+                slotToken: acquired.slotToken,
+                policy: keyedPolicy,
+            };
+            staleSlots = acquired.stale;
+        }
+        try {
+            for (const stale of staleSlots) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, stale.jobId, {
+                    jobId: stale.jobId,
+                    service: job.service,
+                    jobType: job.type,
+                    eventType: "stale",
+                    state: "stale",
+                    previousState: "active",
+                    context: stale.context,
+                    tries: stale.tries,
+                    error: "keyed job lease expired",
+                    timestamp: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+                });
+            }
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
+                jobId: job.id,
+                service: job.service,
+                jobType: job.type,
+                eventType: "started",
+                state: "active",
+                previousState: job.state,
+                context: job.context,
+                tries,
+                timestamp: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+            });
+        }
+        catch (error) {
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlotAfterPublishFailure).call(this, job, lease, error);
+            throw error;
+        }
         try {
-            const result = await this.withActiveJobAndHeartbeat({ ...job, state: "active", tries }, cancellation, heartbeat, handler, metadata);
+            const result = await this.withActiveJobAndHeartbeat({ ...job, state: "active", tries }, cancellation, lease ? __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_keyedHeartbeat).call(this, job, lease, heartbeat) : heartbeat, handler, metadata, lease);
             if (cancellation.isHostShutdown()) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
                 return { outcome: "interrupted", tries };
             }
             if (cancellation.isCancelled()) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
                 return { outcome: "cancelled", tries };
             }
             try {
@@ -114,6 +175,11 @@ export class JobManager {
             catch (error) {
                 throw JobProcessError.failed(error instanceof Error ? error.message : String(error));
             }
+            const release = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
+            if (release === "staleCompletion") {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishStaleCompletionIgnored).call(this, job, tries);
+                return { outcome: "stale_completion_ignored", tries };
+            }
             await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
                 jobId: job.id,
                 service: job.service,
@@ -130,9 +196,11 @@ export class JobManager {
         }
         catch (error) {
             if (cancellation.isHostShutdown()) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
                 return { outcome: "interrupted", tries };
             }
             if (cancellation.isCancelled()) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
                 return { outcome: "cancelled", tries };
             }
             if (error instanceof JobProcessError) {
@@ -144,6 +212,11 @@ export class JobManager {
                         operation: job.type,
                         phase: "handler_result",
                     });
+                    const release = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
+                    if (release === "staleCompletion") {
+                        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishStaleCompletionIgnored).call(this, job, tries);
+                        return { outcome: "stale_completion_ignored", tries };
+                    }
                     await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
                         jobId: job.id,
                         service: job.service,
@@ -164,6 +237,11 @@ export class JobManager {
                     operation: job.type,
                     phase: "handler_result",
                 });
+                const release = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
+                if (release === "staleCompletion") {
+                    await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishStaleCompletionIgnored).call(this, job, tries);
+                    return { outcome: "stale_completion_ignored", tries };
+                }
                 await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
                     jobId: job.id,
                     service: job.service,
@@ -234,7 +312,10 @@ export class JobManager {
             throw new ActiveJobRuntimeError("worker heartbeat unavailable");
         }, f, metadata);
     }
-    async withActiveJobAndHeartbeat(job, cancellation, heartbeat, f, metadata = {}) {
+    async withActiveJobAndHeartbeat(job, cancellation, heartbeat, f, metadata = {}, lease) {
+        const stopAutoHeartbeat = lease
+            ? startAutoHeartbeat(heartbeat, lease.policy.heartbeatIntervalMs)
+            : () => { };
         const activeJob = new ActiveJob(job, cancellation, {
             updateProgress: (progress) => this.emitProgress(job, progress),
             log: (entry) => this.emitLog(job, entry),
@@ -252,7 +333,12 @@ export class JobManager {
         }, {
             redeliveryCount: metadata.redeliveryCount ?? 0,
         });
-        return await f(activeJob);
+        try {
+            return await f(activeJob);
+        }
+        finally {
+            stopAutoHeartbeat();
+        }
     }
 }
 _JobManager_context = new WeakMap(), _JobManager_instances = new WeakSet(), _JobManager_meta = function _JobManager_meta() {
@@ -282,7 +368,231 @@ _JobManager_context = new WeakMap(), _JobManager_instances = new WeakSet(), _Job
         });
         throw error;
     }
+}, _JobManager_submit = async function _JobManager_submit(type, payload, strictCreate) {
+    const binding = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_getQueueBinding).call(this, type);
+    const meta = __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this);
+    const now = meta.nowIso();
+    const id = meta.nextJobId();
+    const context = createJobContext();
+    const namespace = __classPrivateFieldGet(this, _JobManager_context, "f").jobs.namespace;
+    const deadline = computeDeadline(now, binding.defaultDeadlineMs);
+    const job = {
+        id,
+        service: namespace,
+        type,
+        state: "pending",
+        context,
+        payload,
+        createdAt: now,
+        updatedAt: now,
+        tries: 0,
+        maxTries: binding.maxDeliver,
+        ...(deadline ? { deadline } : {}),
+    };
+    const event = {
+        jobId: id,
+        service: job.service,
+        jobType: type,
+        eventType: "created",
+        state: "pending",
+        context,
+        tries: 0,
+        maxTries: binding.maxDeliver,
+        payload,
+        ...(deadline ? { deadline } : {}),
+        timestamp: now,
+    };
+    const keyedPolicy = getKeyPolicy(binding);
+    if (keyedPolicy) {
+        const admission = await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_admitKeyedCreate).call(this, {
+            type,
+            job,
+            event,
+            policy: keyedPolicy,
+            strictCreate,
+        });
+        if (admission.kind !== "accepted" && admission.kind !== "replaced") {
+            return admission;
+        }
+        if (strictCreate && admission.kind === "replaced") {
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_restoreReplacedKeyedReservation).call(this, job, admission.replaced, keyedPolicy);
+            return {
+                kind: "rejected",
+                key: admission.key,
+                reason: "queue-depth",
+                active: admission.state.active.length,
+                queued: admission.state.queued.length,
+                limit: keyedPolicy.queue.maxQueuedPerKey,
+            };
+        }
+        if (admission.kind === "replaced") {
+            try {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishSkipped).call(this, type, admission.replaced, now);
+            }
+            catch (error) {
+                await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_restoreReplacedKeyedReservation).call(this, job, admission.replaced, keyedPolicy);
+                throw error;
+            }
+        }
+        try {
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, type, id, event);
+        }
+        catch (error) {
+            await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_removeQueuedKeyedReservation).call(this, job, keyedPolicy);
+            throw error;
+        }
+        if (admission.kind === "replaced") {
+            return {
+                kind: "replaced",
+                key: admission.key,
+                replaced: {
+                    service: admission.replaced.service,
+                    jobType: admission.replaced.jobType,
+                    id: admission.replaced.id,
+                },
+                job,
+            };
+        }
+        return { kind: "accepted", job, key: admission.key };
+    }
+    await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, type, id, event);
+    return { kind: "accepted", job };
+}, _JobManager_admitKeyedCreate = async function _JobManager_admitKeyedCreate(args) {
+    const coordinator = __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator;
+    if (!coordinator) {
+        throw new Error(`Keyed jobs queue '${args.type}' requires the JOBS_KEYS coordinator`);
+    }
+    return await coordinator.admitCreate({
+        service: args.job.service,
+        jobType: args.type,
+        jobId: args.job.id,
+        payload: args.job.payload,
+        context: args.event.context,
+        createdAt: args.event.timestamp,
+        policy: args.policy,
+        strictCreate: args.strictCreate,
+    });
+}, _JobManager_removeQueuedKeyedReservation = async function _JobManager_removeQueuedKeyedReservation(job, policy) {
+    const coordinator = __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator;
+    if (!coordinator)
+        return;
+    await coordinator.removeQueuedJob({
+        service: job.service,
+        jobType: job.type,
+        jobId: job.id,
+        payload: job.payload,
+        now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+        policy,
+    });
+}, _JobManager_restoreReplacedKeyedReservation = async function _JobManager_restoreReplacedKeyedReservation(job, replaced, policy) {
+    const coordinator = __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator;
+    if (!coordinator)
+        return;
+    await coordinator.restoreReplacedQueuedJob({
+        service: job.service,
+        jobType: job.type,
+        replacementJobId: job.id,
+        replaced,
+        payload: job.payload,
+        now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+        policy,
+    });
+}, _JobManager_publishSkipped = async function _JobManager_publishSkipped(type, replaced, timestamp) {
+    await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, type, replaced.id, {
+        jobId: replaced.id,
+        service: __classPrivateFieldGet(this, _JobManager_context, "f").jobs.namespace,
+        jobType: type,
+        eventType: "skipped",
+        state: "skipped",
+        previousState: "pending",
+        context: replaced.context,
+        tries: 0,
+        error: "replaced by newer keyed job",
+        timestamp,
+    });
+}, _JobManager_keyedHeartbeat = function _JobManager_keyedHeartbeat(job, lease, heartbeat) {
+    return async () => {
+        await heartbeat();
+        const renewed = await __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator.renewHeartbeat({
+            service: job.service,
+            jobType: job.type,
+            jobId: job.id,
+            lease,
+            now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+        });
+        if (renewed.kind === "lost") {
+            recordTrellisError(new ActiveJobRuntimeError("keyed job slot lease was lost"), {
+                surface: "job",
+                direction: "worker",
+                operation: job.type,
+                phase: "heartbeat",
+            });
+        }
+    };
+}, _JobManager_releaseKeyedSlot = async function _JobManager_releaseKeyedSlot(job, lease) {
+    if (!lease)
+        return undefined;
+    const released = await __classPrivateFieldGet(this, _JobManager_context, "f").keyCoordinator.releaseActiveSlot({
+        service: job.service,
+        jobType: job.type,
+        jobId: job.id,
+        lease,
+        now: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+    });
+    return released.kind === "released" ? "released" : "staleCompletion";
+}, _JobManager_releaseKeyedSlotAfterPublishFailure = async function _JobManager_releaseKeyedSlotAfterPublishFailure(job, lease, publishError) {
+    try {
+        await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_releaseKeyedSlot).call(this, job, lease);
+    }
+    catch (cleanupError) {
+        recordTrellisError(cleanupError, {
+            surface: "job",
+            direction: "worker",
+            operation: job.type,
+            phase: "keyed_slot_cleanup",
+        });
+        recordTrellisError(publishError, {
+            surface: "job",
+            direction: "worker",
+            operation: job.type,
+            phase: "publish",
+        });
+    }
+}, _JobManager_publishStaleCompletionIgnored = async function _JobManager_publishStaleCompletionIgnored(job, tries) {
+    await __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_publishJobEvent).call(this, job.type, job.id, {
+        jobId: job.id,
+        service: job.service,
+        jobType: job.type,
+        eventType: "staleCompletionIgnored",
+        state: "stale",
+        previousState: "active",
+        context: job.context,
+        tries,
+        error: "keyed job slot lease was lost before completion",
+        timestamp: __classPrivateFieldGet(this, _JobManager_instances, "m", _JobManager_meta).call(this).nowIso(),
+    });
 };
+function getKeyPolicy(binding) {
+    if (!binding.keyConcurrency) {
+        return undefined;
+    }
+    return normalizeJobKeyPolicy({
+        keyConcurrency: binding.keyConcurrency,
+        queue: binding.queue,
+    });
+}
+function startAutoHeartbeat(heartbeat, intervalMs) {
+    const timer = setInterval(() => {
+        void heartbeat().catch((error) => {
+            recordTrellisError(error, {
+                surface: "job",
+                direction: "worker",
+                phase: "heartbeat",
+            });
+        });
+    }, intervalMs);
+    return () => clearInterval(timer);
+}
 function computeDeadline(now, deadlineMs) {
     if (deadlineMs === undefined) {
         return undefined;