@terreno/jobs 57.16.2 → 57.18.0

package/dist/jobExecutor.d.ts

@@ -21,12 +21,32 @@ export type ExecuteJobOutcome = {
 } | {
     kind: "not_found";
 };
+/** Result of an atomic claim: a row to run, or a reclaimed row dead-lettered with no attempts left. */
+export type JobClaimResult = {
+    claim: JobClaimLock;
+    job: JobDocument;
+    kind: "claimed";
+} | {
+    job: JobDocument;
+    kind: "dead";
+};
 export declare const runClaimedJob: ({ claim, host, job, signal, }: {
     claim: JobClaimLock;
     host: JobExecutionHost;
     job: JobDocument;
     signal: AbortSignal;
 }) => Promise<JobDocument>;
+/**
+ * Atomically claims one due job matching `filter`. An expired `running` row is reclaimed first and
+ * the lost attempt is counted; once that exhausts `maxAttempts` the row is dead-lettered instead of
+ * run again.
+ */
+export declare const claimDueJob: ({ filter, lockTtlMs, now, sort, }: {
+    filter?: Record<string, unknown>;
+    lockTtlMs: number;
+    now?: Date;
+    sort?: Record<string, 1 | -1>;
+}) => Promise<JobClaimResult | undefined>;
 export declare const executeJobById: ({ host, jobId, signal, }: {
     host: JobExecutionHost;
     jobId: string;

package/dist/jobExecutor.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.executeJobById = exports.runClaimedJob = void 0;
+exports.executeJobById = exports.claimDueJob = exports.runClaimedJob = void 0;
 const api_1 = require("@terreno/api");
 const luxon_1 = require("luxon");
 const mongoose_1 = __importDefault(require("mongoose"));
@@ -12,17 +12,10 @@ const claimLock_1 = require("./claimLock");
 const job_1 = require("./models/job");
 const retryBackoff_1 = require("./retryBackoff");
 const TERMINAL_NOOP_STATUSES = new Set(["cancelled", "completed", "dead", "failed"]);
+const LEASE_EXPIRED_ERROR = "Lease expired before the previous attempt finished";
+const LEASE_EXPIRED_ERROR_CLASS = "LeaseExpired";
 const buildLockExpiry = (lockTtlMs, now) => luxon_1.DateTime.fromJSDate(now, { zone: "utc" }).minus({ milliseconds: lockTtlMs }).toJSDate();
 const isLiveLock = (lockedAt, lockExpiry) => Boolean(lockedAt && lockedAt > lockExpiry);
-const readClaimLock = (job) => {
-    if (!job.lockedAt || !job.lockedBy) {
-        throw new Error(`Job ${job._id.toString()} is missing claim lock metadata`);
-    }
-    return {
-        lockedAt: job.lockedAt,
-        lockedBy: job.lockedBy,
-    };
-};
 const classifyConflict = (job, lockExpiry, now) => {
     if (job.runAt > now) {
         return { kind: "conflict", reason: "not_due", runAt: job.runAt };
@@ -122,6 +115,54 @@ const recordHandlerFailure = async ({ claim, error, job, signal, }) => {
     }
     return pending;
 };
+const buildLeaseRenewalIntervalMs = (lockTtlMs) => Math.max(1, Math.floor(lockTtlMs / 3));
+/**
+ * Renews the claim's lockedAt while a handler runs so no other consumer treats the live job as
+ * abandoned. Renewals run one at a time; when a renewal no longer matches the claim, the lease is
+ * lost and the handler signal is aborted.
+ */
+const startLeaseHeartbeat = ({ claim, jobId, leaseController, lockTtlMs, }) => {
+    let currentClaim = claim;
+    let leaseLost = false;
+    let stopped = false;
+    let renewal = Promise.resolve();
+    const renew = async () => {
+        if (stopped || leaseLost) {
+            return;
+        }
+        const renewedAt = luxon_1.DateTime.utc().toJSDate();
+        const renewed = await job_1.Job.findOneAndUpdate({ ...(0, claimLock_1.buildClaimOwnershipFilter)(jobId, currentClaim), status: "running" }, { $set: { lockedAt: renewedAt } }, { returnDocument: "after" });
+        if (!renewed) {
+            leaseLost = true;
+            clearInterval(timer);
+            api_1.logger.warn(`[jobs] Lost the claim lease for job ${jobId.toString()}; aborting handler`);
+            leaseController.abort();
+            return;
+        }
+        currentClaim = { lockedAt: renewedAt, lockedBy: currentClaim.lockedBy };
+    };
+    const timer = setInterval(() => {
+        renewal = renewal.then(renew).catch((error) => {
+            api_1.logger.warn(`[jobs] Failed to renew the claim lease for job ${jobId.toString()}: ${String(error)}`);
+        });
+    }, buildLeaseRenewalIntervalMs(lockTtlMs));
+    return {
+        getClaim: () => currentClaim,
+        isLeaseLost: () => leaseLost,
+        stop: async () => {
+            stopped = true;
+            clearInterval(timer);
+            await renewal;
+        },
+    };
+};
+const linkAbortSignal = (parent, controller) => {
+    if (parent.aborted) {
+        controller.abort();
+        return;
+    }
+    parent.addEventListener("abort", () => controller.abort(), { once: true });
+};
 const runClaimedJob = async ({ claim, host, job, signal, }) => {
     const definition = host.getDefinition(job.name);
     if (!definition) {
@@ -132,7 +173,17 @@ const runClaimedJob = async ({ claim, host, job, signal, }) => {
         labels: { jobId: job._id.toString(), jobName: job.name },
         prefix: "[Job]",
     });
-    const executionSignal = (0, activeJobExecution_1.beginActiveJobExecution)(job._id.toString(), signal);
+    const leaseController = new AbortController();
+    linkAbortSignal(signal, leaseController);
+    const executionSignal = (0, activeJobExecution_1.beginActiveJobExecution)(job._id.toString(), leaseController.signal);
+    const heartbeat = startLeaseHeartbeat({
+        claim,
+        jobId: job._id,
+        leaseController,
+        lockTtlMs: host.getLockTtlMs(),
+    });
+    let handlerError;
+    let handlerFailed = false;
     try {
         await (0, api_1.runWithRequestContext)({ jobId: job._id.toString() }, async () => {
             await definition.handler(job.payload, {
@@ -141,33 +192,91 @@ const runClaimedJob = async ({ claim, host, job, signal, }) => {
                 signal: executionSignal,
             });
         });
-        return recordHandlerSuccess(job._id, claim);
     }
     catch (error) {
-        return recordHandlerFailure({ claim, error, job, signal: executionSignal });
+        handlerFailed = true;
+        handlerError = error;
+    }
+    try {
+        await heartbeat.stop();
+        // Another consumer owns the row now; its result is the one that counts.
+        if (heartbeat.isLeaseLost()) {
+            return job_1.Job.findExactlyOne({ _id: job._id });
+        }
+        const currentClaim = heartbeat.getClaim();
+        if (handlerFailed) {
+            return await recordHandlerFailure({
+                claim: currentClaim,
+                error: handlerError,
+                job,
+                signal: executionSignal,
+            });
+        }
+        return await recordHandlerSuccess(job._id, currentClaim);
     }
     finally {
         (0, activeJobExecution_1.endActiveJobExecution)(job._id.toString());
     }
 };
 exports.runClaimedJob = runClaimedJob;
-const claimJobById = async ({ host, jobId, now = luxon_1.DateTime.utc().toJSDate(), }) => {
-    const lockExpiry = buildLockExpiry(host.getLockTtlMs(), now);
-    const claim = (0, claimLock_1.createClaimLock)(now);
+/**
+ * Atomically claims one due job matching `filter`. An expired `running` row is reclaimed first and
+ * the lost attempt is counted; once that exhausts `maxAttempts` the row is dead-lettered instead of
+ * run again.
+ */
+const claimDueJob = async ({ filter = {}, lockTtlMs, now = luxon_1.DateTime.utc().toJSDate(), sort, }) => {
+    const lockExpiry = buildLockExpiry(lockTtlMs, now);
+    const unlockedOrExpired = [
+        { lockedAt: { $exists: false } },
+        { lockedAt: null },
+        { lockedAt: { $lte: lockExpiry } },
+    ];
+    const queryOptions = { returnDocument: "after", ...(sort ? { sort } : {}) };
+    const reclaimLock = (0, claimLock_1.createClaimLock)(now);
+    const reclaimed = await job_1.Job.findOneAndUpdate({ ...filter, $or: unlockedOrExpired, runAt: { $lte: now }, status: "running" }, {
+        $inc: { attemptCount: 1 },
+        $push: {
+            attempts: { at: now, error: LEASE_EXPIRED_ERROR, errorClass: LEASE_EXPIRED_ERROR_CLASS },
+        },
+        $set: {
+            lastError: LEASE_EXPIRED_ERROR,
+            lockedAt: reclaimLock.lockedAt,
+            lockedBy: reclaimLock.lockedBy,
+        },
+    }, queryOptions);
+    if (reclaimed) {
+        if (reclaimed.attemptCount < reclaimed.maxAttempts) {
+            return { claim: reclaimLock, job: reclaimed, kind: "claimed" };
+        }
+        const dead = await job_1.Job.findOneAndUpdate((0, claimLock_1.buildClaimOwnershipFilter)(reclaimed._id, reclaimLock), {
+            $set: { status: "dead" },
+            $unset: { lockedAt: "", lockedBy: "" },
+        }, { returnDocument: "after" });
+        if (!dead) {
+            throw new Error(`Failed to record dead status for job ${reclaimed._id.toString()}`);
+        }
+        api_1.logger.warn(`[jobs] Job ${reclaimed._id.toString()} (${reclaimed.name}) lost its lease with no attempts left; marked dead`);
+        return { job: dead, kind: "dead" };
+    }
+    const claimLock = (0, claimLock_1.createClaimLock)(now);
     const claimed = await job_1.Job.findOneAndUpdate({
-        _id: jobId,
-        $or: [{ lockedAt: { $exists: false } }, { lockedAt: null }, { lockedAt: { $lte: lockExpiry } }],
+        ...filter,
+        $or: unlockedOrExpired,
         runAt: { $lte: now },
-        status: { $in: ["pending", "running", "scheduled"] },
+        status: { $in: ["pending", "scheduled"] },
     }, {
         $set: {
-            lockedAt: claim.lockedAt,
-            lockedBy: claim.lockedBy,
+            lockedAt: claimLock.lockedAt,
+            lockedBy: claimLock.lockedBy,
             status: "running",
         },
-    }, { returnDocument: "after" });
-    return claimed ?? undefined;
+    }, queryOptions);
+    if (!claimed) {
+        return undefined;
+    }
+    return { claim: claimLock, job: claimed, kind: "claimed" };
 };
+exports.claimDueJob = claimDueJob;
 const executeJobById = async ({ host, jobId, signal, }) => {
     if (!mongoose_1.default.isValidObjectId(jobId)) {
         return { kind: "not_found" };
@@ -187,7 +296,11 @@ const executeJobById = async ({ host, jobId, signal, }) => {
     if (existing.status === "running" && isLiveLock(existing.lockedAt, lockExpiry)) {
         return { kind: "conflict", reason: "locked" };
     }
-    const claimed = await claimJobById({ host, jobId, now });
+    const claimed = await (0, exports.claimDueJob)({
+        filter: { _id: jobId },
+        lockTtlMs: host.getLockTtlMs(),
+        now,
+    });
     if (!claimed) {
         const refreshed = await job_1.Job.findOneOrNone({ _id: jobId });
         if (!refreshed) {
@@ -198,10 +311,13 @@ const executeJobById = async ({ host, jobId, signal, }) => {
         }
         return classifyConflict(refreshed, lockExpiry, now);
     }
+    if (claimed.kind === "dead") {
+        return { job: claimed.job, kind: "executed" };
+    }
     const executed = await (0, exports.runClaimedJob)({
-        claim: readClaimLock(claimed),
+        claim: claimed.claim,
         host,
-        job: claimed,
+        job: claimed.job,
         signal,
     });
     return { job: executed, kind: "executed" };

package/dist/runners/mongoRunner.js

@@ -1,16 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MongoJobRunner = exports.hasWorkerIdPrefix = exports.getWorkerId = void 0;
-const luxon_1 = require("luxon");
-const claimLock_1 = require("../claimLock");
+const api_1 = require("@terreno/api");
 const jobExecutor_1 = require("../jobExecutor");
-const job_1 = require("../models/job");
 const DEFAULT_POLL_INTERVAL_MS = 1_000;
 /** @internal */
-var claimLock_2 = require("../claimLock");
-Object.defineProperty(exports, "getWorkerId", { enumerable: true, get: function () { return claimLock_2.getWorkerId; } });
-Object.defineProperty(exports, "hasWorkerIdPrefix", { enumerable: true, get: function () { return claimLock_2.hasWorkerIdPrefix; } });
-const buildLockExpiry = (lockTtlMs) => luxon_1.DateTime.utc().minus({ milliseconds: lockTtlMs }).toJSDate();
+var claimLock_1 = require("../claimLock");
+Object.defineProperty(exports, "getWorkerId", { enumerable: true, get: function () { return claimLock_1.getWorkerId; } });
+Object.defineProperty(exports, "hasWorkerIdPrefix", { enumerable: true, get: function () { return claimLock_1.hasWorkerIdPrefix; } });
 const waitForAbortOrTimeout = (signal, timeoutMs) => new Promise((resolve) => {
     if (signal.aborted) {
         resolve();
@@ -26,15 +23,6 @@ const waitForAbortOrTimeout = (signal, timeoutMs) => new Promise((resolve) => {
     };
     signal.addEventListener("abort", onAbort, { once: true });
 });
-const readClaimLock = (job) => {
-    if (!job.lockedAt || !job.lockedBy) {
-        throw new Error(`Job ${job._id.toString()} is missing claim lock metadata`);
-    }
-    return {
-        lockedAt: job.lockedAt,
-        lockedBy: job.lockedBy,
-    };
-};
 class MongoJobRunner {
     id = "mongo";
     pollIntervalMs = DEFAULT_POLL_INTERVAL_MS;
@@ -81,29 +69,28 @@ class MongoJobRunner {
         });
     }
     async processNext(options) {
-        const now = luxon_1.DateTime.utc().toJSDate();
-        const lockExpiry = buildLockExpiry(options.jobs.getLockTtlMs());
-        const claim = (0, claimLock_1.createClaimLock)(now);
-        const claimed = await job_1.Job.findOneAndUpdate({
-            $or: [{ lockedAt: { $exists: false } }, { lockedAt: null }, { lockedAt: { $lte: lockExpiry } }],
-            runAt: { $lte: now },
-            status: { $in: ["pending", "running", "scheduled"] },
-        }, {
-            $set: {
-                lockedAt: claim.lockedAt,
-                lockedBy: claim.lockedBy,
-                status: "running",
-            },
-        }, { returnDocument: "after", sort: { runAt: 1 } });
+        const claimed = await (0, jobExecutor_1.claimDueJob)({
+            lockTtlMs: options.jobs.getLockTtlMs(),
+            sort: { runAt: 1 },
+        });
         if (!claimed) {
             return false;
         }
-        await (0, jobExecutor_1.runClaimedJob)({
-            claim: readClaimLock(claimed),
-            host: options.jobs,
-            job: claimed,
-            signal: options.signal,
-        });
+        if (claimed.kind === "dead") {
+            return true;
+        }
+        // A failure to finalize one job (for example a lost lease) must not end the poll loop.
+        try {
+            await (0, jobExecutor_1.runClaimedJob)({
+                claim: claimed.claim,
+                host: options.jobs,
+                job: claimed.job,
+                signal: options.signal,
+            });
+        }
+        catch (error) {
+            api_1.logger.error(`[jobs] Failed to finalize job ${claimed.job._id.toString()}: ${String(error)}`);
+        }
         return true;
     }
 }

package/package.json

@@ -4,7 +4,7 @@
     "url": "https://github.com/TerrenoLabs/terreno/issues"
   },
   "dependencies": {
-    "@terreno/api": "57.16.2",
+    "@terreno/api": "57.18.0",
     "cron": "^4.3.4",
     "express": "^5.2.1",
     "luxon": "^3.7.2"
@@ -12,7 +12,7 @@
   "description": "Durable background jobs plugin for @terreno/api — part of Terreno, the batteries-included TypeScript framework for universal apps.",
   "devDependencies": {
     "@biomejs/biome": "^2.3.6",
-    "@terreno/test": "57.16.2",
+    "@terreno/test": "57.18.0",
     "@types/bun": "^1.4.0",
     "@types/express": "^5.0.6",
     "@types/luxon": "^3.7.1",
@@ -78,5 +78,5 @@
     "test:coverage": "bun run ../scripts/check-coverage.ts"
   },
   "types": "dist/index.d.ts",
-  "version": "57.16.2"
+  "version": "57.18.0"
 }

package/src/jobExecutor.ts

@@ -1,4 +1,4 @@
-import {createScopedLogger, runWithRequestContext} from "@terreno/api";
+import {createScopedLogger, logger, runWithRequestContext} from "@terreno/api";
 import {DateTime} from "luxon";
 import mongoose from "mongoose";
 
@@ -28,23 +28,20 @@ export type ExecuteJobOutcome =
 
 const TERMINAL_NOOP_STATUSES = new Set(["cancelled", "completed", "dead", "failed"]);
 
+const LEASE_EXPIRED_ERROR = "Lease expired before the previous attempt finished";
+const LEASE_EXPIRED_ERROR_CLASS = "LeaseExpired";
+
+/** Result of an atomic claim: a row to run, or a reclaimed row dead-lettered with no attempts left. */
+export type JobClaimResult =
+  | {claim: JobClaimLock; job: JobDocument; kind: "claimed"}
+  | {job: JobDocument; kind: "dead"};
+
 const buildLockExpiry = (lockTtlMs: number, now: Date): Date =>
   DateTime.fromJSDate(now, {zone: "utc"}).minus({milliseconds: lockTtlMs}).toJSDate();
 
 const isLiveLock = (lockedAt: Date | undefined, lockExpiry: Date): boolean =>
   Boolean(lockedAt && lockedAt > lockExpiry);
 
-const readClaimLock = (job: JobDocument): JobClaimLock => {
-  if (!job.lockedAt || !job.lockedBy) {
-    throw new Error(`Job ${job._id.toString()} is missing claim lock metadata`);
-  }
-
-  return {
-    lockedAt: job.lockedAt,
-    lockedBy: job.lockedBy,
-  };
-};
-
 const classifyConflict = (job: JobDocument, lockExpiry: Date, now: Date): ExecuteJobOutcome => {
   if (job.runAt > now) {
     return {kind: "conflict", reason: "not_due", runAt: job.runAt};
@@ -203,6 +200,87 @@ const recordHandlerFailure = async ({
   return pending;
 };
 
+const buildLeaseRenewalIntervalMs = (lockTtlMs: number): number =>
+  Math.max(1, Math.floor(lockTtlMs / 3));
+
+interface LeaseHeartbeat {
+  getClaim: () => JobClaimLock;
+  isLeaseLost: () => boolean;
+  stop: () => Promise<void>;
+}
+
+/**
+ * Renews the claim's lockedAt while a handler runs so no other consumer treats the live job as
+ * abandoned. Renewals run one at a time; when a renewal no longer matches the claim, the lease is
+ * lost and the handler signal is aborted.
+ */
+const startLeaseHeartbeat = ({
+  claim,
+  jobId,
+  leaseController,
+  lockTtlMs,
+}: {
+  claim: JobClaimLock;
+  jobId: JobDocument["_id"];
+  leaseController: AbortController;
+  lockTtlMs: number;
+}): LeaseHeartbeat => {
+  let currentClaim = claim;
+  let leaseLost = false;
+  let stopped = false;
+  let renewal: Promise<void> = Promise.resolve();
+
+  const renew = async (): Promise<void> => {
+    if (stopped || leaseLost) {
+      return;
+    }
+
+    const renewedAt = DateTime.utc().toJSDate();
+    const renewed = await Job.findOneAndUpdate(
+      {...buildClaimOwnershipFilter(jobId, currentClaim), status: "running"},
+      {$set: {lockedAt: renewedAt}},
+      {returnDocument: "after"}
+    );
+
+    if (!renewed) {
+      leaseLost = true;
+      clearInterval(timer);
+      logger.warn(`[jobs] Lost the claim lease for job ${jobId.toString()}; aborting handler`);
+      leaseController.abort();
+      return;
+    }
+
+    currentClaim = {lockedAt: renewedAt, lockedBy: currentClaim.lockedBy};
+  };
+
+  const timer = setInterval(() => {
+    renewal = renewal.then(renew).catch((error: unknown) => {
+      logger.warn(
+        `[jobs] Failed to renew the claim lease for job ${jobId.toString()}: ${String(error)}`
+      );
+    });
+  }, buildLeaseRenewalIntervalMs(lockTtlMs));
+
+  return {
+    getClaim: (): JobClaimLock => currentClaim,
+    isLeaseLost: (): boolean => leaseLost,
+    stop: async (): Promise<void> => {
+      stopped = true;
+      clearInterval(timer);
+      await renewal;
+    },
+  };
+};
+
+const linkAbortSignal = (parent: AbortSignal, controller: AbortController): void => {
+  if (parent.aborted) {
+    controller.abort();
+    return;
+  }
+
+  parent.addEventListener("abort", () => controller.abort(), {once: true});
+};
+
 export const runClaimedJob = async ({
   claim,
   host,
@@ -224,7 +302,18 @@ export const runClaimedJob = async ({
     labels: {jobId: job._id.toString(), jobName: job.name},
     prefix: "[Job]",
   });
-  const executionSignal = beginActiveJobExecution(job._id.toString(), signal);
+  const leaseController = new AbortController();
+  linkAbortSignal(signal, leaseController);
+  const executionSignal = beginActiveJobExecution(job._id.toString(), leaseController.signal);
+  const heartbeat = startLeaseHeartbeat({
+    claim,
+    jobId: job._id,
+    leaseController,
+    lockTtlMs: host.getLockTtlMs(),
+  });
+
+  let handlerError: unknown;
+  let handlerFailed = false;
 
   try {
     await runWithRequestContext({jobId: job._id.toString()}, async () => {
@@ -234,45 +323,123 @@ export const runClaimedJob = async ({
         signal: executionSignal,
       });
     });
-
-    return recordHandlerSuccess(job._id, claim);
   } catch (error: unknown) {
-    return recordHandlerFailure({claim, error, job, signal: executionSignal});
+    handlerFailed = true;
+    handlerError = error;
+  }
+
+  try {
+    await heartbeat.stop();
+
+    // Another consumer owns the row now; its result is the one that counts.
+    if (heartbeat.isLeaseLost()) {
+      return Job.findExactlyOne({_id: job._id});
+    }
+
+    const currentClaim = heartbeat.getClaim();
+    if (handlerFailed) {
+      return await recordHandlerFailure({
+        claim: currentClaim,
+        error: handlerError,
+        job,
+        signal: executionSignal,
+      });
+    }
+
+    return await recordHandlerSuccess(job._id, currentClaim);
   } finally {
     endActiveJobExecution(job._id.toString());
   }
 };
 
-const claimJobById = async ({
-  host,
-  jobId,
+/**
+ * Atomically claims one due job matching `filter`. An expired `running` row is reclaimed first and
+ * the lost attempt is counted; once that exhausts `maxAttempts` the row is dead-lettered instead of
+ * run again.
+ */
+export const claimDueJob = async ({
+  filter = {},
+  lockTtlMs,
   now = DateTime.utc().toJSDate(),
+  sort,
 }: {
-  host: JobExecutionHost;
-  jobId: string;
+  filter?: Record<string, unknown>;
+  lockTtlMs: number;
   now?: Date;
-}): Promise<JobDocument | undefined> => {
-  const lockExpiry = buildLockExpiry(host.getLockTtlMs(), now);
-  const claim = createClaimLock(now);
+  sort?: Record<string, 1 | -1>;
+}): Promise<JobClaimResult | undefined> => {
+  const lockExpiry = buildLockExpiry(lockTtlMs, now);
+  const unlockedOrExpired = [
+    {lockedAt: {$exists: false}},
+    {lockedAt: null},
+    {lockedAt: {$lte: lockExpiry}},
+  ];
+  const queryOptions = {returnDocument: "after" as const, ...(sort ? {sort} : {})};
+
+  const reclaimLock = createClaimLock(now);
+  const reclaimed = await Job.findOneAndUpdate(
+    {...filter, $or: unlockedOrExpired, runAt: {$lte: now}, status: "running"},
+    {
+      $inc: {attemptCount: 1},
+      $push: {
+        attempts: {at: now, error: LEASE_EXPIRED_ERROR, errorClass: LEASE_EXPIRED_ERROR_CLASS},
+      },
+      $set: {
+        lastError: LEASE_EXPIRED_ERROR,
+        lockedAt: reclaimLock.lockedAt,
+        lockedBy: reclaimLock.lockedBy,
+      },
+    },
+    queryOptions
+  );
+
+  if (reclaimed) {
+    if (reclaimed.attemptCount < reclaimed.maxAttempts) {
+      return {claim: reclaimLock, job: reclaimed, kind: "claimed"};
+    }
 
+    const dead = await Job.findOneAndUpdate(
+      buildClaimOwnershipFilter(reclaimed._id, reclaimLock),
+      {
+        $set: {status: "dead"},
+        $unset: {lockedAt: "", lockedBy: ""},
+      },
+      {returnDocument: "after"}
+    );
+
+    if (!dead) {
+      throw new Error(`Failed to record dead status for job ${reclaimed._id.toString()}`);
+    }
+
+    logger.warn(
+      `[jobs] Job ${reclaimed._id.toString()} (${reclaimed.name}) lost its lease with no attempts left; marked dead`
+    );
+    return {job: dead, kind: "dead"};
+  }
+
+  const claimLock = createClaimLock(now);
   const claimed = await Job.findOneAndUpdate(
     {
-      _id: jobId,
-      $or: [{lockedAt: {$exists: false}}, {lockedAt: null}, {lockedAt: {$lte: lockExpiry}}],
+      ...filter,
+      $or: unlockedOrExpired,
       runAt: {$lte: now},
-      status: {$in: ["pending", "running", "scheduled"]},
+      status: {$in: ["pending", "scheduled"]},
     },
     {
       $set: {
-        lockedAt: claim.lockedAt,
-        lockedBy: claim.lockedBy,
+        lockedAt: claimLock.lockedAt,
+        lockedBy: claimLock.lockedBy,
         status: "running",
       },
     },
-    {returnDocument: "after"}
+    queryOptions
   );
 
-  return claimed ?? undefined;
+  if (!claimed) {
+    return undefined;
+  }
+
+  return {claim: claimLock, job: claimed, kind: "claimed"};
 };
 
 export const executeJobById = async ({
@@ -308,7 +475,11 @@ export const executeJobById = async ({
     return {kind: "conflict", reason: "locked"};
   }
 
-  const claimed = await claimJobById({host, jobId, now});
+  const claimed = await claimDueJob({
+    filter: {_id: jobId},
+    lockTtlMs: host.getLockTtlMs(),
+    now,
+  });
   if (!claimed) {
     const refreshed = await Job.findOneOrNone({_id: jobId});
     if (!refreshed) {
@@ -322,10 +493,14 @@ export const executeJobById = async ({
     return classifyConflict(refreshed, lockExpiry, now);
   }
 
+  if (claimed.kind === "dead") {
+    return {job: claimed.job, kind: "executed"};
+  }
+
   const executed = await runClaimedJob({
-    claim: readClaimLock(claimed),
+    claim: claimed.claim,
     host,
-    job: claimed,
+    job: claimed.job,
     signal,
   });
 

package/src/runners/mongoRunner.ts

@@ -1,8 +1,6 @@
-import {DateTime} from "luxon";
+import {logger} from "@terreno/api";
 
-import {createClaimLock, type JobClaimLock} from "../claimLock";
-import {runClaimedJob} from "../jobExecutor";
-import {Job} from "../models/job";
+import {claimDueJob, runClaimedJob} from "../jobExecutor";
 import type {JobDocument} from "../modelTypes";
 import type {JobRunner, JobRunnerStartOptions} from "../types";
 
@@ -11,9 +9,6 @@ const DEFAULT_POLL_INTERVAL_MS = 1_000;
 /** @internal */
 export {getWorkerId, hasWorkerIdPrefix} from "../claimLock";
 
-const buildLockExpiry = (lockTtlMs: number): Date =>
-  DateTime.utc().minus({milliseconds: lockTtlMs}).toJSDate();
-
 const waitForAbortOrTimeout = (signal: AbortSignal, timeoutMs: number): Promise<void> =>
   new Promise((resolve) => {
     if (signal.aborted) {
@@ -34,17 +29,6 @@ const waitForAbortOrTimeout = (signal: AbortSignal, timeoutMs: number): Promise<
     signal.addEventListener("abort", onAbort, {once: true});
   });
 
-const readClaimLock = (job: JobDocument): JobClaimLock => {
-  if (!job.lockedAt || !job.lockedBy) {
-    throw new Error(`Job ${job._id.toString()} is missing claim lock metadata`);
-  }
-
-  return {
-    lockedAt: job.lockedAt,
-    lockedBy: job.lockedBy,
-  };
-};
-
 export class MongoJobRunner implements JobRunner {
   readonly id = "mongo";
   private pollIntervalMs = DEFAULT_POLL_INTERVAL_MS;
@@ -100,36 +84,30 @@ export class MongoJobRunner implements JobRunner {
   }
 
   private async processNext(options: JobRunnerStartOptions): Promise<boolean> {
-    const now = DateTime.utc().toJSDate();
-    const lockExpiry = buildLockExpiry(options.jobs.getLockTtlMs());
-    const claim = createClaimLock(now);
-
-    const claimed = await Job.findOneAndUpdate(
-      {
-        $or: [{lockedAt: {$exists: false}}, {lockedAt: null}, {lockedAt: {$lte: lockExpiry}}],
-        runAt: {$lte: now},
-        status: {$in: ["pending", "running", "scheduled"]},
-      },
-      {
-        $set: {
-          lockedAt: claim.lockedAt,
-          lockedBy: claim.lockedBy,
-          status: "running",
-        },
-      },
-      {returnDocument: "after", sort: {runAt: 1}}
-    );
+    const claimed = await claimDueJob({
+      lockTtlMs: options.jobs.getLockTtlMs(),
+      sort: {runAt: 1},
+    });
 
     if (!claimed) {
       return false;
     }
 
-    await runClaimedJob({
-      claim: readClaimLock(claimed),
-      host: options.jobs,
-      job: claimed,
-      signal: options.signal,
-    });
+    if (claimed.kind === "dead") {
+      return true;
+    }
+
+    // A failure to finalize one job (for example a lost lease) must not end the poll loop.
+    try {
+      await runClaimedJob({
+        claim: claimed.claim,
+        host: options.jobs,
+        job: claimed.job,
+        signal: options.signal,
+      });
+    } catch (error: unknown) {
+      logger.error(`[jobs] Failed to finalize job ${claimed.job._id.toString()}: ${String(error)}`);
+    }
     return true;
   }
 }

package/src/tests/jobsLease.test.ts

@@ -0,0 +1,266 @@
+import {beforeEach, describe, it} from "bun:test";
+import {TerrenoApp, type UserModel as UserModelType} from "@terreno/api";
+import {setupDb, UserModel} from "@terreno/api/testing";
+import {assert} from "chai";
+import {DateTime} from "luxon";
+
+import {JobsApp} from "../jobsApp";
+import {getJobsService} from "../jobsService";
+import {Job} from "../models/job";
+import {JobSchedule} from "../models/jobSchedule";
+
+const typedUserModel = UserModel as unknown as UserModelType;
+
+const waitUntil = async (
+  predicate: () => Promise<boolean>,
+  options?: {intervalMs?: number; timeoutMs?: number}
+): Promise<void> => {
+  const intervalMs = options?.intervalMs ?? 10;
+  const timeoutMs = options?.timeoutMs ?? 5_000;
+  const deadline = DateTime.utc().plus({milliseconds: timeoutMs});
+
+  while (DateTime.utc() < deadline) {
+    if (await predicate()) {
+      return;
+    }
+    await Bun.sleep(intervalMs);
+  }
+
+  throw new Error("waitUntil timed out");
+};
+
+const registerJobsApp = (jobsApp: JobsApp): void => {
+  new TerrenoApp({
+    skipListen: true,
+    userModel: typedUserModel,
+  })
+    .register(jobsApp)
+    .build();
+};
+
+const createStaleRunningJob = async ({
+  maxAttempts,
+  name,
+}: {
+  maxAttempts: number;
+  name: string;
+}): Promise<string> => {
+  const stale = await Job.create({
+    attemptCount: 0,
+    lockedAt: DateTime.utc().minus({seconds: 5}).toJSDate(),
+    lockedBy: "crashed-worker:1:stale-token",
+    maxAttempts,
+    name,
+    payload: {},
+    payloadRedacted: false,
+    runAt: DateTime.utc().minus({seconds: 5}).toJSDate(),
+    status: "running",
+  });
+  return stale._id.toString();
+};
+
+describe("jobs claim lease", () => {
+  beforeEach(async (): Promise<void> => {
+    await setupDb();
+    await Job.deleteMany({});
+    await JobSchedule.deleteMany({});
+  });
+
+  it("keeps the lease of a live handler so a later execute dispatch is rejected as locked", async (): Promise<void> => {
+    let invocations = 0;
+    let concurrent = 0;
+    let maxConcurrent = 0;
+
+    const jobsApp = new JobsApp({lockTtlMs: 100});
+    jobsApp.define("long-once", {
+      handler: async () => {
+        invocations += 1;
+        concurrent += 1;
+        maxConcurrent = Math.max(maxConcurrent, concurrent);
+        await Bun.sleep(400);
+        concurrent -= 1;
+      },
+      retry: {maxAttempts: 1},
+    });
+    registerJobsApp(jobsApp);
+
+    const enqueued = await getJobsService().enqueue({name: "long-once", payload: {}});
+    const jobId = enqueued._id.toString();
+
+    const first = jobsApp.executeQueuedJob(jobId);
+    await Bun.sleep(30);
+    const duplicateWithinTtl = await jobsApp.executeQueuedJob(jobId);
+    await Bun.sleep(220);
+    const duplicateAfterTtl = await jobsApp.executeQueuedJob(jobId);
+    const firstOutcome = await first;
+
+    assert.equal(duplicateWithinTtl.kind, "conflict");
+    assert.equal(duplicateAfterTtl.kind, "conflict");
+    assert.equal(firstOutcome.kind, "executed");
+    assert.equal(invocations, 1);
+    assert.equal(maxConcurrent, 1);
+
+    const row = await Job.findExactlyOne({_id: jobId});
+    assert.equal(row.status, "completed");
+    assert.equal(row.attemptCount, 0);
+  });
+
+  it("runs a long handler once across two poll workers and keeps both workers active", async (): Promise<void> => {
+    let invocations = 0;
+    let concurrent = 0;
+    let maxConcurrent = 0;
+    const handler = async (): Promise<void> => {
+      invocations += 1;
+      concurrent += 1;
+      maxConcurrent = Math.max(maxConcurrent, concurrent);
+      await Bun.sleep(600);
+      concurrent -= 1;
+    };
+
+    const workerA = new JobsApp({lockTtlMs: 150, pollIntervalMs: 20});
+    const workerB = new JobsApp({lockTtlMs: 150, pollIntervalMs: 20});
+    workerA.define("long-poll-once", {handler, retry: {maxAttempts: 1}});
+    workerB.define("long-poll-once", {handler, retry: {maxAttempts: 1}});
+    registerJobsApp(workerA);
+
+    const enqueued = await getJobsService().enqueue({name: "long-poll-once", payload: {}});
+
+    try {
+      await Promise.all([workerA.startWorker(), workerB.startWorker()]);
+      await waitUntil(
+        async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "completed"
+      );
+      await Bun.sleep(200);
+
+      assert.equal(invocations, 1);
+      assert.equal(maxConcurrent, 1);
+      assert.isTrue(workerA.isWorkerActive());
+      assert.isTrue(workerB.isWorkerActive());
+
+      const row = await Job.findExactlyOne({_id: enqueued._id});
+      assert.equal(row.status, "completed");
+      assert.equal(row.attemptCount, 0);
+    } finally {
+      await Promise.all([workerA.stopWorker(), workerB.stopWorker()]);
+    }
+  });
+
+  it("dead-letters an expired running job on execute once its attempts are exhausted", async (): Promise<void> => {
+    let invocations = 0;
+    const jobsApp = new JobsApp({lockTtlMs: 100});
+    jobsApp.define("crashed-once", {
+      handler: async () => {
+        invocations += 1;
+      },
+      retry: {maxAttempts: 1},
+    });
+    registerJobsApp(jobsApp);
+
+    const jobId = await createStaleRunningJob({maxAttempts: 1, name: "crashed-once"});
+    const outcome = await jobsApp.executeQueuedJob(jobId);
+
+    assert.equal(outcome.kind, "executed");
+    assert.equal(invocations, 0);
+    const row = await Job.findExactlyOne({_id: jobId});
+    assert.equal(row.status, "dead");
+    assert.equal(row.attemptCount, 1);
+    assert.lengthOf(row.attempts, 1);
+    assert.isUndefined(row.lockedAt);
+    assert.isUndefined(row.lockedBy);
+  });
+
+  it("dead-letters an expired running job in the poll loop once its attempts are exhausted", async (): Promise<void> => {
+    let invocations = 0;
+    const jobsApp = new JobsApp({lockTtlMs: 100, pollIntervalMs: 20});
+    jobsApp.define("crashed-poll-once", {
+      handler: async () => {
+        invocations += 1;
+      },
+      retry: {maxAttempts: 1},
+    });
+    registerJobsApp(jobsApp);
+
+    const jobId = await createStaleRunningJob({maxAttempts: 1, name: "crashed-poll-once"});
+
+    try {
+      await jobsApp.startWorker();
+      await waitUntil(async () => (await Job.findExactlyOne({_id: jobId})).status === "dead");
+    } finally {
+      await jobsApp.stopWorker();
+    }
+
+    assert.equal(invocations, 0);
+    const row = await Job.findExactlyOne({_id: jobId});
+    assert.equal(row.attemptCount, 1);
+  });
+
+  it("counts a reclaim as an attempt and still runs the job when attempts remain", async (): Promise<void> => {
+    let invocations = 0;
+    const jobsApp = new JobsApp({lockTtlMs: 100});
+    jobsApp.define("crashed-retryable", {
+      handler: async () => {
+        invocations += 1;
+      },
+      retry: {maxAttempts: 3},
+    });
+    registerJobsApp(jobsApp);
+
+    const jobId = await createStaleRunningJob({maxAttempts: 3, name: "crashed-retryable"});
+    const outcome = await jobsApp.executeQueuedJob(jobId);
+
+    assert.equal(outcome.kind, "executed");
+    assert.equal(invocations, 1);
+    const row = await Job.findExactlyOne({_id: jobId});
+    assert.equal(row.status, "completed");
+    assert.equal(row.attemptCount, 1);
+  });
+
+  it("aborts a handler that loses its lease and keeps the poll loop alive", async (): Promise<void> => {
+    let sawAbort = false;
+    let handlerStarted = false;
+
+    const jobsApp = new JobsApp({lockTtlMs: 90, pollIntervalMs: 20});
+    jobsApp.define("lease-lost", {
+      handler: async (_payload, ctx) => {
+        handlerStarted = true;
+        await waitUntil(() => Promise.resolve(ctx.signal.aborted), {timeoutMs: 2_000});
+        sawAbort = true;
+      },
+      retry: {maxAttempts: 1},
+    });
+    jobsApp.define("after-lease-lost", {
+      handler: async () => {},
+    });
+    registerJobsApp(jobsApp);
+
+    const enqueued = await getJobsService().enqueue({name: "lease-lost", payload: {}});
+
+    try {
+      await jobsApp.startWorker();
+      await waitUntil(() => Promise.resolve(handlerStarted));
+      // Another consumer takes the row over (for example after this worker stalled past the TTL).
+      await Job.updateOne(
+        {_id: enqueued._id},
+        {
+          $set: {
+            lockedAt: DateTime.utc().plus({minutes: 5}).toJSDate(),
+            lockedBy: "other-worker:2:other-token",
+          },
+        }
+      );
+      await waitUntil(() => Promise.resolve(sawAbort));
+
+      const followUp = await getJobsService().enqueue({name: "after-lease-lost", payload: {}});
+      await waitUntil(
+        async () => (await Job.findExactlyOne({_id: followUp._id})).status === "completed"
+      );
+
+      assert.isTrue(jobsApp.isWorkerActive());
+      const row = await Job.findExactlyOne({_id: enqueued._id});
+      assert.equal(row.status, "running");
+      assert.equal(row.lockedBy, "other-worker:2:other-token");
+    } finally {
+      await jobsApp.stopWorker();
+    }
+  });
+});

package/src/tests/jobsRetry.test.ts

@@ -364,24 +364,36 @@ describe("jobs retry and dead-letter", () => {
       payload: {},
     });
 
-    await staleWorker.startWorker();
-    await waitUntil(() => Promise.resolve(staleHandlerRunning));
-
-    await reclaimWorker.startWorker();
-    await waitUntil(
-      async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "completed"
-    );
-
-    releaseStaleHandler();
-    await Bun.sleep(150);
-
-    const finalRow = await Job.findExactlyOne({_id: enqueued._id});
-    assert.equal(finalRow.status, "completed");
-    assert.equal(finalRow.attemptCount, 0);
-    assert.lengthOf(finalRow.attempts, 0);
-    assert.isTrue(hasWorkerIdPrefix(finalRow.lockedBy));
-
-    await Promise.all([staleWorker.stopWorker(), reclaimWorker.stopWorker()]);
+    try {
+      await staleWorker.startWorker();
+      await waitUntil(() => Promise.resolve(staleHandlerRunning));
+
+      // The stale worker's lease renewals stop landing (for example a stalled process), so its
+      // lock ages past the TTL and another worker reclaims the row.
+      await Job.updateOne(
+        {_id: enqueued._id},
+        {$set: {lockedAt: DateTime.utc().minus({seconds: 5}).toJSDate()}}
+      );
+
+      await reclaimWorker.startWorker();
+      await waitUntil(
+        async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "completed"
+      );
+
+      releaseStaleHandler();
+      await Bun.sleep(150);
+
+      const finalRow = await Job.findExactlyOne({_id: enqueued._id});
+      assert.equal(finalRow.status, "completed");
+      assert.equal(finalRow.attemptCount, 1);
+      assert.lengthOf(finalRow.attempts, 1);
+      assert.equal(finalRow.attempts[0].errorClass, "LeaseExpired");
+      assert.isTrue(hasWorkerIdPrefix(finalRow.lockedBy));
+      assert.isTrue(staleWorker.isWorkerActive());
+    } finally {
+      releaseStaleHandler();
+      await Promise.all([staleWorker.stopWorker(), reclaimWorker.stopWorker()]);
+    }
   });
 
   it("keeps the worker running after a retryable handler failure", async (): Promise<void> => {