@terreno/jobs 57.4.0

package/dist/jobExecutor.js

@@ -0,0 +1,328 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeJobById = exports.runClaimedJob = void 0;
+var api_1 = require("@terreno/api");
+var luxon_1 = require("luxon");
+var mongoose_1 = __importDefault(require("mongoose"));
+var activeJobExecution_1 = require("./activeJobExecution");
+var claimLock_1 = require("./claimLock");
+var job_1 = require("./models/job");
+var retryBackoff_1 = require("./retryBackoff");
+var TERMINAL_NOOP_STATUSES = new Set(["cancelled", "completed", "dead", "failed"]);
+var buildLockExpiry = function (lockTtlMs, now) {
+    return luxon_1.DateTime.fromJSDate(now, { zone: "utc" }).minus({ milliseconds: lockTtlMs }).toJSDate();
+};
+var isLiveLock = function (lockedAt, lockExpiry) {
+    return Boolean(lockedAt && lockedAt > lockExpiry);
+};
+var readClaimLock = function (job) {
+    if (!job.lockedAt || !job.lockedBy) {
+        throw new Error("Job ".concat(job._id.toString(), " is missing claim lock metadata"));
+    }
+    return {
+        lockedAt: job.lockedAt,
+        lockedBy: job.lockedBy,
+    };
+};
+var classifyConflict = function (job, lockExpiry, now) {
+    if (job.runAt > now) {
+        return { kind: "conflict", reason: "not_due", runAt: job.runAt };
+    }
+    if (job.status === "running" && isLiveLock(job.lockedAt, lockExpiry)) {
+        return { kind: "conflict", reason: "locked" };
+    }
+    return { kind: "conflict", reason: "locked" };
+};
+var recordHandlerSuccess = function (jobId, claim) { return __awaiter(void 0, void 0, void 0, function () {
+    var updated, current;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0: return [4 /*yield*/, job_1.Job.findOneAndUpdate(__assign(__assign({}, (0, claimLock_1.buildClaimOwnershipFilter)(jobId, claim)), { status: "running" }), {
+                    $set: { status: "completed" },
+                    $unset: { lastError: "" },
+                }, { returnDocument: "after" })];
+            case 1:
+                updated = _a.sent();
+                if (!!updated) return [3 /*break*/, 3];
+                return [4 /*yield*/, job_1.Job.findOneOrNone({ _id: jobId })];
+            case 2:
+                current = _a.sent();
+                if ((current === null || current === void 0 ? void 0 : current.status) === "cancelled") {
+                    return [2 /*return*/, current];
+                }
+                throw new Error("Failed to record success for job ".concat(jobId.toString()));
+            case 3: return [2 /*return*/, updated];
+        }
+    });
+}); };
+var recordMissingHandlerFailure = function (job, message, claim) { return __awaiter(void 0, void 0, void 0, function () {
+    var updated;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0: return [4 /*yield*/, job_1.Job.findOneAndUpdate((0, claimLock_1.buildClaimOwnershipFilter)(job._id, claim), {
+                    $set: {
+                        lastError: message,
+                        status: "failed",
+                    },
+                }, { returnDocument: "after" })];
+            case 1:
+                updated = _a.sent();
+                if (!updated) {
+                    throw new Error("Failed to record missing handler for job ".concat(job._id.toString()));
+                }
+                return [2 /*return*/, updated];
+        }
+    });
+}); };
+var recordHandlerFailure = function (_a) { return __awaiter(void 0, [_a], void 0, function (_b) {
+    var current, released, errorMessage, errorClass, attemptedAt, attemptEntry, nextAttemptCount, ownershipFilter, dead, runAt, pending;
+    var claim = _b.claim, error = _b.error, job = _b.job, signal = _b.signal;
+    return __generator(this, function (_c) {
+        switch (_c.label) {
+            case 0:
+                if (!(signal.aborted && (0, claimLock_1.isAbortError)(error))) return [3 /*break*/, 3];
+                return [4 /*yield*/, job_1.Job.findOneOrNone({ _id: job._id })];
+            case 1:
+                current = _c.sent();
+                if ((current === null || current === void 0 ? void 0 : current.status) === "cancelled") {
+                    return [2 /*return*/, current];
+                }
+                return [4 /*yield*/, job_1.Job.findOneAndUpdate((0, claimLock_1.buildClaimOwnershipFilter)(job._id, claim), {
+                        $set: { status: "pending" },
+                        $unset: { lockedAt: "", lockedBy: "" },
+                    }, { returnDocument: "after" })];
+            case 2:
+                released = _c.sent();
+                if (!released) {
+                    throw new Error("Failed to release lock for job ".concat(job._id.toString()));
+                }
+                return [2 /*return*/, released];
+            case 3:
+                errorMessage = error instanceof Error ? error.message : String(error);
+                errorClass = error instanceof Error ? error.name : "Error";
+                attemptedAt = luxon_1.DateTime.utc().toJSDate();
+                attemptEntry = {
+                    at: attemptedAt,
+                    error: errorMessage,
+                    errorClass: errorClass,
+                };
+                nextAttemptCount = job.attemptCount + 1;
+                ownershipFilter = (0, claimLock_1.buildClaimOwnershipFilter)(job._id, claim);
+                if (!(nextAttemptCount >= job.maxAttempts)) return [3 /*break*/, 5];
+                return [4 /*yield*/, job_1.Job.findOneAndUpdate(ownershipFilter, {
+                        $push: { attempts: attemptEntry },
+                        $set: {
+                            attemptCount: nextAttemptCount,
+                            lastError: errorMessage,
+                            status: "dead",
+                        },
+                        $unset: { lockedAt: "", lockedBy: "" },
+                    }, { returnDocument: "after" })];
+            case 4:
+                dead = _c.sent();
+                if (!dead) {
+                    throw new Error("Failed to record dead status for job ".concat(job._id.toString()));
+                }
+                return [2 /*return*/, dead];
+            case 5:
+                runAt = (0, retryBackoff_1.computeRetryRunAt)({
+                    attemptCount: nextAttemptCount,
+                    backoffMs: job.backoffMs,
+                    from: luxon_1.DateTime.utc(),
+                    maxBackoffMs: job.maxBackoffMs,
+                }).toJSDate();
+                return [4 /*yield*/, job_1.Job.findOneAndUpdate(ownershipFilter, {
+                        $push: { attempts: attemptEntry },
+                        $set: {
+                            attemptCount: nextAttemptCount,
+                            lastError: errorMessage,
+                            runAt: runAt,
+                            status: "pending",
+                        },
+                        $unset: { lockedAt: "", lockedBy: "" },
+                    }, { returnDocument: "after" })];
+            case 6:
+                pending = _c.sent();
+                if (!pending) {
+                    throw new Error("Failed to record retry for job ".concat(job._id.toString()));
+                }
+                return [2 /*return*/, pending];
+        }
+    });
+}); };
+var runClaimedJob = function (_a) { return __awaiter(void 0, [_a], void 0, function (_b) {
+    var definition, message, log, executionSignal, error_1;
+    var claim = _b.claim, host = _b.host, job = _b.job, signal = _b.signal;
+    return __generator(this, function (_c) {
+        switch (_c.label) {
+            case 0:
+                definition = host.getDefinition(job.name);
+                if (!definition) {
+                    message = "No handler registered for job name \"".concat(job.name, "\"");
+                    return [2 /*return*/, recordMissingHandlerFailure(job, message, claim)];
+                }
+                log = (0, api_1.createScopedLogger)({
+                    labels: { jobId: job._id.toString(), jobName: job.name },
+                    prefix: "[Job]",
+                });
+                executionSignal = (0, activeJobExecution_1.beginActiveJobExecution)(job._id.toString(), signal);
+                _c.label = 1;
+            case 1:
+                _c.trys.push([1, 3, 4, 5]);
+                return [4 /*yield*/, (0, api_1.runWithRequestContext)({ jobId: job._id.toString() }, function () { return __awaiter(void 0, void 0, void 0, function () {
+                        return __generator(this, function (_a) {
+                            switch (_a.label) {
+                                case 0: return [4 /*yield*/, definition.handler(job.payload, {
+                                        jobId: job._id.toString(),
+                                        log: log,
+                                        signal: executionSignal,
+                                    })];
+                                case 1:
+                                    _a.sent();
+                                    return [2 /*return*/];
+                            }
+                        });
+                    }); })];
+            case 2:
+                _c.sent();
+                return [2 /*return*/, recordHandlerSuccess(job._id, claim)];
+            case 3:
+                error_1 = _c.sent();
+                return [2 /*return*/, recordHandlerFailure({ claim: claim, error: error_1, job: job, signal: executionSignal })];
+            case 4:
+                (0, activeJobExecution_1.endActiveJobExecution)(job._id.toString());
+                return [7 /*endfinally*/];
+            case 5: return [2 /*return*/];
+        }
+    });
+}); };
+exports.runClaimedJob = runClaimedJob;
+var claimJobById = function (_a) { return __awaiter(void 0, [_a], void 0, function (_b) {
+    var lockExpiry, claim, claimed;
+    var host = _b.host, jobId = _b.jobId, _c = _b.now, now = _c === void 0 ? luxon_1.DateTime.utc().toJSDate() : _c;
+    return __generator(this, function (_d) {
+        switch (_d.label) {
+            case 0:
+                lockExpiry = buildLockExpiry(host.getLockTtlMs(), now);
+                claim = (0, claimLock_1.createClaimLock)(now);
+                return [4 /*yield*/, job_1.Job.findOneAndUpdate({
+                        _id: jobId,
+                        $or: [{ lockedAt: { $exists: false } }, { lockedAt: null }, { lockedAt: { $lte: lockExpiry } }],
+                        runAt: { $lte: now },
+                        status: { $in: ["pending", "running", "scheduled"] },
+                    }, {
+                        $set: {
+                            lockedAt: claim.lockedAt,
+                            lockedBy: claim.lockedBy,
+                            status: "running",
+                        },
+                    }, { returnDocument: "after" })];
+            case 1:
+                claimed = _d.sent();
+                return [2 /*return*/, claimed !== null && claimed !== void 0 ? claimed : undefined];
+        }
+    });
+}); };
+var executeJobById = function (_a) { return __awaiter(void 0, [_a], void 0, function (_b) {
+    var existing, now, lockExpiry, claimed, refreshed, executed;
+    var host = _b.host, jobId = _b.jobId, signal = _b.signal;
+    return __generator(this, function (_c) {
+        switch (_c.label) {
+            case 0:
+                if (!mongoose_1.default.isValidObjectId(jobId)) {
+                    return [2 /*return*/, { kind: "not_found" }];
+                }
+                return [4 /*yield*/, job_1.Job.findOneOrNone({ _id: jobId })];
+            case 1:
+                existing = _c.sent();
+                if (!existing) {
+                    return [2 /*return*/, { kind: "not_found" }];
+                }
+                now = luxon_1.DateTime.utc().toJSDate();
+                lockExpiry = buildLockExpiry(host.getLockTtlMs(), now);
+                if (TERMINAL_NOOP_STATUSES.has(existing.status)) {
+                    return [2 /*return*/, { job: existing, kind: "noop" }];
+                }
+                if (existing.runAt > now) {
+                    return [2 /*return*/, { kind: "conflict", reason: "not_due", runAt: existing.runAt }];
+                }
+                if (existing.status === "running" && isLiveLock(existing.lockedAt, lockExpiry)) {
+                    return [2 /*return*/, { kind: "conflict", reason: "locked" }];
+                }
+                return [4 /*yield*/, claimJobById({ host: host, jobId: jobId, now: now })];
+            case 2:
+                claimed = _c.sent();
+                if (!!claimed) return [3 /*break*/, 4];
+                return [4 /*yield*/, job_1.Job.findOneOrNone({ _id: jobId })];
+            case 3:
+                refreshed = _c.sent();
+                if (!refreshed) {
+                    return [2 /*return*/, { kind: "not_found" }];
+                }
+                if (TERMINAL_NOOP_STATUSES.has(refreshed.status)) {
+                    return [2 /*return*/, { job: refreshed, kind: "noop" }];
+                }
+                return [2 /*return*/, classifyConflict(refreshed, lockExpiry, now)];
+            case 4: return [4 /*yield*/, (0, exports.runClaimedJob)({
+                    claim: readClaimLock(claimed),
+                    host: host,
+                    job: claimed,
+                    signal: signal,
+                })];
+            case 5:
+                executed = _c.sent();
+                return [2 /*return*/, { job: executed, kind: "executed" }];
+        }
+    });
+}); };
+exports.executeJobById = executeJobById;

package/dist/jobsApp.d.ts

@@ -0,0 +1,61 @@
+import { type AdminContribution, type AnyTerrenoAccess, type TerrenoPlugin } from "@terreno/api";
+import type express from "express";
+import { type ExecuteJobOutcome } from "./jobExecutor";
+import { type ExecuteAuthVerifier } from "./routes/jobsExecute";
+import type { JobDefinition, JobRunner, JobsAdminRedactPayload, JobsRunnerHost } from "./types";
+export interface JobsAppOptions {
+    accessControl?: AnyTerrenoAccess;
+    basePath?: string;
+    /** Verifies inbound execute HTTP requests (required when the execute route is mounted). */
+    executeAuth?: ExecuteAuthVerifier;
+    lockTtlMs?: number;
+    /** Mount internal `POST {basePath}/execute` for cloud/custom dispatch. */
+    mountExecuteRoute?: boolean;
+    pollIntervalMs?: number;
+    /** Admin list/detail payload projection. Default omits payload; identity hook may expose raw payload. */
+    redactPayload?: JobsAdminRedactPayload;
+    runner?: JobRunner;
+    timezone?: string;
+}
+/**
+ * TerrenoPlugin for durable background jobs.
+ *
+ * Registering `JobsApp` wires the plugin into the Express app but does **not**
+ * start a worker poll loop. Call {@link JobsApp.startWorker} explicitly from the
+ * API process or a dedicated worker entrypoint.
+ */
+export declare class JobsApp implements JobsRunnerHost, TerrenoPlugin {
+    private readonly options;
+    private readonly runner;
+    private readonly service;
+    private abortController;
+    private workerActive;
+    private workerPromise;
+    private workerStartPromise;
+    constructor(options?: JobsAppOptions);
+    define(name: string, definition: JobDefinition): this;
+    getBasePath(): string;
+    getDefinition(name: string): JobDefinition | undefined;
+    getDefaultTimezone(): string;
+    getLockTtlMs(): number;
+    /** Execute a persisted job by id from an external dispatch surface (queue consumer, execute HTTP). */
+    executeQueuedJob(jobId: string, signal?: AbortSignal): Promise<ExecuteJobOutcome>;
+    getPollIntervalMs(): number;
+    reconcileSchedules(): Promise<void>;
+    reconcileSchedulesIfDirty(): Promise<void>;
+    tickSchedules(now?: Date): Promise<void>;
+    register(app: express.Application, openApi?: unknown): void;
+    adminContribution(): AdminContribution;
+    private shouldMountExecuteRoute;
+    isWorkerActive(): boolean;
+    /**
+     * Starts the background worker poll loop without awaiting it.
+     *
+     * Fire-and-forget: resolves once the runner has been scheduled. Runner errors are
+     * logged and worker state is reset; they are not thrown from this method.
+     */
+    startWorker(): Promise<void>;
+    stopWorker(): Promise<void>;
+    private resetWorkerState;
+    private beginWorkerStart;
+}