@terreno/jobs 57.4.0

package/src/jobExecutor.ts

@@ -0,0 +1,333 @@
+import {createScopedLogger, runWithRequestContext} from "@terreno/api";
+import {DateTime} from "luxon";
+import mongoose from "mongoose";
+
+import {beginActiveJobExecution, endActiveJobExecution} from "./activeJobExecution";
+import {
+  buildClaimOwnershipFilter,
+  createClaimLock,
+  isAbortError,
+  type JobClaimLock,
+} from "./claimLock";
+import {Job} from "./models/job";
+import type {JobAttempt, JobDocument} from "./modelTypes";
+import {computeRetryRunAt} from "./retryBackoff";
+import type {JobDefinition} from "./types";
+
+export interface JobExecutionHost {
+  getDefinition(name: string): JobDefinition | undefined;
+  getLockTtlMs(): number;
+}
+
+export type ExecuteJobOutcome =
+  | {job: JobDocument; kind: "executed"}
+  | {job: JobDocument; kind: "noop"}
+  | {kind: "conflict"; reason: "locked"}
+  | {kind: "conflict"; reason: "not_due"; runAt: Date}
+  | {kind: "not_found"};
+
+const TERMINAL_NOOP_STATUSES = new Set(["cancelled", "completed", "dead", "failed"]);
+
+const buildLockExpiry = (lockTtlMs: number, now: Date): Date =>
+  DateTime.fromJSDate(now, {zone: "utc"}).minus({milliseconds: lockTtlMs}).toJSDate();
+
+const isLiveLock = (lockedAt: Date | undefined, lockExpiry: Date): boolean =>
+  Boolean(lockedAt && lockedAt > lockExpiry);
+
+const readClaimLock = (job: JobDocument): JobClaimLock => {
+  if (!job.lockedAt || !job.lockedBy) {
+    throw new Error(`Job ${job._id.toString()} is missing claim lock metadata`);
+  }
+
+  return {
+    lockedAt: job.lockedAt,
+    lockedBy: job.lockedBy,
+  };
+};
+
+const classifyConflict = (job: JobDocument, lockExpiry: Date, now: Date): ExecuteJobOutcome => {
+  if (job.runAt > now) {
+    return {kind: "conflict", reason: "not_due", runAt: job.runAt};
+  }
+
+  if (job.status === "running" && isLiveLock(job.lockedAt, lockExpiry)) {
+    return {kind: "conflict", reason: "locked"};
+  }
+
+  return {kind: "conflict", reason: "locked"};
+};
+
+const recordHandlerSuccess = async (
+  jobId: JobDocument["_id"],
+  claim: JobClaimLock
+): Promise<JobDocument> => {
+  const updated = await Job.findOneAndUpdate(
+    {
+      ...buildClaimOwnershipFilter(jobId, claim),
+      status: "running",
+    },
+    {
+      $set: {status: "completed"},
+      $unset: {lastError: ""},
+    },
+    {returnDocument: "after"}
+  );
+
+  if (!updated) {
+    const current = await Job.findOneOrNone({_id: jobId});
+    if (current?.status === "cancelled") {
+      return current;
+    }
+
+    throw new Error(`Failed to record success for job ${jobId.toString()}`);
+  }
+
+  return updated;
+};
+
+const recordMissingHandlerFailure = async (
+  job: JobDocument,
+  message: string,
+  claim: JobClaimLock
+): Promise<JobDocument> => {
+  const updated = await Job.findOneAndUpdate(
+    buildClaimOwnershipFilter(job._id, claim),
+    {
+      $set: {
+        lastError: message,
+        status: "failed",
+      },
+    },
+    {returnDocument: "after"}
+  );
+
+  if (!updated) {
+    throw new Error(`Failed to record missing handler for job ${job._id.toString()}`);
+  }
+
+  return updated;
+};
+
+const recordHandlerFailure = async ({
+  claim,
+  error,
+  job,
+  signal,
+}: {
+  claim: JobClaimLock;
+  error: unknown;
+  job: JobDocument;
+  signal: AbortSignal;
+}): Promise<JobDocument> => {
+  if (signal.aborted && isAbortError(error)) {
+    const current = await Job.findOneOrNone({_id: job._id});
+    if (current?.status === "cancelled") {
+      return current;
+    }
+
+    const released = await Job.findOneAndUpdate(
+      buildClaimOwnershipFilter(job._id, claim),
+      {
+        $set: {status: "pending"},
+        $unset: {lockedAt: "", lockedBy: ""},
+      },
+      {returnDocument: "after"}
+    );
+
+    if (!released) {
+      throw new Error(`Failed to release lock for job ${job._id.toString()}`);
+    }
+
+    return released;
+  }
+
+  const errorMessage = error instanceof Error ? error.message : String(error);
+  const errorClass = error instanceof Error ? error.name : "Error";
+  const attemptedAt = DateTime.utc().toJSDate();
+  const attemptEntry: JobAttempt = {
+    at: attemptedAt,
+    error: errorMessage,
+    errorClass,
+  };
+  const nextAttemptCount = job.attemptCount + 1;
+  const ownershipFilter = buildClaimOwnershipFilter(job._id, claim);
+
+  if (nextAttemptCount >= job.maxAttempts) {
+    const dead = await Job.findOneAndUpdate(
+      ownershipFilter,
+      {
+        $push: {attempts: attemptEntry},
+        $set: {
+          attemptCount: nextAttemptCount,
+          lastError: errorMessage,
+          status: "dead",
+        },
+        $unset: {lockedAt: "", lockedBy: ""},
+      },
+      {returnDocument: "after"}
+    );
+
+    if (!dead) {
+      throw new Error(`Failed to record dead status for job ${job._id.toString()}`);
+    }
+
+    return dead;
+  }
+
+  const runAt = computeRetryRunAt({
+    attemptCount: nextAttemptCount,
+    backoffMs: job.backoffMs,
+    from: DateTime.utc(),
+    maxBackoffMs: job.maxBackoffMs,
+  }).toJSDate();
+
+  const pending = await Job.findOneAndUpdate(
+    ownershipFilter,
+    {
+      $push: {attempts: attemptEntry},
+      $set: {
+        attemptCount: nextAttemptCount,
+        lastError: errorMessage,
+        runAt,
+        status: "pending",
+      },
+      $unset: {lockedAt: "", lockedBy: ""},
+    },
+    {returnDocument: "after"}
+  );
+
+  if (!pending) {
+    throw new Error(`Failed to record retry for job ${job._id.toString()}`);
+  }
+
+  return pending;
+};
+
+export const runClaimedJob = async ({
+  claim,
+  host,
+  job,
+  signal,
+}: {
+  claim: JobClaimLock;
+  host: JobExecutionHost;
+  job: JobDocument;
+  signal: AbortSignal;
+}): Promise<JobDocument> => {
+  const definition = host.getDefinition(job.name);
+  if (!definition) {
+    const message = `No handler registered for job name "${job.name}"`;
+    return recordMissingHandlerFailure(job, message, claim);
+  }
+
+  const log = createScopedLogger({
+    labels: {jobId: job._id.toString(), jobName: job.name},
+    prefix: "[Job]",
+  });
+  const executionSignal = beginActiveJobExecution(job._id.toString(), signal);
+
+  try {
+    await runWithRequestContext({jobId: job._id.toString()}, async () => {
+      await definition.handler(job.payload, {
+        jobId: job._id.toString(),
+        log,
+        signal: executionSignal,
+      });
+    });
+
+    return recordHandlerSuccess(job._id, claim);
+  } catch (error: unknown) {
+    return recordHandlerFailure({claim, error, job, signal: executionSignal});
+  } finally {
+    endActiveJobExecution(job._id.toString());
+  }
+};
+
+const claimJobById = async ({
+  host,
+  jobId,
+  now = DateTime.utc().toJSDate(),
+}: {
+  host: JobExecutionHost;
+  jobId: string;
+  now?: Date;
+}): Promise<JobDocument | undefined> => {
+  const lockExpiry = buildLockExpiry(host.getLockTtlMs(), now);
+  const claim = createClaimLock(now);
+
+  const claimed = await Job.findOneAndUpdate(
+    {
+      _id: jobId,
+      $or: [{lockedAt: {$exists: false}}, {lockedAt: null}, {lockedAt: {$lte: lockExpiry}}],
+      runAt: {$lte: now},
+      status: {$in: ["pending", "running", "scheduled"]},
+    },
+    {
+      $set: {
+        lockedAt: claim.lockedAt,
+        lockedBy: claim.lockedBy,
+        status: "running",
+      },
+    },
+    {returnDocument: "after"}
+  );
+
+  return claimed ?? undefined;
+};
+
+export const executeJobById = async ({
+  host,
+  jobId,
+  signal,
+}: {
+  host: JobExecutionHost;
+  jobId: string;
+  signal: AbortSignal;
+}): Promise<ExecuteJobOutcome> => {
+  if (!mongoose.isValidObjectId(jobId)) {
+    return {kind: "not_found"};
+  }
+
+  const existing = await Job.findOneOrNone({_id: jobId});
+  if (!existing) {
+    return {kind: "not_found"};
+  }
+
+  const now = DateTime.utc().toJSDate();
+  const lockExpiry = buildLockExpiry(host.getLockTtlMs(), now);
+
+  if (TERMINAL_NOOP_STATUSES.has(existing.status)) {
+    return {job: existing, kind: "noop"};
+  }
+
+  if (existing.runAt > now) {
+    return {kind: "conflict", reason: "not_due", runAt: existing.runAt};
+  }
+
+  if (existing.status === "running" && isLiveLock(existing.lockedAt, lockExpiry)) {
+    return {kind: "conflict", reason: "locked"};
+  }
+
+  const claimed = await claimJobById({host, jobId, now});
+  if (!claimed) {
+    const refreshed = await Job.findOneOrNone({_id: jobId});
+    if (!refreshed) {
+      return {kind: "not_found"};
+    }
+
+    if (TERMINAL_NOOP_STATUSES.has(refreshed.status)) {
+      return {job: refreshed, kind: "noop"};
+    }
+
+    return classifyConflict(refreshed, lockExpiry, now);
+  }
+
+  const executed = await runClaimedJob({
+    claim: readClaimLock(claimed),
+    host,
+    job: claimed,
+    signal,
+  });
+
+  return {job: executed, kind: "executed"};
+};

package/src/jobsApp.ts

@@ -0,0 +1,255 @@
+import {
+  type AdminContribution,
+  type AnyTerrenoAccess,
+  logger,
+  type TerrenoPlugin,
+} from "@terreno/api";
+import type express from "express";
+
+import {type ExecuteJobOutcome, executeJobById} from "./jobExecutor";
+import {JobsService, registerJobsService} from "./jobsService";
+import {Job} from "./models/job";
+import {JobSchedule} from "./models/jobSchedule";
+import {registerJobsAdminRoutes} from "./routes/jobsAdmin";
+import {type ExecuteAuthVerifier, registerJobsExecuteRoute} from "./routes/jobsExecute";
+import {MongoJobRunner} from "./runners/mongoRunner";
+import type {JobDefinition, JobRunner, JobsAdminRedactPayload, JobsRunnerHost} from "./types";
+
+export interface JobsAppOptions {
+  accessControl?: AnyTerrenoAccess;
+  basePath?: string;
+  /** Verifies inbound execute HTTP requests (required when the execute route is mounted). */
+  executeAuth?: ExecuteAuthVerifier;
+  lockTtlMs?: number;
+  /** Mount internal `POST {basePath}/execute` for cloud/custom dispatch. */
+  mountExecuteRoute?: boolean;
+  pollIntervalMs?: number;
+  /** Admin list/detail payload projection. Default omits payload; identity hook may expose raw payload. */
+  redactPayload?: JobsAdminRedactPayload;
+  runner?: JobRunner;
+  timezone?: string;
+}
+
+/**
+ * TerrenoPlugin for durable background jobs.
+ *
+ * Registering `JobsApp` wires the plugin into the Express app but does **not**
+ * start a worker poll loop. Call {@link JobsApp.startWorker} explicitly from the
+ * API process or a dedicated worker entrypoint.
+ */
+export class JobsApp implements JobsRunnerHost, TerrenoPlugin {
+  private readonly options: JobsAppOptions;
+  private readonly runner: JobRunner;
+  private readonly service: JobsService;
+  private abortController: AbortController | undefined;
+  private workerActive = false;
+  private workerPromise: Promise<void> | undefined;
+  private workerStartPromise: Promise<void> | undefined;
+
+  constructor(options?: JobsAppOptions) {
+    this.options = options ?? {};
+    this.runner = this.options.runner ?? new MongoJobRunner();
+    this.service = new JobsService({
+      defaultTimezone: this.options.timezone ?? "UTC",
+      lockTtlMs: this.options.lockTtlMs,
+      runner: this.runner,
+    });
+  }
+
+  define(name: string, definition: JobDefinition): this {
+    this.service.define(name, definition);
+    if (this.workerActive && definition.schedule) {
+      this.service.markSchedulesDirty();
+    }
+    return this;
+  }
+
+  getBasePath(): string {
+    return this.options.basePath ?? "/jobs";
+  }
+
+  getDefinition(name: string): JobDefinition | undefined {
+    return this.service.getDefinition(name);
+  }
+
+  getDefaultTimezone(): string {
+    return this.service.getDefaultTimezone();
+  }
+
+  getLockTtlMs(): number {
+    return this.service.getLockTtlMs();
+  }
+
+  /** Execute a persisted job by id from an external dispatch surface (queue consumer, execute HTTP). */
+  async executeQueuedJob(
+    jobId: string,
+    signal: AbortSignal = AbortSignal.timeout(30 * 60 * 1_000)
+  ): Promise<ExecuteJobOutcome> {
+    return executeJobById({
+      host: this,
+      jobId,
+      signal,
+    });
+  }
+
+  getPollIntervalMs(): number {
+    return this.options.pollIntervalMs ?? 1_000;
+  }
+
+  async reconcileSchedules(): Promise<void> {
+    await this.service.reconcileSchedules();
+  }
+
+  async reconcileSchedulesIfDirty(): Promise<void> {
+    await this.service.reconcileSchedulesIfDirty();
+  }
+
+  async tickSchedules(now?: Date): Promise<void> {
+    await this.service.tickSchedules(now);
+  }
+
+  register(app: express.Application, openApi?: unknown): void {
+    registerJobsService(this.service);
+
+    registerJobsAdminRoutes({
+      app,
+      options: {
+        accessControl: this.options.accessControl,
+        basePath: this.getBasePath(),
+        jobsService: this.service,
+        openApi,
+        redactPayload: this.options.redactPayload,
+      },
+    });
+
+    if (this.shouldMountExecuteRoute()) {
+      if (!this.options.executeAuth) {
+        throw new Error(
+          "JobsApp execute route is enabled but executeAuth is missing. Provide executeAuth when mountExecuteRoute is true or the runner requires the execute route."
+        );
+      }
+
+      registerJobsExecuteRoute({
+        app,
+        options: {
+          basePath: this.getBasePath(),
+          executeAuth: this.options.executeAuth,
+          host: this.service,
+        },
+      });
+    }
+
+    void Job.init().catch((error: unknown) => {
+      logger.error(`[jobs] Failed to build Job indexes: ${String(error)}`);
+    });
+
+    void JobSchedule.init().catch((error: unknown) => {
+      logger.error(`[jobs] Failed to build JobSchedule indexes: ${String(error)}`);
+    });
+  }
+
+  adminContribution(): AdminContribution {
+    return {
+      customScreens: [
+        {
+          displayName: "Jobs",
+          icon: "clock",
+          name: "jobs",
+        },
+      ],
+      homeWidgets: [{displayName: "Jobs", icon: "clock", id: "jobs"}],
+    };
+  }
+
+  private shouldMountExecuteRoute(): boolean {
+    if (this.options.mountExecuteRoute) {
+      return true;
+    }
+
+    return this.runner.requiresExecuteRoute === true;
+  }
+
+  isWorkerActive(): boolean {
+    return this.workerActive;
+  }
+
+  /**
+   * Starts the background worker poll loop without awaiting it.
+   *
+   * Fire-and-forget: resolves once the runner has been scheduled. Runner errors are
+   * logged and worker state is reset; they are not thrown from this method.
+   */
+  async startWorker(): Promise<void> {
+    if (this.workerActive) {
+      return;
+    }
+
+    if (this.workerStartPromise !== undefined) {
+      await this.workerStartPromise;
+      return;
+    }
+
+    this.workerStartPromise = this.beginWorkerStart();
+    try {
+      await this.workerStartPromise;
+    } finally {
+      this.workerStartPromise = undefined;
+    }
+  }
+
+  async stopWorker(): Promise<void> {
+    if (!this.workerActive) {
+      return;
+    }
+
+    const stopPromise = this.runner.stop?.();
+    this.abortController?.abort();
+
+    try {
+      await this.workerPromise;
+    } catch {
+      // startWorker is fire-and-forget; runner errors are logged in workerPromise.catch.
+    }
+
+    await stopPromise;
+
+    this.resetWorkerState();
+  }
+
+  private resetWorkerState(): void {
+    this.workerActive = false;
+    this.abortController = undefined;
+    this.workerPromise = undefined;
+  }
+
+  private async beginWorkerStart(): Promise<void> {
+    if (this.workerActive) {
+      return;
+    }
+
+    if (!this.runner.start) {
+      throw new Error(`Job runner "${this.runner.id}" does not support start()`);
+    }
+
+    this.workerActive = true;
+
+    try {
+      await this.service.reconcileSchedules();
+
+      this.abortController = new AbortController();
+      this.workerPromise = this.runner
+        .start({
+          jobs: this,
+          pollIntervalMs: this.getPollIntervalMs(),
+          signal: this.abortController.signal,
+        })
+        .catch((error: unknown) => {
+          this.resetWorkerState();
+          logger.error(`[jobs] Worker stopped with error: ${String(error)}`);
+        });
+    } catch (error: unknown) {
+      this.resetWorkerState();
+      throw error;
+    }
+  }
+}