@terreno/jobs 57.4.0

package/src/jobsService.ts

@@ -0,0 +1,395 @@
+import {APIError, NotFoundError} from "@terreno/api";
+import {DateTime} from "luxon";
+import mongoose from "mongoose";
+import {abortActiveJobExecution} from "./activeJobExecution";
+import {dispatchJobToRunner} from "./dispatchCompensation";
+import {Job} from "./models/job";
+import {JobSchedule} from "./models/jobSchedule";
+import type {JobDocument, JobScheduleDocument} from "./modelTypes";
+import {DEFAULT_BACKOFF_MS, DEFAULT_MAX_BACKOFF_MS} from "./retryBackoff";
+import {
+  computeInitialNextRunAt,
+  resolveScheduleTimezone,
+  validateScheduleDefinition,
+} from "./scheduleCron";
+import {tickDueSchedules} from "./scheduler";
+import type {EnqueueJobParams, JobDefinition, JobRunner} from "./types";
+
+const DEFAULT_MAX_ATTEMPTS = 5;
+const DEFAULT_TIMEZONE = "UTC";
+const UNKNOWN_JOB_NAME_TITLE = "Unknown job name";
+
+const isDuplicateKeyError = (error: unknown): boolean =>
+  Boolean(
+    error &&
+      typeof error === "object" &&
+      "code" in error &&
+      (error as {code: unknown}).code === 11_000
+  );
+
+export class JobsService {
+  private readonly defaultTimezone: string;
+  private readonly definitions = new Map<string, JobDefinition>();
+  private readonly lockTtlMs: number;
+  private readonly runner: JobRunner | undefined;
+  private schedulesDirty = false;
+
+  constructor(options?: {defaultTimezone?: string; lockTtlMs?: number; runner?: JobRunner}) {
+    this.defaultTimezone = options?.defaultTimezone ?? DEFAULT_TIMEZONE;
+    this.lockTtlMs = options?.lockTtlMs ?? 15 * 60 * 1000;
+    this.runner = options?.runner;
+  }
+
+  define(name: string, definition: JobDefinition): void {
+    if (definition.schedule) {
+      validateScheduleDefinition({
+        cron: definition.schedule.cron,
+        defaultTimezone: this.defaultTimezone,
+        scheduleTimezone: definition.schedule.timezone,
+      });
+    }
+
+    this.definitions.set(name, definition);
+  }
+
+  getDefinition(name: string): JobDefinition | undefined {
+    return this.definitions.get(name);
+  }
+
+  getDefaultTimezone(): string {
+    return this.defaultTimezone;
+  }
+
+  getLockTtlMs(): number {
+    return this.lockTtlMs;
+  }
+
+  isSchedulesDirty(): boolean {
+    return this.schedulesDirty;
+  }
+
+  markSchedulesDirty(): void {
+    this.schedulesDirty = true;
+  }
+
+  async reconcileSchedulesIfDirty(): Promise<void> {
+    if (!this.schedulesDirty) {
+      return;
+    }
+
+    await this.reconcileSchedules();
+  }
+
+  async reconcileSchedules(): Promise<void> {
+    for (const [name, definition] of this.definitions) {
+      if (!definition.schedule) {
+        continue;
+      }
+
+      const timezone = resolveScheduleTimezone({
+        defaultTimezone: this.defaultTimezone,
+        scheduleTimezone: definition.schedule.timezone,
+      });
+      const cron = definition.schedule.cron;
+      const existing = await JobSchedule.findOneOrNone({name});
+
+      if (!existing) {
+        await JobSchedule.findOneAndUpdate(
+          {name},
+          {
+            $setOnInsert: {
+              cron,
+              enabled: true,
+              handlerName: name,
+              nextRunAt: computeInitialNextRunAt(cron, timezone),
+              timezone,
+            },
+          },
+          {upsert: true}
+        );
+        continue;
+      }
+
+      const cronTimezoneChanged = existing.cron !== cron || existing.timezone !== timezone;
+
+      const update: {
+        cron: string;
+        handlerName: string;
+        nextRunAt?: Date;
+        timezone: string;
+      } = {
+        cron,
+        handlerName: name,
+        timezone,
+      };
+
+      if (cronTimezoneChanged) {
+        update.nextRunAt = computeInitialNextRunAt(cron, timezone);
+      }
+
+      await JobSchedule.updateOne({_id: existing._id}, {$set: update});
+    }
+
+    this.schedulesDirty = false;
+  }
+
+  async tickSchedules(now?: Date): Promise<void> {
+    await tickDueSchedules({
+      jobsService: this,
+      now,
+    });
+  }
+
+  async enqueue(params: EnqueueJobParams): Promise<JobDocument> {
+    const definition = this.definitions.get(params.name);
+    if (!definition) {
+      throw new APIError({status: 400, title: UNKNOWN_JOB_NAME_TITLE});
+    }
+
+    const runAt = params.runAt ?? DateTime.utc().toJSDate();
+    const maxAttempts = definition.retry?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+    const backoffMs = definition.retry?.backoffMs ?? DEFAULT_BACKOFF_MS;
+    const maxBackoffMs = definition.retry?.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
+    const idempotencyKey = params.idempotencyKey?.trim();
+
+    if (idempotencyKey) {
+      const existing = await Job.findOneOrNone({
+        idempotencyKey,
+        name: params.name,
+      });
+      if (existing) {
+        return existing;
+      }
+    }
+
+    const createPayload = {
+      attemptCount: 0,
+      backoffMs,
+      maxAttempts,
+      maxBackoffMs,
+      name: params.name,
+      payload: params.payload,
+      payloadRedacted: false,
+      runAt,
+      status: "pending" as const,
+      ...(idempotencyKey ? {idempotencyKey} : {}),
+      ...(params.scheduleId ? {scheduleId: new mongoose.Types.ObjectId(params.scheduleId)} : {}),
+    };
+
+    let job: JobDocument;
+    try {
+      job = await Job.create(createPayload);
+    } catch (error: unknown) {
+      if (!idempotencyKey || !isDuplicateKeyError(error)) {
+        throw error;
+      }
+
+      return Job.findExactlyOne({
+        idempotencyKey,
+        name: params.name,
+      });
+    }
+
+    if (this.runner) {
+      await dispatchJobToRunner(this.runner, job);
+    }
+
+    return job;
+  }
+
+  async adminRetryJob(jobId: string): Promise<{original: JobDocument; retry: JobDocument}> {
+    if (!mongoose.isValidObjectId(jobId)) {
+      throw new NotFoundError("Job not found");
+    }
+
+    const original = await Job.findOneOrNone({_id: jobId});
+    if (!original) {
+      throw new NotFoundError("Job not found");
+    }
+
+    if (original.retriedById) {
+      throw new APIError({status: 409, title: "Job already retried"});
+    }
+
+    if (original.status !== "dead" && original.status !== "failed") {
+      throw new APIError({
+        detail: `Status is ${original.status}`,
+        status: 400,
+        title: "Job cannot be retried",
+      });
+    }
+
+    const definition = this.definitions.get(original.name);
+    if (!definition) {
+      throw new APIError({status: 400, title: UNKNOWN_JOB_NAME_TITLE});
+    }
+
+    const now = DateTime.utc().toJSDate();
+    const retry = await Job.create({
+      attemptCount: 0,
+      attempts: [],
+      backoffMs: original.backoffMs,
+      maxAttempts: original.maxAttempts,
+      maxBackoffMs: original.maxBackoffMs,
+      name: original.name,
+      payload: original.payload,
+      payloadRedacted: original.payloadRedacted,
+      retriedFromId: original._id,
+      runAt: now,
+      scheduleId: original.scheduleId,
+      status: "pending",
+    });
+
+    const linked = await Job.findOneAndUpdate(
+      {
+        _id: original._id,
+        retriedById: {$exists: false},
+        status: {$in: ["dead", "failed"]},
+      },
+      {$set: {retriedById: retry._id}},
+      {returnDocument: "after"}
+    );
+
+    if (!linked) {
+      await Job.deleteOne({_id: retry._id});
+      const current = await Job.findOneOrNone({_id: original._id});
+      if (current?.retriedById) {
+        throw new APIError({status: 409, title: "Job already retried"});
+      }
+
+      throw new APIError({status: 409, title: "Job state changed"});
+    }
+
+    if (this.runner) {
+      await dispatchJobToRunner(this.runner, retry);
+    }
+
+    return {original: linked, retry};
+  }
+
+  async adminRequeueJob(jobId: string): Promise<JobDocument> {
+    if (!mongoose.isValidObjectId(jobId)) {
+      throw new NotFoundError("Job not found");
+    }
+
+    const now = DateTime.utc().toJSDate();
+    const requeued = await Job.findOneAndUpdate(
+      {
+        _id: jobId,
+        retriedById: {$exists: false},
+        status: {$in: ["cancelled", "dead", "failed"]},
+      },
+      {
+        $set: {runAt: now, status: "pending"},
+        $unset: {lockedAt: "", lockedBy: ""},
+      },
+      {returnDocument: "after"}
+    );
+
+    if (!requeued) {
+      const existing = await Job.findOneOrNone({_id: jobId});
+      if (!existing) {
+        throw new NotFoundError("Job not found");
+      }
+
+      if (existing.retriedById) {
+        throw new APIError({status: 409, title: "Job already retried"});
+      }
+
+      throw new APIError({
+        detail: `Status is ${existing.status}`,
+        status: 400,
+        title: "Job cannot be requeued",
+      });
+    }
+
+    if (this.runner) {
+      await dispatchJobToRunner(this.runner, requeued);
+    }
+
+    return requeued;
+  }
+
+  async adminCancelJob(jobId: string): Promise<JobDocument> {
+    if (!mongoose.isValidObjectId(jobId)) {
+      throw new NotFoundError("Job not found");
+    }
+
+    const cancelled = await Job.findOneAndUpdate(
+      {
+        _id: jobId,
+        status: {$in: ["pending", "running", "scheduled"]},
+      },
+      {
+        $set: {status: "cancelled"},
+        $unset: {lockedAt: "", lockedBy: ""},
+      },
+      {returnDocument: "after"}
+    );
+
+    if (!cancelled) {
+      const existing = await Job.findOneOrNone({_id: jobId});
+      if (!existing) {
+        throw new NotFoundError("Job not found");
+      }
+
+      throw new APIError({
+        detail: `Status is ${existing.status}`,
+        status: 400,
+        title: "Job cannot be cancelled",
+      });
+    }
+
+    abortActiveJobExecution(jobId);
+    return cancelled;
+  }
+
+  async pauseSchedule(name: string): Promise<JobScheduleDocument> {
+    const updated = await JobSchedule.findOneAndUpdate(
+      {name},
+      {$set: {enabled: false}},
+      {returnDocument: "after"}
+    );
+
+    if (!updated) {
+      throw new NotFoundError("Schedule not found");
+    }
+
+    return updated;
+  }
+
+  async resumeSchedule(name: string): Promise<JobScheduleDocument> {
+    const updated = await JobSchedule.findOneAndUpdate(
+      {name},
+      {$set: {enabled: true}},
+      {returnDocument: "after"}
+    );
+
+    if (!updated) {
+      throw new NotFoundError("Schedule not found");
+    }
+
+    return updated;
+  }
+}
+
+let registeredJobsService: JobsService | undefined;
+
+export const getJobsService = (): JobsService => {
+  if (!registeredJobsService) {
+    throw new APIError({status: 500, title: "JobsApp is not registered"});
+  }
+  return registeredJobsService;
+};
+
+/** Returns the registered jobs service when `JobsApp` is mounted; otherwise `undefined`. */
+export const tryGetJobsService = (): JobsService | undefined => registeredJobsService;
+
+export const registerJobsService = (service: JobsService): void => {
+  registeredJobsService = service;
+};
+
+/** Clears the process-wide jobs service. Tests use this so AdminApp falls back to in-process runs. */
+export const unregisterJobsService = (): void => {
+  registeredJobsService = undefined;
+};

package/src/modelTypes.ts

@@ -0,0 +1,78 @@
+import type {FindExactlyOnePlugin, FindOneOrNonePlugin} from "@terreno/api";
+import type mongoose from "mongoose";
+
+export type JobStatus =
+  | "cancelled"
+  | "completed"
+  | "dead"
+  | "failed"
+  | "pending"
+  | "running"
+  | "scheduled";
+
+export const IN_FLIGHT_JOB_STATUSES: readonly JobStatus[] = ["pending", "running", "scheduled"];
+
+export interface JobAttempt {
+  at: Date;
+  error?: string;
+  errorClass?: string;
+}
+
+export type JobMethods = Record<never, never>;
+
+export interface JobDocument extends mongoose.Document<mongoose.Types.ObjectId>, JobMethods {
+  attemptCount: number;
+  attempts: JobAttempt[];
+  backoffMs: number;
+  created: Date;
+  idempotencyKey?: string;
+  lastError?: string;
+  lockedAt?: Date;
+  lockedBy?: string;
+  maxAttempts: number;
+  maxBackoffMs: number;
+  name: string;
+  payload?: unknown;
+  payloadRedacted: boolean;
+  retriedById?: mongoose.Types.ObjectId;
+  retriedFromId?: mongoose.Types.ObjectId;
+  runAt: Date;
+  scheduleId?: mongoose.Types.ObjectId;
+  status: JobStatus;
+  updated: Date;
+}
+
+export interface JobStatics
+  extends FindExactlyOnePlugin<JobDocument>,
+    FindOneOrNonePlugin<JobDocument> {}
+
+export interface JobModel extends mongoose.Model<JobDocument>, JobStatics {}
+
+export type JobSchema = mongoose.Schema<JobDocument, JobModel, JobMethods>;
+
+export type JobScheduleMethods = Record<never, never>;
+
+export interface JobScheduleDocument
+  extends mongoose.Document<mongoose.Types.ObjectId>,
+    JobScheduleMethods {
+  created: Date;
+  cron: string;
+  enabled: boolean;
+  handlerName: string;
+  name: string;
+  nextRunAt: Date;
+  timezone: string;
+  updated: Date;
+}
+
+export interface JobScheduleStatics
+  extends FindExactlyOnePlugin<JobScheduleDocument>,
+    FindOneOrNonePlugin<JobScheduleDocument> {}
+
+export interface JobScheduleModel extends mongoose.Model<JobScheduleDocument>, JobScheduleStatics {}
+
+export type JobScheduleSchema = mongoose.Schema<
+  JobScheduleDocument,
+  JobScheduleModel,
+  JobScheduleMethods
+>;

package/src/models/job.ts

@@ -0,0 +1,129 @@
+import {createdUpdatedPlugin, findExactlyOne, findOneOrNone} from "@terreno/api";
+import mongoose from "mongoose";
+
+import type {JobDocument, JobModel, JobSchema} from "../modelTypes";
+
+const jobSchema: JobSchema = new mongoose.Schema<JobDocument, JobModel>(
+  {
+    attemptCount: {
+      default: 0,
+      description: "Number of execution attempts recorded for this job",
+      type: Number,
+    },
+    attempts: {
+      default: [],
+      description: "History of execution attempts including errors",
+      type: [
+        {
+          _id: false,
+          at: {
+            description: "When this execution attempt ran",
+            required: true,
+            type: Date,
+          },
+          error: {
+            description: "Error message captured for this attempt",
+            type: String,
+          },
+          errorClass: {
+            description: "Classified error category for this attempt",
+            type: String,
+          },
+        },
+      ],
+    },
+    backoffMs: {
+      default: 1_000,
+      description: "Base delay in milliseconds for exponential retry backoff",
+      type: Number,
+    },
+    idempotencyKey: {
+      description: "Optional deduplication key scoped to the job name",
+      type: String,
+    },
+    lastError: {
+      description: "Most recent error message from a failed attempt",
+      type: String,
+    },
+    lockedAt: {
+      description: "When this job row was claimed by a worker",
+      type: Date,
+    },
+    lockedBy: {
+      description: "Worker identity that claimed this job row",
+      type: String,
+    },
+    maxAttempts: {
+      default: 5,
+      description: "Maximum execution attempts before the job is dead-lettered",
+      type: Number,
+    },
+    maxBackoffMs: {
+      default: 15 * 60 * 1_000,
+      description: "Maximum retry delay in milliseconds before backoff is capped",
+      type: Number,
+    },
+    name: {
+      description: "Registered handler name for this job",
+      index: true,
+      required: true,
+      type: String,
+    },
+    payload: {
+      description: "JSON-serializable input passed to the handler",
+      type: mongoose.Schema.Types.Mixed,
+    },
+    payloadRedacted: {
+      default: false,
+      description: "Whether the stored payload has been redacted for admin display",
+      type: Boolean,
+    },
+    retriedById: {
+      description: "Follow-up job created by an admin retry action",
+      ref: "Job",
+      type: mongoose.Schema.Types.ObjectId,
+    },
+    retriedFromId: {
+      description: "Original job row that this retry was created from",
+      ref: "Job",
+      type: mongoose.Schema.Types.ObjectId,
+    },
+    runAt: {
+      description: "Earliest time a worker may claim this job",
+      index: true,
+      required: true,
+      type: Date,
+    },
+    scheduleId: {
+      description: "Recurring schedule that enqueued this job, when applicable",
+      ref: "JobSchedule",
+      type: mongoose.Schema.Types.ObjectId,
+    },
+    status: {
+      description: "Current lifecycle status of the job row",
+      enum: ["cancelled", "completed", "dead", "failed", "pending", "running", "scheduled"],
+      index: true,
+      required: true,
+      type: String,
+    },
+  },
+  {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
+);
+
+jobSchema.plugin(createdUpdatedPlugin);
+jobSchema.plugin(findOneOrNone);
+jobSchema.plugin(findExactlyOne);
+
+jobSchema.index(
+  // biome-ignore assist/source/useSortedKeys: IP specifies (name, idempotencyKey) index field order
+  {name: 1, idempotencyKey: 1},
+  {
+    partialFilterExpression: {idempotencyKey: {$exists: true, $type: "string"}},
+    unique: true,
+  }
+);
+// biome-ignore assist/source/useSortedKeys: IP specifies (status, runAt) claim index field order
+jobSchema.index({status: 1, runAt: 1});
+jobSchema.index({scheduleId: 1, status: 1});
+
+export const Job = mongoose.model<JobDocument, JobModel>("Job", jobSchema);

package/src/models/jobSchedule.ts

@@ -0,0 +1,57 @@
+import {createdUpdatedPlugin, findExactlyOne, findOneOrNone} from "@terreno/api";
+import mongoose from "mongoose";
+
+import type {JobScheduleDocument, JobScheduleModel, JobScheduleSchema} from "../modelTypes";
+
+const jobScheduleSchema: JobScheduleSchema = new mongoose.Schema<
+  JobScheduleDocument,
+  JobScheduleModel
+>(
+  {
+    cron: {
+      description: "Cron expression that controls when this schedule fires",
+      required: true,
+      type: String,
+    },
+    enabled: {
+      default: true,
+      description: "Whether the worker should enqueue jobs for this schedule",
+      type: Boolean,
+    },
+    handlerName: {
+      description: "Registered job handler name invoked for each schedule tick",
+      required: true,
+      type: String,
+    },
+    name: {
+      description: "Unique schedule name matching the registered job definition",
+      required: true,
+      type: String,
+      unique: true,
+    },
+    nextRunAt: {
+      description: "Next UTC instant when the scheduler should evaluate this schedule",
+      index: true,
+      required: true,
+      type: Date,
+    },
+    timezone: {
+      default: "UTC",
+      description: "IANA timezone used to interpret the cron expression",
+      required: true,
+      type: String,
+    },
+  },
+  {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
+);
+
+jobScheduleSchema.plugin(createdUpdatedPlugin);
+jobScheduleSchema.plugin(findOneOrNone);
+jobScheduleSchema.plugin(findExactlyOne);
+
+jobScheduleSchema.index({enabled: 1, nextRunAt: 1});
+
+export const JobSchedule = mongoose.model<JobScheduleDocument, JobScheduleModel>(
+  "JobSchedule",
+  jobScheduleSchema
+);

package/src/retryBackoff.ts

@@ -0,0 +1,34 @@
+import type {DateTime} from "luxon";
+
+export const DEFAULT_BACKOFF_MS = 1_000;
+export const DEFAULT_MAX_BACKOFF_MS = 15 * 60 * 1_000;
+
+export interface RetryBackoffOptions {
+  backoffMs: number;
+  maxBackoffMs: number;
+}
+
+export interface ComputeRetryDelayMsParams extends RetryBackoffOptions {
+  attemptCount: number;
+  random?: () => number;
+}
+
+export interface ComputeRetryRunAtParams extends ComputeRetryDelayMsParams {
+  from: DateTime;
+}
+
+const equalJitterFactor = (random: () => number): number => 0.5 + random() * 0.5;
+
+/** @internal */
+export const computeRetryDelayMs = (params: ComputeRetryDelayMsParams): number => {
+  const random = params.random ?? Math.random;
+  const exponent = Math.max(0, params.attemptCount - 1);
+  const baseDelayMs = params.backoffMs * 2 ** exponent;
+  const cappedDelayMs = Math.min(params.maxBackoffMs, baseDelayMs);
+  return Math.round(cappedDelayMs * equalJitterFactor(random));
+};
+
+export const computeRetryRunAt = (params: ComputeRetryRunAtParams): DateTime =>
+  params.from.plus({
+    milliseconds: computeRetryDelayMs(params),
+  });