@softure-ai/jobs 0.1.0 → 0.1.1

package/src/options.ts

@@ -7,14 +7,34 @@ const DAY_SECONDS = 24 * 60 * 60;
 
 const backoffSchema = z
   .strictObject({
-    /** Delay before the second attempt; each later one doubles it. */
-    baseSeconds: z.number().int().min(1).max(DAY_SECONDS).default(10),
-    /** The longest delay between two attempts. */
-    maxSeconds: z.number().int().min(1).max(7 * DAY_SECONDS).default(3600),
-  })
-  .refine((backoff) => backoff.maxSeconds >= backoff.baseSeconds, {
-    message: "maxSeconds must be at least baseSeconds",
-    path: ["maxSeconds"],
+      /** Delay before the second attempt; each later one doubles it. */
+      baseSeconds: z.number().int().min(1).max(DAY_SECONDS).default(10),
+      /** The longest delay between two attempts. */
+      maxSeconds: z.number().int().min(1).max(7 * DAY_SECONDS).default(3600),
+    })
+    .refine((backoff) => backoff.maxSeconds >= backoff.baseSeconds, {
+      message: "maxSeconds must be at least baseSeconds",
+      path: ["maxSeconds"],
+    });
+
+  const EVERY = /^([1-9][0-9]{0,4})(m|h|d)$/;
+  const UNIT_SECONDS = { m: 60, h: 60 * 60, d: DAY_SECONDS } as const;
+  const MAX_EVERY_SECONDS = 7 * DAY_SECONDS;
+  const EVERY_HINT = "must be a whole number of minutes, hours or days (e.g. 15m, 1h, 1d), from 1m to 7d";
+
+  /** Seconds of an `every` text such as `15m`, `1h` or `1d`; null when it is not one or is outside 1m to 7d. */
+  export function parseEverySeconds(every: string): number | null {
+    const match = EVERY.exec(every);
+    if (match === null) return null;
+    const seconds = Number(match[1]) * UNIT_SECONDS[match[2] as keyof typeof UNIT_SECONDS];
+    return seconds <= MAX_EVERY_SECONDS ? seconds : null;
+  }
+
+  const scheduleSchema = z.strictObject({
+    /** How often the schedule runs: `15m`, `1h`, `1d`. Runs start on whole periods counted from 1970-01-01 00:00 UTC. */
+    every: z.string().refine((every) => parseEverySeconds(every) !== null, { message: EVERY_HINT }),
+    /** The payload of every run; default `{}`. */
+    payload: z.record(z.string(), z.unknown()).default({}),
   });
 
 const queueSchema = z.strictObject({
@@ -40,25 +60,41 @@ const retentionSchema = z.strictObject({
   cancelledDays: z.number().int().min(0).max(3650).default(7),
 });
 
-export const jobsOptionsSchema = z.strictObject({
-  /** The queues, by name. A job is enqueued to one of them and runs with its settings. */
-  queues: z.record(z.string(), queueSchema).superRefine((queues, context) => {
-    const names = Object.keys(queues);
-    if (names.length === 0) {
-      context.addIssue({ code: "custom", message: "must define at least one queue" });
-    }
-    for (const name of names.filter((candidate) => !QUEUE_NAME.test(candidate))) {
-      context.addIssue({ code: "custom", path: [name], message: QUEUE_NAME_HINT });
+export const jobsOptionsSchema = z
+  .strictObject({
+    /** The queues, by name. A job is enqueued to one of them and runs with its settings. */
+    queues: z.record(z.string(), queueSchema).default({}).superRefine(checkNames),
+    /**
+     * Recurring runs, by queue name: the worker enqueues one job per period (`every`) however many workers run. A
+     * schedule's queue takes the default queue settings unless `queues` names it.
+     */
+    schedules: z.record(z.string(), scheduleSchema).default({}).superRefine(checkNames),
+    /** How long an idle worker waits before it looks for due jobs again. */
+    pollIntervalMs: z.number().int().min(50).max(60_000).default(1000),
+    /** How long a stopping worker waits for running handlers before it releases their jobs. */
+    shutdownGraceMs: z.number().int().min(0).max(10 * 60_000).default(20_000),
+    /** How long finished jobs are kept; the worker prunes once an hour. */
+    retention: retentionSchema.default({ succeededDays: 7, failedDays: 30, cancelledDays: 7 }),
+  })
+  .superRefine((options, context) => {
+    if (Object.keys(options.queues).length === 0 && Object.keys(options.schedules).length === 0) {
+      context.addIssue({ code: "custom", path: ["queues"], message: "must define at least one queue or schedule" });
     }
-  }),
-  /** How long an idle worker waits before it looks for due jobs again. */
-  pollIntervalMs: z.number().int().min(50).max(60_000).default(1000),
-  /** How long a stopping worker waits for running handlers before it releases their jobs. */
-  shutdownGraceMs: z.number().int().min(0).max(10 * 60_000).default(20_000),
-  /** How long finished jobs are kept; the worker prunes once an hour. */
-  retention: retentionSchema.default({ succeededDays: 7, failedDays: 30, cancelledDays: 7 }),
-});
+  })
+  .transform((options) => {
+    const scheduledOnly = Object.keys(options.schedules).filter((name) => !Object.hasOwn(options.queues, name));
+    return { ...options, queues: { ...options.queues, ...Object.fromEntries(scheduledOnly.map((name) => [name, queueSchema.parse({})])) } };
+  });
+
+function checkNames(record: Record<string, unknown>, context: z.RefinementCtx): void {
+  for (const name of Object.keys(record).filter((candidate) => !QUEUE_NAME.test(candidate))) {
+    context.addIssue({ code: "custom", path: [name], message: QUEUE_NAME_HINT });
+  }
+}
 
 export type JobsOptionsInput = z.input<typeof jobsOptionsSchema>;
 export type JobsOptions = z.output<typeof jobsOptionsSchema>;
 export type JobQueueOptions = JobsOptions["queues"][string];
+export type JobScheduleOptions = JobsOptions["schedules"][string];
+/** Schedules as an app or a package writes them, e.g. a package's ready-made prune schedules. */
+export type JobSchedulesInput = NonNullable<JobsOptionsInput["schedules"]>;

package/src/schema.ts

@@ -1,4 +1,4 @@
-// Drizzle view of the module's table (migrations/0001_create_jobs.sql). The migration is the source of truth; this
+// Drizzle view of the module's tables (migrations/). The migrations are the source of truth; this
 // file only types the queries.
 import { bigint, integer, jsonb, pgSchema, text, timestamp, uuid } from "drizzle-orm/pg-core";
 
@@ -23,3 +23,11 @@ export const jobs = jobsSchema.table("jobs", {
 });
 
 export type JobRow = typeof jobs.$inferSelect;
+
+export const schedules = jobsSchema.table("schedules", {
+  name: text("name").primaryKey(),
+  lastSlotAt: timestamp("last_slot_at", { withTimezone: true }).notNull(),
+  lastSucceededAt: timestamp("last_succeeded_at", { withTimezone: true }),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
+  updatedAt: timestamp("updated_at", { withTimezone: true }).notNull(),
+});

package/src/server/claim.ts

@@ -6,6 +6,7 @@ import { jobs } from "../schema.js";
 import type { JobsContext } from "./context.js";
 import { toJob } from "./job-row.js";
 import { getJobsOptions, getQueue } from "./options.js";
+import { recordScheduleSuccess } from "./schedules.js";
 
 /** A job this worker holds: the job and the token its writes are fenced with. */
 export interface ClaimedJob {
@@ -87,7 +88,10 @@ export async function extendJobLease(ctx: JobsContext, claimed: ClaimedJob): Pro
   return rows.length > 0;
 }
 
-/** Records the end of an attempt. False when the job is no longer this claim's: nothing was written. */
+/**
+ * Records the end of an attempt; a success of a scheduled queue also updates its schedule, in the same transaction.
+ * False when the job is no longer this claim's: nothing was written.
+ */
 export async function completeJob(ctx: JobsContext, claimed: ClaimedJob, completion: Completion): Promise<boolean> {
   const now = ctx.clock.now();
   const unlocked = { lockedUntil: null, lockedBy: null, claimToken: null, updatedAt: now };
@@ -97,8 +101,12 @@ export async function completeJob(ctx: JobsContext, claimed: ClaimedJob, complet
       : completion.kind === "retry"
         ? { ...unlocked, status: "pending" as const, runAt: completion.runAt, lastError: completion.error }
         : { ...unlocked, status: "failed" as const, finishedAt: now, lastError: completion.error };
-  const rows = await ctx.db.update(jobs).set(set).where(isHeld(claimed)).returning();
-  return rows.length > 0;
+  return ctx.db.transaction(async (tx) => {
+    const rows = await tx.update(jobs).set(set).where(isHeld(claimed)).returning();
+    if (rows.length === 0) return false;
+    if (completion.kind === "succeeded") await recordScheduleSuccess({ ...ctx, db: tx }, claimed.job.queue);
+    return true;
+  });
 }
 
 /**

package/src/server/index.ts

@@ -16,4 +16,11 @@ export { consoleJobsLogger, type JobsLogFields, type JobsLogger } from "./logger
 export { getQueueStats, pruneJobs, type QueueStats } from "./maintenance.js";
 export { getJobsOptions } from "./options.js";
 export { runClaimedJob, type RunClaimedJobOptions } from "./process.js";
+export {
+  enqueueDueSchedules,
+  getScheduleSlot,
+  SCHEDULE_DEDUP_KEY,
+  type EnqueueDueSchedulesInput,
+  type ScheduleState,
+} from "./schedules.js";
 export { runDueJobs, type RunDueJobsOptions, type RunDueJobsReport } from "./run-due.js";

package/src/server/maintenance.ts

@@ -4,6 +4,7 @@ import { JOB_STATUSES, type JobStatus } from "../contract.js";
 import { jobs } from "../schema.js";
 import type { JobsContext } from "./context.js";
 import { getJobsOptions } from "./options.js";
+import { getScheduleStates, type ScheduleState } from "./schedules.js";
 
 const DAY_MS = 24 * 60 * 60 * 1000;
 /** Rows deleted per statement, so pruning a large backlog never holds one long lock. */
@@ -39,11 +40,13 @@ export interface QueueStats {
   readonly due: number;
   /** The `run_at` of the oldest due pending job, or null: how far behind the workers are. */
   readonly oldestDueAt: Date | null;
+  /** The queue's schedule, when `schedules` names it: when it last ran and whether it is late. */
+  readonly schedule: ScheduleState | null;
 }
 
 /**
- * Counts per queue: every configured queue, plus any queue that still has rows in the table. For a dashboard, an ops
- * script or a health check of the app's own (`ops({ checks })`).
+ * Counts per queue: every configured queue, plus any queue that still has rows in the table, with the state of its
+ * schedule. For a dashboard, an ops script or a health check of the app's own (`ops({ checks })`).
  */
 export async function getQueueStats(ctx: JobsContext): Promise<QueueStats[]> {
   const now = ctx.clock.now();
@@ -56,12 +59,13 @@ export async function getQueueStats(ctx: JobsContext): Promise<QueueStats[]> {
     .from(jobs)
     .where(and(eq(jobs.status, "pending"), sql`${jobs.runAt} <= ${now}`))
     .groupBy(jobs.queue);
+  const scheduleStates = await getScheduleStates(ctx);
 
   const names = new Set([...Object.keys(getJobsOptions(ctx.config).queues), ...counted.map((row) => row.queue)]);
   return [...names].map((queue) => {
     const counts = Object.fromEntries(JOB_STATUSES.map((status) => [status, 0])) as Record<JobStatus, number>;
     for (const row of counted.filter((candidate) => candidate.queue === queue)) counts[row.status] = row.count;
     const due = backlog.find((row) => row.queue === queue);
-    return { queue, counts, due: due?.due ?? 0, oldestDueAt: due?.oldestDueAt ?? null };
+    return { queue, counts, due: due?.due ?? 0, oldestDueAt: due?.oldestDueAt ?? null, schedule: scheduleStates.get(queue) ?? null };
   });
 }

package/src/server/run-due.ts

@@ -7,6 +7,7 @@ import { consoleJobsLogger, type JobsLogger } from "./logger.js";
 import { getJobsOptions } from "./options.js";
 import { runClaimedJob } from "./process.js";
 import { checkHandlers } from "./handler-check.js";
+import { enqueueDueSchedules } from "./schedules.js";
 
 const DEFAULT_MAX_JOBS = 1000;
 const NEVER_ABORTED = new AbortController().signal;
@@ -33,11 +34,12 @@ export interface RunDueJobsReport {
 }
 
 /**
- * Claims and runs due jobs until none is due (a retried job is due later, so it waits for the clock). Returns what
+ * Enqueues the due runs of the schedules among its queues, then claims and runs due jobs until none is due (a retried job is due later, so it waits for the clock). Returns what
  * happened. Throws on a handler for a queue the config does not define; database failures propagate.
  */
 export async function runDueJobs(ctx: JobsContext, options: RunDueJobsOptions): Promise<RunDueJobsReport> {
   const queues = checkHandlers(getJobsOptions(ctx.config), options.handlers, options.queues);
+  await enqueueDueSchedules(ctx, { queues });
   const report = { succeeded: 0, retried: 0, failed: 0, stale: 0 };
   const maxJobs = options.maxJobs ?? DEFAULT_MAX_JOBS;
   const runOptions = {

package/src/server/schedules.ts

@@ -0,0 +1,121 @@
+// Recurring schedules: each schedule's runs start on whole periods counted from the Unix epoch, so every worker
+// computes the same period. A worker takes a period with a conditional upsert of `jobs.schedules` and enqueues its run
+// in the same transaction, so each period is enqueued once however many workers race for it.
+import { eq, inArray, lt } from "drizzle-orm";
+import { parseEverySeconds, type JobScheduleOptions, type JobsOptions } from "../options.js";
+import { schedules } from "../schema.js";
+import type { JobsContext } from "./context.js";
+import { enqueueJob } from "./enqueue.js";
+import { getJobsOptions } from "./options.js";
+
+/**
+ * The dedup key of every scheduled run: while one run of a schedule is pending or running (retrying a failure, say),
+ * the next period joins it instead of piling up a second run behind it.
+ */
+export const SCHEDULE_DEDUP_KEY = "schedule";
+
+/** A schedule's health as `getQueueStats` reports it. */
+export interface ScheduleState {
+  readonly everySeconds: number;
+  /** The start of the latest period a run was enqueued for; null before any worker saw the schedule. */
+  readonly lastEnqueuedAt: Date | null;
+  /** When a job of the schedule's queue last succeeded; null when none has yet. */
+  readonly lastSucceededAt: Date | null;
+  /**
+   * True when no run succeeded within two periods (counted from when a worker first saw the schedule), or no worker
+   * has seen it at all: no worker runs its queue, or its runs keep failing.
+   */
+  readonly isLate: boolean;
+}
+
+export interface EnqueueDueSchedulesInput {
+  /** Only the schedules of these queues; default every schedule. */
+  readonly queues?: readonly string[];
+}
+
+/** The start of the period `now` falls in. */
+export function getScheduleSlot(schedule: JobScheduleOptions, now: Date): Date {
+  const periodMs = getEverySeconds(schedule) * 1000;
+  return new Date(Math.floor(now.getTime() / periodMs) * periodMs);
+}
+
+/**
+ * Enqueues a run for every schedule whose current period has none yet, and returns their queue names. Missed periods
+ * (no worker ran) collapse into one run for the current period. The worker calls it; an app that drains its queues
+ * from a cron (`runDueJobs`) gets it from there. Database failures propagate.
+ */
+export async function enqueueDueSchedules(ctx: JobsContext, input: EnqueueDueSchedulesInput = {}): Promise<string[]> {
+  const options = getJobsOptions(ctx.config);
+  const enqueued: string[] = [];
+  for (const name of getScheduleNames(options, input.queues)) {
+    const slot = getScheduleSlot(getSchedule(options, name), ctx.clock.now());
+    if (await enqueueScheduledRun(ctx, name, slot)) enqueued.push(name);
+  }
+  return enqueued;
+}
+
+/**
+ * Takes `slot` for the schedule `name` and enqueues its run, in one transaction. False when the schedule already has
+ * this period or a later one (another worker took it). Throws on a name that is not a schedule.
+ */
+export async function enqueueScheduledRun(ctx: JobsContext, name: string, slot: Date): Promise<boolean> {
+  const schedule = getSchedule(getJobsOptions(ctx.config), name);
+  const now = ctx.clock.now();
+  return ctx.db.transaction(async (tx) => {
+    const [taken] = await tx
+      .insert(schedules)
+      .values({ name, lastSlotAt: slot, createdAt: now, updatedAt: now })
+      .onConflictDoUpdate({
+        target: schedules.name,
+        set: { lastSlotAt: slot, updatedAt: now },
+        setWhere: lt(schedules.lastSlotAt, slot),
+      })
+      .returning({ name: schedules.name });
+    if (taken === undefined) return false;
+    await enqueueJob({ ...ctx, db: tx }, { queue: name, payload: schedule.payload, dedupKey: SCHEDULE_DEDUP_KEY });
+    return true;
+  });
+}
+
+/** Notes a succeeded job of a scheduled queue (any job of it, a run enqueued by hand too). */
+export async function recordScheduleSuccess(ctx: JobsContext, queue: string): Promise<void> {
+  if (!Object.hasOwn(getJobsOptions(ctx.config).schedules, queue)) return;
+  const now = ctx.clock.now();
+  await ctx.db.update(schedules).set({ lastSucceededAt: now, updatedAt: now }).where(eq(schedules.name, queue));
+}
+
+/** The state of every configured schedule, by queue name. */
+export async function getScheduleStates(ctx: JobsContext): Promise<Map<string, ScheduleState>> {
+  const options = getJobsOptions(ctx.config);
+  const names = Object.keys(options.schedules);
+  if (names.length === 0) return new Map();
+  const rows = await ctx.db.select().from(schedules).where(inArray(schedules.name, names));
+  const now = ctx.clock.now().getTime();
+  return new Map(
+    names.map((name) => {
+      const everySeconds = getEverySeconds(getSchedule(options, name));
+      const row = rows.find((candidate) => candidate.name === name);
+      const since = row?.lastSucceededAt ?? row?.createdAt ?? null;
+      const isLate = since === null || now - since.getTime() > 2 * everySeconds * 1000;
+      return [name, { everySeconds, lastEnqueuedAt: row?.lastSlotAt ?? null, lastSucceededAt: row?.lastSucceededAt ?? null, isLate }];
+    }),
+  );
+}
+
+/** The schedules among `queues` (default all), in configuration order. */
+export function getScheduleNames(options: JobsOptions, queues: readonly string[] | undefined): string[] {
+  return Object.keys(options.schedules).filter((name) => queues === undefined || queues.includes(name));
+}
+
+export function getSchedule(options: JobsOptions, name: string): JobScheduleOptions {
+  const schedule = Object.hasOwn(options.schedules, name) ? options.schedules[name] : undefined;
+  if (schedule === undefined) throw new Error(`@softure-ai/jobs: "${name}" is not a configured schedule`);
+  return schedule;
+}
+
+function getEverySeconds(schedule: JobScheduleOptions): number {
+  const seconds = parseEverySeconds(schedule.every);
+  // The options schema refused any other value at startup.
+  if (seconds === null) throw new Error(`@softure-ai/jobs: invalid schedule period "${schedule.every}"`);
+  return seconds;
+}

package/src/worker/run-job-worker.ts

@@ -1,6 +1,6 @@
-// The worker loop: claims due jobs of every queue it has a handler for, up to each queue's concurrency, runs them,
-// keeps their leases alive, prunes old jobs once an hour, and on abort stops claiming, waits the grace period and
-// releases what is still running.
+// The worker loop: enqueues the due runs of the schedules among its queues, claims due jobs of every queue it has a
+// handler for, up to each queue's concurrency, runs them, keeps their leases alive, prunes old jobs once an hour, and
+// on abort stops claiming, waits the grace period and releases what is still running.
 import { hostname } from "node:os";
 import { systemClock, type Clock, type SoftureConfig } from "@softure-ai/core";
 import type { Queryable } from "@softure-ai/db";
@@ -12,6 +12,7 @@ import { consoleJobsLogger, type JobsLogger } from "../server/logger.js";
 import { pruneJobs } from "../server/maintenance.js";
 import { getJobsOptions, getQueue } from "../server/options.js";
 import { runClaimedJob } from "../server/process.js";
+import { enqueueScheduledRun, getSchedule, getScheduleNames, getScheduleSlot } from "../server/schedules.js";
 
 const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
 /** A lease is extended three times per lease period, so one slow write does not lose it. */
@@ -60,6 +61,9 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
   const running = new Map<string, Set<RunningJob>>(queues.map((queue) => [queue, new Set()]));
   const wakeUp = createWakeUp();
   let lastPruneAt = Number.NEGATIVE_INFINITY;
+  // Per schedule, the latest period this worker already settled (enqueued, or found taken), so it writes once per period.
+  const settledSlots = new Map<string, number>();
+  const scheduleNames = getScheduleNames(jobsOptions, queues);
 
   const idle = Object.keys(jobsOptions.queues).filter((queue) => !queues.includes(queue));
   if (idle.length > 0 && options.queues === undefined) {
@@ -98,6 +102,17 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
   };
 
   while (!options.signal.aborted) {
+    for (const name of scheduleNames) {
+      const slot = getScheduleSlot(getSchedule(jobsOptions, name), ctx.clock.now());
+      if ((settledSlots.get(name) ?? Number.NEGATIVE_INFINITY) >= slot.getTime()) continue;
+      try {
+        const isEnqueued = await enqueueScheduledRun(ctx, name, slot);
+        settledSlots.set(name, slot.getTime());
+        if (isEnqueued) logger.info("scheduled run enqueued", { queue: name, slot: slot.toISOString() });
+      } catch (error) {
+        logger.error("could not enqueue a scheduled run; trying again next round", { queue: name, error });
+      }
+    }
     for (const queue of queues) {
       const slots = running.get(queue) ?? new Set<RunningJob>();
       const free = getQueue(jobsOptions, queue).concurrency - slots.size;