@softure-ai/jobs 0.1.1 → 0.1.3

package/migrations/0003_add_schedule_failures.sql

@@ -0,0 +1,8 @@
+-- The last terminal failure of a scheduled queue, so getQueueStats can tell whether it fails since its last success.
+-- Rollback: ALTER TABLE jobs.schedules DROP COLUMN last_failed_at, DROP COLUMN last_failure_code; then
+--   DELETE FROM softure.migrations WHERE module = 'jobs' AND version = 3;
+ALTER TABLE schedules
+  -- When a job of the schedule's queue last ended `failed`.
+  ADD COLUMN last_failed_at timestamptz,
+  -- That job's error code (`last_error`), never an error's text.
+  ADD COLUMN last_failure_code text;

package/module.json

@@ -1,6 +1,6 @@
 {
   "id": "jobs",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "dependsOn": {},
   "dbSchema": "jobs",
   "tables": ["jobs", "schedules"],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@softure-ai/jobs",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "SOFTURE AI jobs module: background jobs in Postgres. Enqueue inside the caller's transaction with an optional dedup key, claim with FOR UPDATE SKIP LOCKED under a lease, fenced completion, retry with exponential backoff and a terminal failed state, a worker process with graceful shutdown, and an in-process runner for tests.",
   "license": "MIT",
   "type": "module",

package/src/index.ts

@@ -15,7 +15,7 @@ export const MODULE_ID = "jobs";
 export const jobs = defineModule({
   manifest: {
     id: MODULE_ID,
-    version: "0.1.1",
+    version: "0.1.3",
     dependsOn: {},
     dbSchema: "jobs",
     tables: ["jobs", "schedules"],
@@ -47,6 +47,8 @@ export {
   type JobSchedulesInput,
   type JobsOptions,
   type JobsOptionsInput,
+  type LocalTimeScheduleOptions,
+  type PeriodScheduleOptions,
 } from "./options.js";
 export { defineJobHandler, failJob, retryJobLater, type JobHandler, type JobHandlerContext, type JobHandlers } from "./handlers.js";
 export { jobs as jobsTable, schedules as jobSchedulesTable } from "./schema.js";

package/src/options.ts

@@ -1,5 +1,6 @@
 // The options an app passes to `jobs({ ... })` in softure.config.ts, parsed at startup.
 import { z } from "zod";
+import { isTimeZone } from "./zoned-time.js";
 
 const QUEUE_NAME = /^[a-z][a-z0-9_.-]{0,62}$/;
 const QUEUE_NAME_HINT = "is not a queue name: lowercase letters, digits, _ . and -, starting with a letter";
@@ -30,12 +31,57 @@ const backoffSchema = z
     return seconds <= MAX_EVERY_SECONDS ? seconds : null;
   }
 
-  const scheduleSchema = z.strictObject({
-    /** How often the schedule runs: `15m`, `1h`, `1d`. Runs start on whole periods counted from 1970-01-01 00:00 UTC. */
-    every: z.string().refine((every) => parseEverySeconds(every) !== null, { message: EVERY_HINT }),
-    /** The payload of every run; default `{}`. */
-    payload: z.record(z.string(), z.unknown()).default({}),
-  });
+  const AT_TIME = /^([01][0-9]|2[0-3]):[0-5][0-9]$/;
+  const AT_HINT = "must be times of day as HH:MM (00:00 to 23:59), each once";
+  const MAX_AT_TIMES = 48;
+
+  const scheduleSchema = z
+    .strictObject({
+      /** How often the schedule runs: `15m`, `1h`, `1d`. Runs start on whole periods counted from 1970-01-01 00:00 UTC. */
+      every: z.string().refine((every) => parseEverySeconds(every) !== null, { message: EVERY_HINT }).optional(),
+      /** Instead of `every`: the local times of day the schedule runs at, `HH:MM`, in `timeZone`. */
+      at: z
+        .array(z.string().regex(AT_TIME, { message: AT_HINT }))
+        .min(1)
+        .max(MAX_AT_TIMES)
+        .refine((times) => new Set(times).size === times.length, { message: AT_HINT })
+        .optional(),
+      /** The IANA time zone of `at`, such as `Europe/Warsaw`; required with `at`. */
+      timeZone: z.string().refine(isTimeZone, { message: "is not a time zone the runtime knows (an IANA name such as Europe/Warsaw)" }).optional(),
+      /** The payload of every run; default `{}`. */
+      payload: z.record(z.string(), z.unknown()).default({}),
+    })
+    .superRefine((schedule, context) => {
+      if ((schedule.every === undefined) === (schedule.at === undefined)) {
+        context.addIssue({ code: "custom", path: ["every"], message: "set either every or at, not both" });
+      }
+      if (schedule.at !== undefined && schedule.timeZone === undefined) {
+        context.addIssue({ code: "custom", path: ["timeZone"], message: "is required with at" });
+      }
+      if (schedule.every !== undefined && schedule.timeZone !== undefined) {
+        context.addIssue({ code: "custom", path: ["timeZone"], message: "applies to at only; every runs in UTC" });
+      }
+    })
+    .transform((schedule): PeriodScheduleOptions | LocalTimeScheduleOptions => {
+      if (schedule.at !== undefined && schedule.timeZone !== undefined) {
+        return { at: [...schedule.at].sort(), timeZone: schedule.timeZone, payload: schedule.payload };
+      }
+      // The refinement above leaves every set whenever at is not.
+      return { every: schedule.every ?? "", payload: schedule.payload };
+    });
+
+/** A schedule that runs once per period, aligned to UTC. */
+export interface PeriodScheduleOptions {
+  readonly every: string;
+  readonly payload: Record<string, unknown>;
+}
+
+/** A schedule that runs at local times of day in a time zone; `at` is sorted. */
+export interface LocalTimeScheduleOptions {
+  readonly at: readonly string[];
+  readonly timeZone: string;
+  readonly payload: Record<string, unknown>;
+}
 
 const queueSchema = z.strictObject({
   /** Attempts before a job is `failed` for good; the first run counts. */
@@ -65,8 +111,9 @@ export const jobsOptionsSchema = z
     /** The queues, by name. A job is enqueued to one of them and runs with its settings. */
     queues: z.record(z.string(), queueSchema).default({}).superRefine(checkNames),
     /**
-     * Recurring runs, by queue name: the worker enqueues one job per period (`every`) however many workers run. A
-     * schedule's queue takes the default queue settings unless `queues` names it.
+     * Recurring runs, by queue name: the worker enqueues one job per period (`every`) or per local time of day (`at`
+     * in `timeZone`) however many workers run. A schedule's queue takes the default queue settings unless `queues`
+     * names it.
      */
     schedules: z.record(z.string(), scheduleSchema).default({}).superRefine(checkNames),
     /** How long an idle worker waits before it looks for due jobs again. */

package/src/schema.ts

@@ -28,6 +28,8 @@ export const schedules = jobsSchema.table("schedules", {
   name: text("name").primaryKey(),
   lastSlotAt: timestamp("last_slot_at", { withTimezone: true }).notNull(),
   lastSucceededAt: timestamp("last_succeeded_at", { withTimezone: true }),
+  lastFailedAt: timestamp("last_failed_at", { withTimezone: true }),
+  lastFailureCode: text("last_failure_code"),
   createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
   updatedAt: timestamp("updated_at", { withTimezone: true }).notNull(),
 });

package/src/server/claim.ts

@@ -6,7 +6,7 @@ import { jobs } from "../schema.js";
 import type { JobsContext } from "./context.js";
 import { toJob } from "./job-row.js";
 import { getJobsOptions, getQueue } from "./options.js";
-import { recordScheduleSuccess } from "./schedules.js";
+import { recordScheduleFailure, recordScheduleSuccess } from "./schedules.js";
 
 /** A job this worker holds: the job and the token its writes are fenced with. */
 export interface ClaimedJob {
@@ -89,7 +89,8 @@ export async function extendJobLease(ctx: JobsContext, claimed: ClaimedJob): Pro
 }
 
 /**
- * Records the end of an attempt; a success of a scheduled queue also updates its schedule, in the same transaction.
+ * Records the end of an attempt; a success or a terminal failure of a scheduled queue also updates its schedule, in
+ * the same transaction.
  * False when the job is no longer this claim's: nothing was written.
  */
 export async function completeJob(ctx: JobsContext, claimed: ClaimedJob, completion: Completion): Promise<boolean> {
@@ -105,6 +106,7 @@ export async function completeJob(ctx: JobsContext, claimed: ClaimedJob, complet
     const rows = await tx.update(jobs).set(set).where(isHeld(claimed)).returning();
     if (rows.length === 0) return false;
     if (completion.kind === "succeeded") await recordScheduleSuccess({ ...ctx, db: tx }, claimed.job.queue);
+    if (completion.kind === "failed") await recordScheduleFailure({ ...ctx, db: tx }, claimed.job.queue, completion.error);
     return true;
   });
 }
@@ -132,18 +134,22 @@ export async function releaseJob(ctx: JobsContext, claimed: ClaimedJob): Promise
 }
 
 async function failExhaustedLeases(ctx: JobsContext, queue: string, now: Date): Promise<void> {
-  await ctx.db
-    .update(jobs)
-    .set({
-      status: "failed",
-      lastError: LEASE_EXPIRED,
-      finishedAt: now,
-      lockedUntil: null,
-      lockedBy: null,
-      claimToken: null,
-      updatedAt: now,
-    })
-    .where(and(eq(jobs.queue, queue), eq(jobs.status, "running"), lte(jobs.lockedUntil, now), gte(jobs.attempts, jobs.maxAttempts)));
+  await ctx.db.transaction(async (tx) => {
+    const failed = await tx
+      .update(jobs)
+      .set({
+        status: "failed",
+        lastError: LEASE_EXPIRED,
+        finishedAt: now,
+        lockedUntil: null,
+        lockedBy: null,
+        claimToken: null,
+        updatedAt: now,
+      })
+      .where(and(eq(jobs.queue, queue), eq(jobs.status, "running"), lte(jobs.lockedUntil, now), gte(jobs.attempts, jobs.maxAttempts)))
+      .returning({ id: jobs.id });
+    if (failed.length > 0) await recordScheduleFailure({ ...ctx, db: tx }, queue, LEASE_EXPIRED);
+  });
 }
 
 function isHeld(claimed: ClaimedJob) {

package/src/server/schedules.ts

@@ -1,9 +1,10 @@
-// Recurring schedules: each schedule's runs start on whole periods counted from the Unix epoch, so every worker
-// computes the same period. A worker takes a period with a conditional upsert of `jobs.schedules` and enqueues its run
+// Recurring schedules: a period schedule's runs start on whole periods counted from the Unix epoch, a local-time
+// schedule's at its listed times of day in its time zone, so every worker computes the same slot. A worker takes a period with a conditional upsert of `jobs.schedules` and enqueues its run
 // in the same transaction, so each period is enqueued once however many workers race for it.
 import { eq, inArray, lt } from "drizzle-orm";
-import { parseEverySeconds, type JobScheduleOptions, type JobsOptions } from "../options.js";
+import { parseEverySeconds, type JobScheduleOptions, type JobsOptions, type LocalTimeScheduleOptions } from "../options.js";
 import { schedules } from "../schema.js";
+import { getWallClock, getZonedInstant } from "../zoned-time.js";
 import type { JobsContext } from "./context.js";
 import { enqueueJob } from "./enqueue.js";
 import { getJobsOptions } from "./options.js";
@@ -16,16 +17,23 @@ export const SCHEDULE_DEDUP_KEY = "schedule";
 
 /** A schedule's health as `getQueueStats` reports it. */
 export interface ScheduleState {
+  /** The period; for a local-time schedule the longest gap between two of its times of day (24h for one time). */
   readonly everySeconds: number;
   /** The start of the latest period a run was enqueued for; null before any worker saw the schedule. */
   readonly lastEnqueuedAt: Date | null;
   /** When a job of the schedule's queue last succeeded; null when none has yet. */
   readonly lastSucceededAt: Date | null;
+  /** When a job of the schedule's queue last ended `failed` (attempts used up, or failed for good); null when none has. */
+  readonly lastFailedAt: Date | null;
+  /** That job's error code (the handler's code, `jobs.handler_threw`, `jobs.lease_expired`…); null when none failed. */
+  readonly lastFailureCode: string | null;
   /**
    * True when no run succeeded within two periods (counted from when a worker first saw the schedule), or no worker
    * has seen it at all: no worker runs its queue, or its runs keep failing.
    */
   readonly isLate: boolean;
+  /** True when the latest outcome is a failure: `lastFailedAt` is later than `lastSucceededAt`, or nothing succeeded. */
+  readonly isFailing: boolean;
 }
 
 export interface EnqueueDueSchedulesInput {
@@ -33,8 +41,12 @@ export interface EnqueueDueSchedulesInput {
   readonly queues?: readonly string[];
 }
 
-/** The start of the period `now` falls in. */
+/**
+ * The slot `now` falls in: the start of its period, or the latest of the schedule's local times at or before `now`.
+ * Slots only move forward as `now` does, across DST changes too.
+ */
 export function getScheduleSlot(schedule: JobScheduleOptions, now: Date): Date {
+  if ("at" in schedule) return new Date(getLocalTimeSlot(schedule, now.getTime()));
   const periodMs = getEverySeconds(schedule) * 1000;
   return new Date(Math.floor(now.getTime() / periodMs) * periodMs);
 }
@@ -84,6 +96,13 @@ export async function recordScheduleSuccess(ctx: JobsContext, queue: string): Pr
   await ctx.db.update(schedules).set({ lastSucceededAt: now, updatedAt: now }).where(eq(schedules.name, queue));
 }
 
+/** Notes a failed job of a scheduled queue (terminal, not a retried attempt) with its error code. */
+export async function recordScheduleFailure(ctx: JobsContext, queue: string, code: string): Promise<void> {
+  if (!Object.hasOwn(getJobsOptions(ctx.config).schedules, queue)) return;
+  const now = ctx.clock.now();
+  await ctx.db.update(schedules).set({ lastFailedAt: now, lastFailureCode: code, updatedAt: now }).where(eq(schedules.name, queue));
+}
+
 /** The state of every configured schedule, by queue name. */
 export async function getScheduleStates(ctx: JobsContext): Promise<Map<string, ScheduleState>> {
   const options = getJobsOptions(ctx.config);
@@ -97,7 +116,11 @@ export async function getScheduleStates(ctx: JobsContext): Promise<Map<string, S
       const row = rows.find((candidate) => candidate.name === name);
       const since = row?.lastSucceededAt ?? row?.createdAt ?? null;
       const isLate = since === null || now - since.getTime() > 2 * everySeconds * 1000;
-      return [name, { everySeconds, lastEnqueuedAt: row?.lastSlotAt ?? null, lastSucceededAt: row?.lastSucceededAt ?? null, isLate }];
+      const lastSucceededAt = row?.lastSucceededAt ?? null;
+      const lastFailedAt = row?.lastFailedAt ?? null;
+      const isFailing = lastFailedAt !== null && (lastSucceededAt === null || lastFailedAt > lastSucceededAt);
+      const lastFailureCode = row?.lastFailureCode ?? null;
+      return [name, { everySeconds, lastEnqueuedAt: row?.lastSlotAt ?? null, lastSucceededAt, lastFailedAt, lastFailureCode, isLate, isFailing }];
     }),
   );
 }
@@ -114,8 +137,38 @@ export function getSchedule(options: JobsOptions, name: string): JobScheduleOpti
 }
 
 function getEverySeconds(schedule: JobScheduleOptions): number {
+  if ("at" in schedule) return getLongestGapSeconds(schedule.at);
   const seconds = parseEverySeconds(schedule.every);
   // The options schema refused any other value at startup.
   if (seconds === null) throw new Error(`@softure-ai/jobs: invalid schedule period "${schedule.every}"`);
   return seconds;
 }
+
+function getLocalTimeSlot(schedule: LocalTimeScheduleOptions, now: number): number {
+  const today = getWallClock(now, schedule.timeZone);
+  // The calendar day before today's local date; Date.UTC rolls day 0 over to the month before.
+  const yesterday = new Date(Date.UTC(today.year, today.month - 1, today.day - 1));
+  const days = [
+    { year: yesterday.getUTCFullYear(), month: yesterday.getUTCMonth() + 1, day: yesterday.getUTCDate() },
+    { year: today.year, month: today.month, day: today.day },
+  ];
+  let slot = Number.NEGATIVE_INFINITY;
+  for (const day of days) {
+    for (const time of schedule.at) {
+      const [hour, minute] = time.split(":").map(Number);
+      const instant = getZonedInstant({ ...day, hour: hour ?? 0, minute: minute ?? 0 }, schedule.timeZone);
+      if (instant <= now && instant > slot) slot = instant;
+    }
+  }
+  return slot;
+}
+
+/** The longest gap between two consecutive times of day, wrapping past midnight; `times` is sorted. */
+function getLongestGapSeconds(times: readonly string[]): number {
+  const minutes = times.map((time) => {
+    const [hour, minute] = time.split(":").map(Number);
+    return (hour ?? 0) * 60 + (minute ?? 0);
+  });
+  const gaps = minutes.map((minute, index) => (minutes[(index + 1) % minutes.length] ?? minute) - minute + (index === minutes.length - 1 ? 24 * 60 : 0));
+  return Math.max(...gaps) * 60;
+}

package/src/worker/health.ts

@@ -0,0 +1,64 @@
+// The worker's heartbeat: a file the loop touches after each round whose claims reached the database, and the check
+// an image runs (`node worker.mjs --health`) to tell a running loop from a process that only stays alive.
+import { rmSync, statSync, writeFileSync } from "node:fs";
+import type { SoftureConfig } from "@softure-ai/core";
+import type { JobsLogger } from "../server/logger.js";
+import { getJobsOptions } from "../server/options.js";
+
+/** The age limit never drops below this, so a slow round (an hourly prune) does not read as a stuck loop. */
+const MIN_HEALTH_MAX_AGE_MS = 30_000;
+const POLL_INTERVALS_PER_MAX_AGE = 5;
+
+export type JobWorkerHealth = { ok: true; ageMs: number } | { ok: false; reason: "missing" } | { ok: false; reason: "stale"; ageMs: number };
+
+export interface CheckJobWorkerHealthInput {
+  readonly healthFile: string;
+  /** The oldest heartbeat that still counts; `getJobWorkerHealthMaxAgeMs(config)` by default in the process. */
+  readonly maxAgeMs: number;
+  /** Epoch ms to compare the file's modification time with; default `Date.now()`. */
+  readonly now?: number;
+}
+
+/** The default age limit: five poll intervals, at least 30 s. */
+export function getJobWorkerHealthMaxAgeMs(config: SoftureConfig): number {
+  return Math.max(MIN_HEALTH_MAX_AGE_MS, POLL_INTERVALS_PER_MAX_AGE * getJobsOptions(config).pollIntervalMs);
+}
+
+/** Whether the heartbeat file exists and is no older than `maxAgeMs`. Reads the file system only. */
+export function checkJobWorkerHealth(input: CheckJobWorkerHealthInput): JobWorkerHealth {
+  let mtimeMs: number;
+  try {
+    mtimeMs = statSync(input.healthFile).mtimeMs;
+  } catch {
+    return { ok: false, reason: "missing" };
+  }
+  const ageMs = Math.max(0, Math.round((input.now ?? Date.now()) - mtimeMs));
+  return ageMs > input.maxAgeMs ? { ok: false, reason: "stale", ageMs } : { ok: true, ageMs };
+}
+
+/** Writes and removes the heartbeat file; a failure is logged once until a write succeeds again, never thrown. */
+export function createHealthFile(healthFile: string, logger: JobsLogger): { touch: () => void; remove: () => void } {
+  let hasFailed = false;
+  const report = (operation: string, error: unknown) => {
+    if (hasFailed) return;
+    hasFailed = true;
+    logger.error(`could not ${operation} the worker's health file`, { healthFile, error });
+  };
+  return {
+    touch: () => {
+      try {
+        writeFileSync(healthFile, new Date().toISOString());
+        hasFailed = false;
+      } catch (error) {
+        report("write", error);
+      }
+    },
+    remove: () => {
+      try {
+        rmSync(healthFile, { force: true });
+      } catch (error) {
+        report("remove", error);
+      }
+    },
+  };
+}

package/src/worker/index.ts

@@ -1,4 +1,5 @@
-// The worker of @softure-ai/jobs: the loop (`runJobWorker`) and the process entry an app's `scripts/worker.ts` calls
-// (`runJobWorkerProcess`).
+// The worker of @softure-ai/jobs: the loop (`runJobWorker`), the process entry an app's `scripts/worker.ts` calls
+// (`runJobWorkerProcess`) and the check of its heartbeat (`checkJobWorkerHealth`).
+export { checkJobWorkerHealth, getJobWorkerHealthMaxAgeMs, type CheckJobWorkerHealthInput, type JobWorkerHealth } from "./health.js";
 export { runJobWorkerProcess, type RunJobWorkerProcessOptions } from "./process.js";
 export { runJobWorker, type RunJobWorkerOptions } from "./run-job-worker.js";

package/src/worker/process.ts

@@ -1,17 +1,21 @@
 // The worker as a process: what an app's `scripts/worker.ts` calls. It opens its own database pool, turns SIGTERM and
-// SIGINT into a graceful stop, closes the pool and reports an exit code.
+// SIGINT into a graceful stop, closes the pool and reports an exit code. With `--health` it checks the heartbeat file
+// of a running worker instead.
 import { errorLogLabel, type SoftureConfig } from "@softure-ai/core";
 import { openCommandDatabase, type Database } from "@softure-ai/db";
 import type { JobHandlers } from "../handlers.js";
 import { consoleJobsLogger, type JobsLogger } from "../server/logger.js";
 import { getJobsOptions } from "../server/options.js";
+import { checkJobWorkerHealth, getJobWorkerHealthMaxAgeMs } from "./health.js";
 import { runJobWorker, type RunJobWorkerOptions } from "./run-job-worker.js";
 
 const EXIT_OK = 0;
 const EXIT_FAILED = 1;
 const STOP_SIGNALS = ["SIGTERM", "SIGINT"] as const;
+const HEALTH_FLAG = "--health";
 
-export interface RunJobWorkerProcessOptions extends Partial<Pick<RunJobWorkerOptions, "queues" | "clock" | "workerId" | "random" | "heartbeatMs">> {
+export interface RunJobWorkerProcessOptions
+  extends Partial<Pick<RunJobWorkerOptions, "queues" | "clock" | "workerId" | "random" | "heartbeatMs" | "healthFile">> {
   readonly config: SoftureConfig;
   readonly handlers: JobHandlers;
   readonly logger?: JobsLogger;
@@ -19,11 +23,17 @@ export interface RunJobWorkerProcessOptions extends Partial<Pick<RunJobWorkerOpt
   readonly database?: Database;
   /** Stops the worker instead of SIGTERM and SIGINT (tests). */
   readonly signal?: AbortSignal;
+  /** The oldest heartbeat `--health` accepts; default `getJobWorkerHealthMaxAgeMs(config)` (five poll intervals, at least 30 s). */
+  readonly healthMaxAgeMs?: number;
+  /** The process arguments; default `process.argv.slice(2)`. `--health` checks `healthFile` instead of running. */
+  readonly argv?: readonly string[];
 }
 
 /**
  * Runs the worker until SIGTERM or SIGINT, then stops gracefully. Resolves to the exit code and sets
- * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened.
+ * `process.exitCode`: 0 after a clean stop, 1 when it could not start or the database could not be opened. Started
+ * with `--health`, it opens nothing and checks the running worker's `healthFile` instead: 0 when its loop ran within
+ * `healthMaxAgeMs`, else 1.
  *
  *   // scripts/worker.ts
  *   import { runJobWorkerProcess } from "@softure-ai/jobs/worker";
@@ -32,11 +42,29 @@ export interface RunJobWorkerProcessOptions extends Partial<Pick<RunJobWorkerOpt
  *   await runJobWorkerProcess({ config, handlers });
  */
 export async function runJobWorkerProcess(options: RunJobWorkerProcessOptions): Promise<number> {
-  const exitCode = await runProcess(options);
+  const argv = options.argv ?? process.argv.slice(2);
+  const exitCode = argv.includes(HEALTH_FLAG) ? checkHealth(options) : await runProcess(options);
   process.exitCode = exitCode;
   return exitCode;
 }
 
+function checkHealth(options: RunJobWorkerProcessOptions): number {
+  const logger = options.logger ?? consoleJobsLogger;
+  if (options.healthFile === undefined) {
+    logger.error(`${HEALTH_FLAG} needs the worker's healthFile option; pass healthFile to runJobWorkerProcess`);
+    return EXIT_FAILED;
+  }
+  const maxAgeMs = options.healthMaxAgeMs ?? getJobWorkerHealthMaxAgeMs(options.config);
+  const health = checkJobWorkerHealth({ healthFile: options.healthFile, maxAgeMs });
+  if (health.ok) {
+    logger.info(`the worker's loop ran ${String(health.ageMs)} ms ago`);
+    return EXIT_OK;
+  }
+  if (health.reason === "missing") logger.error(`the worker's health file is missing: ${options.healthFile}`);
+  else logger.error(`the worker's loop last ran ${String(health.ageMs)} ms ago (limit ${String(maxAgeMs)} ms)`);
+  return EXIT_FAILED;
+}
+
 async function runProcess(options: RunJobWorkerProcessOptions): Promise<number> {
   const logger = options.logger ?? consoleJobsLogger;
   const database = options.config.database;

package/src/worker/run-job-worker.ts

@@ -13,6 +13,7 @@ import { pruneJobs } from "../server/maintenance.js";
 import { getJobsOptions, getQueue } from "../server/options.js";
 import { runClaimedJob } from "../server/process.js";
 import { enqueueScheduledRun, getSchedule, getScheduleNames, getScheduleSlot } from "../server/schedules.js";
+import { createHealthFile } from "./health.js";
 
 const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
 /** A lease is extended three times per lease period, so one slow write does not lose it. */
@@ -35,6 +36,11 @@ export interface RunJobWorkerOptions {
   readonly random?: () => number;
   /** How often a running job's lease is extended; default a third of the queue's `leaseSeconds`. */
   readonly heartbeatMs?: number;
+  /**
+   * A file the loop touches after each round whose claims reached the database (removed at start and after a clean
+   * stop), for a health check of the loop: `checkJobWorkerHealth`. Default none.
+   */
+  readonly healthFile?: string;
 }
 
 interface RunningJob {
@@ -69,6 +75,9 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
   if (idle.length > 0 && options.queues === undefined) {
     logger.warn("configured queues without a handler are not run by this worker", { queues: idle.join(", ") });
   }
+  const healthFile = options.healthFile === undefined ? undefined : createHealthFile(options.healthFile, logger);
+  // A file left by an earlier run of the same container must not pass the check before this loop ran a round.
+  healthFile?.remove();
   logger.info("worker started", { workerId, queues: queues.join(", ") });
 
   const start = (claimed: ClaimedJob): RunningJob => {
@@ -113,6 +122,7 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
         logger.error("could not enqueue a scheduled run; trying again next round", { queue: name, error });
       }
     }
+    let isClaimFailed = false;
     for (const queue of queues) {
       const slots = running.get(queue) ?? new Set<RunningJob>();
       const free = getQueue(jobsOptions, queue).concurrency - slots.size;
@@ -121,6 +131,7 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
         const claimed = await claimJobs(ctx, { queue, limit: free, workerId });
         for (const job of claimed) slots.add(start(job));
       } catch (error) {
+        isClaimFailed = true;
         logger.error("could not claim jobs", { queue, error });
       }
     }
@@ -133,10 +144,12 @@ export async function runJobWorker(options: RunJobWorkerOptions): Promise<void>
         logger.error("could not prune finished jobs", { error });
       }
     }
+    if (!isClaimFailed) healthFile?.touch();
     await wakeUp.wait(jobsOptions.pollIntervalMs, options.signal);
   }
 
   await stop(running, { ctx, logger, graceMs: jobsOptions.shutdownGraceMs });
+  healthFile?.remove();
   logger.info("worker stopped", { workerId });
 }
 

package/src/zoned-time.ts

@@ -0,0 +1,75 @@
+// Wall-clock time in an IANA time zone, with the platform's Intl data and no date library.
+
+const HOUR_MS = 60 * 60 * 1000;
+const formatters = new Map<string, Intl.DateTimeFormat>();
+
+/** A date and a time of day as a clock in some time zone shows them; `month` counts from 1. */
+export interface WallClock {
+  readonly year: number;
+  readonly month: number;
+  readonly day: number;
+  readonly hour: number;
+  readonly minute: number;
+}
+
+/** True when `timeZone` is an IANA zone the runtime knows, such as `Europe/Warsaw` or `UTC`. */
+export function isTimeZone(timeZone: string): boolean {
+  try {
+    getFormatter(timeZone);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** The wall clock of `instant` in `timeZone`, to the minute. */
+export function getWallClock(instant: number, timeZone: string): WallClock {
+  const parts = Object.fromEntries(getFormatter(timeZone).formatToParts(new Date(instant)).map((part) => [part.type, part.value]));
+  return {
+    year: Number(parts.year),
+    month: Number(parts.month),
+    day: Number(parts.day),
+    hour: Number(parts.hour),
+    minute: Number(parts.minute),
+  };
+}
+
+/**
+ * The instant a clock in `timeZone` shows `wallClock`. A time that occurs twice (the hour a DST change repeats) gives
+ * its first occurrence; a time that does not occur (the hour a DST change skips) gives the instant the same distance
+ * after the change, so 02:30 on a day that jumps from 02:00 to 03:00 gives 03:30.
+ */
+export function getZonedInstant(wallClock: WallClock, timeZone: string): number {
+  const asUtc = toUtcMs(wallClock);
+  // One DST change a day at most: the offsets half a day before and after cover both sides of it.
+  const offsetBefore = getOffsetMs(asUtc - 12 * HOUR_MS, timeZone);
+  const offsetAfter = getOffsetMs(asUtc + 12 * HOUR_MS, timeZone);
+  const matching = [asUtc - offsetBefore, asUtc - offsetAfter].filter((instant) => toUtcMs(getWallClock(instant, timeZone)) === asUtc);
+  return matching.length > 0 ? Math.min(...matching) : asUtc - offsetBefore;
+}
+
+function getOffsetMs(instant: number, timeZone: string): number {
+  const minute = Math.floor(instant / 60_000) * 60_000;
+  return toUtcMs(getWallClock(minute, timeZone)) - minute;
+}
+
+function toUtcMs(wallClock: WallClock): number {
+  return Date.UTC(wallClock.year, wallClock.month - 1, wallClock.day, wallClock.hour, wallClock.minute);
+}
+
+function getFormatter(timeZone: string): Intl.DateTimeFormat {
+  let formatter = formatters.get(timeZone);
+  if (formatter === undefined) {
+    formatter = new Intl.DateTimeFormat("en-US", {
+      timeZone,
+      hourCycle: "h23",
+      year: "numeric",
+      month: "numeric",
+      day: "numeric",
+      hour: "numeric",
+      minute: "numeric",
+    });
+    formatters.set(timeZone, formatter);
+  }
+  return formatter;
+}