@ultimat3/jobs 1.2.0 → 2.0.0

package/src/scheduler.ts

@@ -1,90 +1,30 @@
-// The `task` primitive + the `scheduler` role. A task NEVER does work: it enqueues jobs, so
-// retries, idempotency and observability all come from the job machinery instead of being
-// re-invented per cron.
-//
-// `tz` is required by the type. A cron without a timezone is a bug waiting for March: `0 3 *
-// * *` in a DST-observing zone runs twice or zero times on the switch day, and "the nightly
-// digest went out at 2am and again at 3am" is not a mystery anyone should have to debug.
+// The `scheduler` role: one node walks every registered task's cron, dispatches the occurrences
+// it owes and enqueues their jobs. The `task` primitive it reads lives in `task.ts`.
 //
 // Exactly one node dispatches per tick, enforced by a Postgres advisory lock. Two schedulers
 // double-enqueue every task; the idempotency key would absorb it, but leader election means
-// the queue never sees the duplicate at all.
+// the queue never sees the duplicate at all. One ROUND at a time is the same rule inside one
+// process: the loop re-arms on the round it just finished, and any other caller joins that
+// round rather than opening a second one over the same `lastFiredAt`.
 
 import type { Clock } from '@ultimat3/core';
-import { assert, logger } from '@ultimat3/core';
+import { isUltimateError, logger, onShutdown } from '@ultimat3/core';
 import { instant, nextCronOccurrence } from '@ultimat3/time';
 import { nowMs } from './clock';
-import type { EnqueueResult, JobDriver } from './driver';
-import { JobNameTakenError } from './errors';
-import type { AnyJobHandle } from './job';
-import type { EnqueueOptions } from './outbox';
-
-/** `[[sendDigest, {}]]` — a job handle plus its input. */
-export type TaskEnqueueEntry = readonly [AnyJobHandle, unknown];
+import type { JobDriver } from './driver';
+import type { TaskHandle, TaskJobResult } from './task';
+import { registeredTasks } from './task';
 
 /**
- * What to do when the scheduler was down across one or more occurrences.
- * `skip` (default) waits for the next one; `run-once` fires a single catch-up; `run-all`
- * fires one per missed occurrence, bounded by `maxCatchUp`.
+ * A round that failed, as log fields. `message` alone throws away the half of an `UltimateError`
+ * that makes it actionable — the operator reading `jobs.scheduler.tick-failed` at 3am needs the
+ * stable code to search on and the `fix:` to run, not a sentence.
  */
-export type CatchUpPolicy = 'skip' | 'run-once' | 'run-all';
-
-export interface TaskDefinition {
-  /** Omit it: `defineApi({ tasks })` assigns the export name. Set it only to pin the name the
-   * scheduler's `lastFiredAt` and occurrence lock are already keyed by. */
-  readonly name?: string;
-  readonly cron: string;
-  /** REQUIRED IANA zone, e.g. `'UTC'`, `'America/New_York'`. */
-  readonly tz: string;
-  /**
-   * Builds the entries for ONE occurrence, given that occurrence's instant in epoch ms.
-   *
-   * The argument exists because catch-up does: a tick dispatched late, or replayed for a
-   * missed occurrence, has a wall clock that no longer matches the occurrence being fired.
-   * A payload derived from `Date.now()` there is silently for the wrong day — and the
-   * scheduler's own key is occurrence-scoped, so nothing downstream catches it.
-   */
-  enqueue: (occurrenceMs: number) => readonly TaskEnqueueEntry[];
-  readonly catchUp?: CatchUpPolicy;
-  readonly maxCatchUp?: number;
-}
-
-/** One entry's outcome, from a scheduled dispatch or a manual `task.enqueue()` alike. */
-export interface TaskJobResult {
-  readonly job: string;
-  readonly result: EnqueueResult;
-}
-
-/** JSON-safe view of a task for the manifest, `/_x` and the MCP dev server. */
-export interface TaskDescriptor {
-  readonly kind: 'task';
-  readonly name: string;
-  readonly cron: string;
-  readonly tz: string;
-  readonly catchUp: CatchUpPolicy;
-  readonly maxCatchUp: number;
-  readonly jobs: readonly string[];
-}
-
-export interface TaskHandle {
-  readonly kind: 'task';
-  readonly name: string;
-  readonly cron: string;
-  readonly tz: string;
-  readonly catchUp: CatchUpPolicy;
-  readonly maxCatchUp: number;
-  /**
-   * Entries for `occurrenceMs`. Defaults to now, which is the honest answer for the two
-   * callers that have no occurrence: a manual `task.enqueue()` and `describe()`, which only
-   * wants the job names.
-   */
-  entries(occurrenceMs?: number): readonly TaskEnqueueEntry[];
-  /**
-   * Fire this task's declared entries now, through the same facade `JobHandle.enqueue` uses —
-   * the backfill and "run it again" path, with no scheduler and no leader involved.
-   */
-  enqueue(options?: EnqueueOptions): Promise<readonly TaskJobResult[]>;
-  describe(): TaskDescriptor;
+function failureFields(error: unknown): Record<string, unknown> {
+  const message = error instanceof Error ? error.message : String(error);
+  return isUltimateError(error)
+    ? { error: message, code: error.code, cause: error.cause, fix: error.fix }
+    : { error: message };
 }
 
 /** Resolves the next fire time. Injected so scheduling logic is testable without a cron impl. */
@@ -94,154 +34,13 @@ const defaultCronResolver: CronResolver = (cron, options) =>
   // Instant is a branded Date, so it satisfies CronResolver's Date return directly.
   nextCronOccurrence(cron, options.tz, instant(options.from));
 
-const registry = new Map<string, TaskHandle>();
-let anonymous = 0;
-
-/** Job's store, for tasks: proof `task()` built the handle, plus whether it named itself. */
-interface TaskOrigin {
-  readonly declaredName: boolean;
-  /** The export name already stamped, once one has been. `undefined` while still provisional. */
-  readonly exportName?: string;
-}
-
-const origin = new WeakMap<object, TaskOrigin>();
-
-/**
- * `Intl` carries the runtime's copy of the tz database and rejects anything not in it with a
- * `RangeError`, so it is the only check that can tell `America/Bogota` from `Bogota`.
- */
-function isIanaZone(tz: string): boolean {
-  try {
-    return new Intl.DateTimeFormat('en-US', { timeZone: tz }).resolvedOptions().timeZone.length > 0;
-  } catch {
-    return false;
-  }
-}
-
-export function task(definition: TaskDefinition): TaskHandle {
-  anonymous += 1;
-  const name = definition.name ?? `anonymous-task-${anonymous}`;
-  // Runtime backstop; the type already makes an omitted tz a build error.
-  assert(
-    typeof definition.tz === 'string' && definition.tz.length > 0,
-    `task "${name}" needs an explicit IANA tz — a cron without a timezone is a bug`,
-    `add tz to task("${name}"), e.g. tz: 'UTC' — an unzoned cron silently drifts by an hour at every DST transition`,
-  );
-  // A non-empty string is not a timezone. `tz: 'Bogota'` would otherwise resolve every
-  // occurrence in UTC and the cron would run five hours off, silently, forever.
-  assert(
-    isIanaZone(definition.tz),
-    `task "${name}" has tz "${definition.tz}", which is not a zone in the IANA tz database`,
-    `use the full zone id on task("${name}"), e.g. tz: 'America/Bogota' — list the valid ones with: bun -e "console.log(Intl.supportedValuesOf('timeZone').join('\\n'))"`,
-  );
-
-  const handle: TaskHandle = {
-    kind: 'task',
-    name,
-    cron: definition.cron,
-    tz: definition.tz,
-    catchUp: definition.catchUp ?? 'skip',
-    maxCatchUp: definition.maxCatchUp ?? 10,
-    // `nowMs()` and not `Date.now()`: every reading of time in this package goes through a
-    // Clock so a frozen one cannot be bypassed.
-    entries: (occurrenceMs: number = nowMs()) => definition.enqueue(occurrenceMs),
-    async enqueue(options?: EnqueueOptions): Promise<readonly TaskJobResult[]> {
-      const fired: TaskJobResult[] = [];
-      for (const [handleForJob, input] of handle.entries()) {
-        // The job's PLAIN key, deliberately not `dispatch()`'s `task:occurrence:key`: that one
-        // is occurrence-scoped so two schedulers cannot double-fire the same tick, and reusing
-        // it here would make a manual run dedupe against whichever occurrence it landed in.
-        fired.push({ job: handleForJob.name, result: await handleForJob.enqueue(input, options) });
-      }
-      return fired;
-    },
-    // Reads `handle`, never the captured `name`: `nameTasks()` rebinds the property in place.
-    describe(): TaskDescriptor {
-      return {
-        kind: 'task',
-        name: handle.name,
-        cron: handle.cron,
-        tz: handle.tz,
-        catchUp: handle.catchUp,
-        maxCatchUp: handle.maxCatchUp,
-        // Declaration order: a task's entries are a sequence, not a set.
-        jobs: handle.entries().map(([entry]) => entry.name),
-      };
-    },
-  };
-  origin.set(handle, { declaredName: definition.name !== undefined });
-  // Refused here, not at `registerTask`: a second `task({ name: 'nightly' })` would otherwise
-  // replace the seated handle, and the scheduler's persisted `lastFiredAt` — keyed by that name —
-  // would silently start driving a different cron. The anonymous names cannot collide.
-  if (registry.has(name)) throw new JobNameTakenError({ kind: 'task', name });
-  registry.set(name, handle);
-  return handle;
-}
-
-/** Structural, exactly as `isJobHandle` is: only a handle `task()` built has a cron behind it. */
-export function isTaskHandle(value: unknown): value is TaskHandle {
-  return (
-    typeof value === 'object' &&
-    value !== null &&
-    (value as { kind?: unknown }).kind === 'task' &&
-    origin.has(value)
-  );
-}
-
-/**
- * Register `target` under `name`, stamped onto the handle the module exported — the scheduler's
- * occurrence key is `task:occurrenceMs:jobKey`, so the task's name is what stops two nodes
- * double-firing a tick, and a copy under a second name would defeat it.
- *
- * A definition that supplied its own `name` keeps it, for the same reason a job's does: the
- * scheduler's persisted `lastFiredAt` is keyed by that name.
- */
-export function registerTask(name: string, target: TaskHandle): TaskHandle {
-  const source = origin.get(target);
-  const key = source?.declaredName === true ? target.name : name;
-  const seated = registry.get(key);
-  // The same handle under the same name is one registration seen twice — `defineApi` and the
-  // framework's module scan both reach the same declaration file. A DIFFERENT task under a taken
-  // name is the ambiguity to refuse.
-  if (seated !== undefined) {
-    if (seated !== target) throw new JobNameTakenError({ kind: 'task', name: key });
-    return target;
-  }
-  // One handle exported under two names: the rebind below is in place, so the second alias would
-  // move the occurrence key the scheduler dedupes ticks on.
-  if (source?.exportName !== undefined && source.exportName !== key)
-    throw new JobNameTakenError({ kind: 'task', name: key });
-  registry.delete(target.name);
-  Object.defineProperty(target, 'name', { value: key, configurable: true });
-  if (source !== undefined) origin.set(target, { ...source, exportName: key });
-  registry.set(key, target);
-  return target;
-}
-
-/** `registerTasks(module)` is the call app code makes; this is the same rules over a record. */
-export function nameTasks(record: Readonly<Record<string, TaskHandle>>): void {
-  for (const [exportName, handle] of Object.entries(record)) registerTask(exportName, handle);
-}
-
-export function registeredTasks(): readonly TaskHandle[] {
-  return [...registry.values()].sort((a, b) => a.name.localeCompare(b.name));
-}
-
-export function getTask(name: string): TaskHandle | undefined {
-  return registry.get(name);
-}
-
-export function resetTasks(): void {
-  registry.clear();
-  anonymous = 0;
-}
-
 export interface LeaderElection {
   acquire(): Promise<boolean>;
   release(): Promise<void>;
 }
 
-/** Single-node default: always the leader. Multi-node uses `createPgLeader()`. */
+/** Single-node default: always the leader. Multi-node uses `createPgLeaseLeader()` — never
+ * `createPgLeader()`, whose advisory lock dies with the pooled connection that took it. */
 export function soleLeader(): LeaderElection {
   return {
     acquire: () => Promise.resolve(true),
@@ -272,9 +71,12 @@ export interface SchedulerOptions {
   readonly leader?: LeaderElection;
   readonly state?: SchedulerState;
   readonly cron?: CronResolver;
+  /** Gap between the end of one dispatch round and the start of the next. Default 1s. */
   readonly tickIntervalMs?: number;
   /** Defaults to every registered task. */
   readonly tasks?: readonly TaskHandle[];
+  /** Default true. Registers a SIGTERM drain via `onShutdown`, exactly as the worker does. */
+  readonly drainOnShutdown?: boolean;
 }
 
 export interface DispatchedOccurrence {
@@ -286,19 +88,31 @@ export interface DispatchedOccurrence {
 
 export interface Scheduler {
   start(): void;
-  stop(): Promise<void>;
-  /** One dispatch round. Returns what it enqueued — tests call this, not the timer. */
+  /** Stop dispatching, wait out the round in flight, then hand the leader lock back. */
+  stop(reason?: string): Promise<void>;
+  /**
+   * One dispatch round. Returns what it enqueued — tests call this, not the timer. A call
+   * landing on a round already in flight JOINS that round; there is never a second one.
+   */
   tick(): Promise<readonly DispatchedOccurrence[]>;
   nextRunFor(handle: TaskHandle, from?: Date): Date;
 }
 
 export function createScheduler(options: SchedulerOptions): Scheduler {
-  const state = options.state ?? createMemorySchedulerState();
+  const schedulerState = options.state ?? createMemorySchedulerState();
   const resolveCron = options.cron ?? defaultCronResolver;
   const tickIntervalMs = options.tickIntervalMs ?? 1_000;
   const leader = options.leader ?? soleLeader();
-  let timer: ReturnType<typeof setInterval> | undefined;
+  let timer: ReturnType<typeof setTimeout> | undefined;
   let isLeader = false;
+  /** The drain's state, keyed on the same four values the worker's is. */
+  let state: 'idle' | 'running' | 'draining' | 'stopped' = 'idle';
+  /** The dispatch round in flight — what a second caller joins and what `stop()` waits out. */
+  let round: Promise<readonly DispatchedOccurrence[]> | undefined;
+  /** The `onShutdown` registration this scheduler holds while it runs. Handed back by `stop()`. */
+  let releaseShutdownHook: (() => void) | undefined;
+  /** The teardown in flight, so a SIGTERM landing on a manual stop joins it. */
+  let stopping: Promise<void> | undefined;
 
   const nextRunFor = (handle: TaskHandle, from?: Date): Date =>
     resolveCron(handle.cron, { tz: handle.tz, from: from ?? new Date(nowMs(options.clock)) });
@@ -343,7 +157,7 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
       });
       jobs.push({ job: handleForJob.name, result });
     }
-    await state.markFired(handle.name, occurrenceMs);
+    await schedulerState.markFired(handle.name, occurrenceMs);
     logger.info('jobs.scheduler.dispatched', {
       task: handle.name,
       occurrence: new Date(occurrenceMs).toISOString(),
@@ -354,21 +168,41 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     return { task: handle.name, occurrenceMs, jobs, catchUp };
   };
 
-  const tick = async (): Promise<readonly DispatchedOccurrence[]> => {
-    if (!isLeader) {
-      isLeader = await leader.acquire();
-      if (!isLeader) return [];
+  /** The drain's one question: may this scheduler still dispatch an occurrence? */
+  const dispatching = (): boolean => state !== 'draining' && state !== 'stopped';
+
+  const runRound = async (): Promise<readonly DispatchedOccurrence[]> => {
+    // Never take the lock a drain is on its way to releasing: a round that acquired it here
+    // would still be enqueueing after `stop()` handed the occurrence to the next node.
+    if (!dispatching()) return [];
+    // Asked EVERY round, not only while `isLeader` is false. A lease-backed election (the one a
+    // pooled executor can use — `createPgLeaseLeader`) expires, so `acquire()` is also its
+    // renewal, and a node that cached `isLeader = true` would keep dispatching past a lease
+    // another node has already taken. `soleLeader` answers true every time, and `createPgLeader`
+    // holds its grant internally, so this is a no-op for both.
+    const held = await leader.acquire();
+    if (!held) {
+      // Demoted, or never elected. Nothing to release — a lease we no longer hold is not ours to
+      // hand back, and `teardown` reads this same flag before it calls `release()`.
+      if (isLeader) logger.warn('jobs.scheduler.leadership-lost', { at: nowMs(options.clock) });
+      isLeader = false;
+      return [];
     }
+    isLeader = true;
 
     const at = nowMs(options.clock);
     const tasks = options.tasks ?? registeredTasks();
     const dispatched: DispatchedOccurrence[] = [];
 
     for (const handle of tasks) {
-      const last = await state.lastFiredAt(handle.name);
+      // Re-read per task, not once on entry: a `stop()` between two tasks means stop now, not
+      // at the next round. A task not reached simply fires next time — its `lastFiredAt` is
+      // untouched — while the occurrence this round already began is the one `stop()` waits for.
+      if (!dispatching()) break;
+      const last = await schedulerState.lastFiredAt(handle.name);
       if (last === undefined) {
         // First sight of this task: arm it, never fire retroactively for all of history.
-        await state.markFired(handle.name, nextRunFor(handle, new Date(at)).getTime() - 1);
+        await schedulerState.markFired(handle.name, nextRunFor(handle, new Date(at)).getTime() - 1);
         continue;
       }
 
@@ -382,7 +216,17 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
       }
       if (handle.catchUp === 'run-once') {
         const first = due[0];
-        if (first !== undefined) dispatched.push(await dispatch(handle, first, due.length > 1));
+        if (first !== undefined) {
+          dispatched.push(await dispatch(handle, first, due.length > 1));
+          // `dispatch` leaves the watermark on the occurrence it RAN — the earliest missed one
+          // here — so the next round found occurrences 2..n still due and fired the second, then
+          // the third, one per tick until the backlog drained: 24 nightly digests a second apart
+          // after a day down. "One catch-up" means the rest are DROPPED, and dropping an
+          // occurrence is moving the watermark past it. `at` rather than the last element of
+          // `due`, which `maxCatchUp` truncates: every occurrence at or before `at` is missed by
+          // definition, and this policy fires none of them.
+          if (first !== at) await schedulerState.markFired(handle.name, at);
+        }
         continue;
       }
       for (const occurrence of due) {
@@ -393,24 +237,94 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     return dispatched;
   };
 
+  /**
+   * One round, and never two at once. A round slower than `tickIntervalMs` used to leave the
+   * timer starting a second one over the same `lastFiredAt`: both read the same watermark, both
+   * walked the same occurrences and both dispatched them. The occurrence key deduped the JOBS,
+   * so nothing downstream showed it — but the loser re-marked `lastFiredAt`, reported
+   * occurrences it never enqueued, and under `run-all` interleaved a catch-up sequence with
+   * itself. Joining also gives `stop()` the one promise it has to wait out.
+   */
+  const tick = (): Promise<readonly DispatchedOccurrence[]> => {
+    round ??= runRound().finally(() => {
+      round = undefined;
+    });
+    return round;
+  };
+
+  /**
+   * Re-arms on the round it just finished, never on a fixed period — the interval is the GAP
+   * between rounds, which is what makes overlap impossible at the source rather than caught by
+   * a guard. Cron accuracy does not pay for it: an occurrence is computed from the clock, so a
+   * few ms of drift between rounds moves nothing.
+   */
+  const schedule = (): void => {
+    timer = setTimeout(() => {
+      void tick()
+        .catch((error: unknown) => {
+          logger.error('jobs.scheduler.tick-failed', failureFields(error));
+        })
+        .finally(() => {
+          if (state === 'running') schedule();
+        });
+    }, tickIntervalMs);
+  };
+
+  const teardown = async (reason: string): Promise<void> => {
+    state = 'draining';
+    if (timer !== undefined) clearTimeout(timer);
+    timer = undefined;
+    logger.info('jobs.scheduler.draining', { reason, dispatching: round !== undefined });
+    try {
+      // The round this stop races runs to the end first. Releasing the advisory lock under a
+      // live dispatch hands the next node a task this one is still enqueueing for, and both
+      // then own the same occurrence — the exact double-fire leader election exists to prevent.
+      // Settled, not awaited: a round that failed is its own caller's to see, and the lock still
+      // has to go back.
+      await Promise.allSettled([round]);
+      if (isLeader) await leader.release();
+    } finally {
+      // Whatever the release did, this scheduler is done. `isLeader` false because a lock this
+      // process no longer holds — or failed to hand back — must never be re-used as if it did,
+      // and the hook goes back: one left registered dispatches through a stopped scheduler on
+      // the next process-wide drain, and keeps this closure and its driver alive with it.
+      isLeader = false;
+      state = 'stopped';
+      releaseShutdownHook?.();
+      releaseShutdownHook = undefined;
+    }
+  };
+
+  const stop = async (reason = 'stop'): Promise<void> => {
+    if (state === 'stopped') return;
+    // One teardown, joined rather than repeated — the worker's rule, for the same reason: a
+    // SIGTERM landing on a manual stop must wait out the same round, not release the lock a
+    // second time behind it. Cleared as it settles, so a scheduler started again stops again.
+    stopping ??= teardown(reason).finally(() => {
+      stopping = undefined;
+    });
+    await stopping;
+  };
+
   return {
     start() {
-      if (timer !== undefined) return;
-      timer = setInterval(() => {
-        void tick().catch((error: unknown) => {
-          logger.error('jobs.scheduler.tick-failed', {
-            error: error instanceof Error ? error.message : String(error),
-          });
+      // Only from a standstill. A start mid-drain would re-arm the loop on a lock the drain is
+      // about to release, and stack a second shutdown hook on the one still running.
+      if (state !== 'idle' && state !== 'stopped') return;
+      state = 'running';
+      // 'accept' phase: stop dispatching before core waits on in-flight work — an occurrence
+      // enqueued during the drain is work nothing in this process is left to run. The
+      // unregister is kept, never discarded: `stop()` hands it back, so start -> stop -> start
+      // holds ONE hook rather than one per start.
+      if (options.drainOnShutdown !== false) {
+        releaseShutdownHook = onShutdown('jobs.scheduler', () => stop('SIGTERM'), {
+          phase: 'accept',
         });
-      }, tickIntervalMs);
+      }
+      schedule();
       logger.info('jobs.scheduler.started', { tasks: (options.tasks ?? registeredTasks()).length });
     },
-    async stop() {
-      if (timer !== undefined) clearInterval(timer);
-      timer = undefined;
-      if (isLeader) await leader.release();
-      isLeader = false;
-    },
+    stop,
     tick,
     nextRunFor,
   };