@ultimat3/jobs 1.2.0 → 3.0.0

package/src/retry-classification.ts

@@ -0,0 +1,112 @@
+// One retry decision from two questions the executor used to ask only half of: "are there
+// attempts left?" (./retry) and "is this error worth trying again at all?" (core's classification).
+// The backoff arithmetic stays in ./retry — nothing here recomputes a delay `nextRetry` owns.
+
+import type { ErrorRetry } from '@ultimat3/core';
+import {
+  DEFAULT_ERROR_RETRY,
+  declaredErrorRetry,
+  isErrorRetry,
+  isUltimateError,
+} from '@ultimat3/core';
+import { toMs } from './clock';
+import type { Random, RetryDecision, RetryPolicy } from './retry';
+import { DEFAULT_RETRY, nextRetry } from './retry';
+
+/** Why this attempt was the last one. Absent while the job is still being retried. */
+export type JobStopReason = 'terminal' | 'attempts-exhausted';
+
+export interface JobRetryDecision extends RetryDecision {
+  readonly stoppedBy: JobStopReason | undefined;
+  /** The classification consulted, or `undefined` when nobody classified the thrown code. */
+  readonly classification: ErrorRetry | undefined;
+}
+
+/**
+ * The classification that was DECLARED for this throw, or `undefined` when there is none.
+ *
+ * Deliberately not `error.retry` alone. That field is `init.retry ?? retryFor(code)` and
+ * `retryFor` fails closed, so every unclassified `UltimateError` already carries `terminal` —
+ * reading it would dead-letter the first attempt of every job in every app whose codes nobody has
+ * classified yet. So `terminal` counts only when it can have come from somewhere: an explicit
+ * per-instance override is indistinguishable from the default here, which is why an UNCLASSIFIED
+ * code carrying an instance `retry: 'terminal'` is read as unclassified. Register the code
+ * (`registerErrorRetry({ X_YOUR_CODE: 'terminal' })`) to have it honoured — one way, and the same
+ * way every other package declares it.
+ */
+export function classifyThrown(error: unknown): ErrorRetry | undefined {
+  if (!isUltimateError(error)) return undefined;
+  const retry: unknown = error.retry;
+  if (!isErrorRetry(retry)) return undefined;
+  // Anything other than the fail-closed default can only have come from the code table or from an
+  // explicit override, so it is somebody's answer either way.
+  if (retry !== DEFAULT_ERROR_RETRY) return retry;
+  return declaredErrorRetry(error.code) === undefined ? undefined : retry;
+}
+
+/**
+ * The delay a `retry-after` error NAMED, in ms. `retryAfterSeconds` on the error's `meta` is the
+ * framework's one spelling for it — `@ultimat3/http`'s `rateLimited` writes it and the 429's
+ * `Retry-After` header renders it — so a job and an HTTP client read the same number.
+ */
+export function statedDelayMs(error: unknown): number | undefined {
+  if (!isUltimateError(error)) return undefined;
+  const seconds: unknown = error.meta?.['retryAfterSeconds'];
+  if (typeof seconds !== 'number' || !Number.isFinite(seconds) || seconds < 0) return undefined;
+  return Math.round(seconds * 1_000);
+}
+
+/**
+ * Retry, dead-letter, and when. `terminal` stops here on the attempt that failed — the same code
+ * run again is the same answer, and the attempts left are a queue slot, a provider bill, and (the
+ * case that forced this) three more wrong passwords at a site that locks the account after three.
+ *
+ * Everything else keeps the attempt count in charge: `retry-after` only replaces the delay, never
+ * the ceiling, and an unclassified code takes exactly the path it took before this existed.
+ */
+export function nextRetryForError(
+  policy: RetryPolicy,
+  attempt: number,
+  error: unknown,
+  random?: Random,
+): JobRetryDecision {
+  const classification = classifyThrown(error);
+  if (classification === 'terminal') {
+    return {
+      retry: false,
+      delayMs: 0,
+      // The policy still decides park-or-drop: `deadLetter: false` means this app does not keep
+      // failed jobs, and that is not a preference a classification gets to overturn.
+      deadLetter: policy.deadLetter ?? true,
+      nextAttempt: attempt,
+      stoppedBy: 'terminal',
+      classification,
+    };
+  }
+
+  const decision = nextRetry(policy, attempt, random);
+  if (!decision.retry) {
+    return { ...decision, stoppedBy: 'attempts-exhausted', classification };
+  }
+  if (classification !== 'retry-after')
+    return { ...decision, stoppedBy: undefined, classification };
+
+  const stated = statedDelayMs(error);
+  if (stated === undefined) return { ...decision, stoppedBy: undefined, classification };
+  // Clamped by the policy's own ceiling, which is what `maxDelay` is for: a responder naming a
+  // day is still a responder this deployment has not agreed to wait a day for.
+  const cap = toMs(policy.maxDelay ?? DEFAULT_RETRY.maxDelay);
+  return { ...decision, delayMs: Math.min(stated, cap), stoppedBy: undefined, classification };
+}
+
+/**
+ * What the job ROW records. `lastError` is the one failure field a row carries, so a dead letter
+ * that stopped at attempt 1 of 5 has to explain itself there or `x jobs show` reads as a silent
+ * early stop. Only the terminal verdict is appended: exhaustion is already legible from
+ * `attempt === maxAttempts`.
+ */
+export function recordedFailure(message: string, decision: JobRetryDecision): string {
+  return decision.stoppedBy === 'terminal'
+    ? `${message} — not retried: this code is classified terminal, so every remaining attempt fails the same way`
+    : message;
+}

package/src/retry.ts

@@ -48,7 +48,12 @@ export function backoffDelayMs(policy: RetryPolicy, attempt: number, random?: Ra
   else raw = base * 2 ** (step - 1);
 
   const capped = Math.min(raw, cap);
-  if (policy.jitter !== true) return Math.round(capped);
+  // `?? DEFAULT_RETRY.jitter`, like every other option above. It read `!== true`, so an omitted
+  // `jitter` meant OFF while the field's own doc says "Equal jitter … by default" — a burst of
+  // failures then retried in lockstep, which is the thundering herd the default exists to break.
+  // Masked for jobs declared through `job()` (it merges the defaults) and live for every direct
+  // caller of this exported function, `retrySchedule` included.
+  if ((policy.jitter ?? DEFAULT_RETRY.jitter) !== true) return Math.round(capped);
   const roll = (random ?? Math.random)();
   return Math.round(capped / 2 + (capped / 2) * roll);
 }

package/src/run-signal.ts

@@ -0,0 +1,50 @@
+// The signal ONE job run is cancelled by: the caller's `Ctx.signal`, the lease heartbeat, and
+// whatever else the worker holds over the run. `AbortSignal.any` composed the same thing and could
+// not be undone — an app wiring a process-lifetime controller into `WorkerOptions.context()` grew
+// one composite per job for the life of the worker, and nothing could abort the result either.
+
+export interface RunSignal {
+  /** Handed to the run as `ctx.signal`; dies with the run. */
+  readonly signal: AbortSignal;
+  /** Cancel this run. The first reason wins, exactly as `AbortController.abort` already does. */
+  abort(reason: unknown): void;
+  /**
+   * Stop following the sources. Idempotent, and it never aborts: a run that settled leaves its
+   * signal in whatever state it ended in, it just stops being the caller's problem.
+   */
+  dispose(): void;
+}
+
+/**
+ * One controller per run, following every source it was given. A source already aborted aborts the
+ * run at composition, carrying its own reason — the same semantics `AbortSignal.any` has, minus
+ * the part that cannot be handed back.
+ *
+ * `undefined` and a non-`AbortSignal` are both skipped rather than refused: `Ctx.signal` is
+ * non-optional in the type and still arrives missing across a cast (`@ultimat3/http`'s `asCtx`, a
+ * test's `{} as Ctx`), and a job that crashed on a missing field is worse than a job with no
+ * caller to follow.
+ */
+export function createRunSignal(sources: readonly (AbortSignal | undefined)[]): RunSignal {
+  const controller = new AbortController();
+  const detach: (() => void)[] = [];
+
+  for (const source of sources) {
+    if (!(source instanceof AbortSignal)) continue;
+    if (source.aborted) {
+      controller.abort(source.reason);
+      continue;
+    }
+    const forward = (): void => controller.abort(source.reason);
+    source.addEventListener('abort', forward, { once: true });
+    detach.push(() => source.removeEventListener('abort', forward));
+  }
+
+  return {
+    signal: controller.signal,
+    abort: (reason) => controller.abort(reason),
+    dispose: () => {
+      for (const off of detach.splice(0)) off();
+    },
+  };
+}

package/src/scheduler-pg.ts

@@ -0,0 +1,103 @@
+// The scheduler's durable halves: the watermark it decides "missed" against, and leader election
+// that survives a pooled connection. Both were `Map`s and a `() => true` in the shipped boot, and
+// both failures are silent — a redeployed pod arms to tomorrow and never reports the 03:00 run the
+// pod it replaced dropped, and two pods in a rolling update both dispatch every task.
+
+import type { Clock } from '@ultimat3/core';
+import { uuid } from '@ultimat3/core';
+import { nowMs } from './clock';
+import type { PgExecutor } from './driver-pg';
+import {
+  SQL_LEADER_ACQUIRE,
+  SQL_LEADER_RELEASE,
+  SQL_SCHEDULER_STATE_GET,
+  SQL_SCHEDULER_STATE_MARK,
+} from './driver-pg-sql';
+import type { LeaderElection, SchedulerState } from './scheduler';
+
+/**
+ * `x_scheduler_state`, one row per task. The watermark is what makes "missed" a decidable
+ * question at all: `catchUp`, `maxCatchUp` and `run-once` are all relative to it, so a scheduler
+ * whose watermark dies with its process has none of them — it takes the arming branch on every
+ * boot and drops every occurrence between the two processes with nothing logged.
+ */
+export function pgSchedulerState(executor: PgExecutor): SchedulerState {
+  return {
+    async lastFiredAt(taskName) {
+      const rows = await executor.query<{ last_fired_at: number | string | null }>(
+        SQL_SCHEDULER_STATE_GET,
+        [taskName],
+      );
+      const value = rows[0]?.last_fired_at;
+      return value === null || value === undefined ? undefined : Number(value);
+    },
+    async markFired(taskName, occurrenceMs) {
+      await executor.query(SQL_SCHEDULER_STATE_MARK, [taskName, occurrenceMs]);
+    },
+  };
+}
+
+export interface PgLeaseLeaderOptions {
+  readonly executor: PgExecutor;
+  /** One key per elected role. Default `'scheduler'`. */
+  readonly lockKey?: string;
+  /** This node's identity. Defaults to a per-process uuid — never a hostname a pod reuses. */
+  readonly holder?: string;
+  /**
+   * How long a grant survives without a renewal. Must be comfortably longer than the scheduler's
+   * tick interval, or a slow round loses the lock to a standby mid-dispatch. Default 30s against
+   * a 1s tick.
+   */
+  readonly ttlMs?: number;
+  readonly clock?: Clock;
+}
+
+export const DEFAULT_LEADER_TTL_MS = 30_000;
+
+/**
+ * Leader election as an expiring row, which is what makes it correct on the executor this package
+ * is actually handed: a POOL. `createPgLeader`'s `pg_try_advisory_lock` is session-scoped, and a
+ * session on a pool ends the moment the connection goes back — so every node reads itself as
+ * leader and a rolling update double-fires every task. `@ultimat3/realtime`'s `PgAdvisoryLock`
+ * solves the same problem by owning its connection; this package holds no wire protocol, so it
+ * solves it with a row instead.
+ *
+ * `acquire()` is also the RENEWAL, so the scheduler calling it every round both keeps the lease
+ * alive and learns the round it stops being leader. A crashed node's lease is reclaimed by expiry
+ * with nothing to clean up, which is the one property the advisory lock had and a plain
+ * `insert ... on conflict do nothing` would not.
+ */
+export function createPgLeaseLeader(options: PgLeaseLeaderOptions): LeaderElection {
+  const lockKey = options.lockKey ?? 'scheduler';
+  const holder = options.holder ?? `scheduler-${uuid()}`;
+  const ttlMs = options.ttlMs ?? DEFAULT_LEADER_TTL_MS;
+  return {
+    async acquire() {
+      const rows = await options.executor.query<{ holder: string }>(SQL_LEADER_ACQUIRE, [
+        lockKey,
+        holder,
+        ttlMs,
+      ]);
+      return rows[0]?.holder === holder;
+    },
+    async release() {
+      await options.executor.query(SQL_LEADER_RELEASE, [lockKey, holder]);
+    },
+  };
+}
+
+/** Exposed so a test — and `x jobs ls` — can say which node currently holds the lease. */
+export async function currentLeader(
+  executor: PgExecutor,
+  lockKey = 'scheduler',
+  clock?: Clock,
+): Promise<string | undefined> {
+  const rows = await executor.query<{ holder: string; expires_at: number | string }>(
+    `select holder, (extract(epoch from expires_at) * 1000)::bigint as expires_at
+       from x_scheduler_leader where lock_key = $1`,
+    [lockKey],
+  );
+  const row = rows[0];
+  if (row === undefined) return undefined;
+  return Number(row.expires_at) > nowMs(clock) ? row.holder : undefined;
+}