@ultimat3/jobs 29.2.0 → 29.3.0

package/README.md

@@ -1069,6 +1069,28 @@ unlabelled gauges, `job_worker_in_flight` and `job_worker_capacity`, each the SU
 `ceil((queue_depth + Σ job_worker_in_flight) / job_worker_capacity)`. A fleet-wide ceiling is a
 different number: `maxInFlight` × replicas, or `job.concurrency` for one job.
 
+**A scaler reading Postgres** (KEDA's `postgresql` trigger) runs the statement below — the
+package's `scaler-depth-fixture.ts`, proved against a real Postgres (`scaler-depth.job.test.ts`)
+and pinned to this block (`scaler-depth-readme.test.ts`); KEDA binds
+nothing, so the queues are inlined as `'{a,b}'::text[]`: running jobs plus due,
+claimable ones — minus the jobs a **draining** worker still holds. A worker that stops claiming (a
+retire, a SIGTERM, `stop()`) announces at once with `draining`, and its registry row keeps
+`draining_at` from that first announce (`workers()` → `drainingAt`). Those jobs are finished by
+the worker holding them; counted, a deploy scaled the new pods up for the old pods' runs for as
+long as the longest of them took. A claimer whose row lapsed is counted again — the queue hands the
+job back. `As of 2026-10-30`.
+
+```sql
+select count(*)::int as depth
+  from x_jobs j
+ where j.queue = any('{banks,banks-long}'::text[])
+   and ((j.state = 'running'
+         and not exists (select 1 from x_job_workers w where w.id = j.claimed_by
+                            and w.draining_at is not null and w.expires_at > now()))
+     or (j.state in ('ready', 'delayed', 'suspended') and j.run_at <= now()
+         and not exists (select 1 from x_job_pauses p where p.kind = 'queue' and p.name = j.queue)))
+```
+
 ### One run per key
 
 A number caps the whole job. `{ key, limit, whenBusy }` caps it **per key** — "at most one run
@@ -1147,6 +1169,8 @@ driven by the frozen clock. Renewal failures are not swallowed:
 |---|---|
 | a renewal failed, the window still has room | `jobs.heartbeat.failed` (warn) |
 | a whole window passed with none landing | `jobs.lease.lost` (error) + `job_leases_lost_total{queue}` |
+| the row refused the renewal and another worker holds it (or it cannot be read) | `jobs.lease.lost` (error, `reason: not-ours`) + the counter |
+| the row refused the renewal because it was **cancelled** (`x jobs cancel`, an app's stop) | `jobs.lease.cancelled` (info), the attempt aborted the same way — never counted |
 
 The second one means the queue is free to hand that job to another worker while this one is
 still running it — at-least-once turning into twice. Alert on any non-zero rate. The window is
@@ -1160,6 +1184,11 @@ already **on the wire**, not only for the next one: a clean completion acks the
 pool was slow enough. The fleet slot's `jobs.worker.slot-lost` had the same shape and the same
 fix (`renewal-timer.ts`).
 
+Nor for a job being settled (`As of 2026-10-30`): the heartbeat stops when the worker SENDS the
+`ack` / `nack`, not when `executeJob` returns. In between ran the `onSettled` hook and the step
+listing, and a renewal there matched no row — `jobs.lease.lost` for a job that had finished, with
+the hook's `ctx.signal` aborted under it.
+
 ## The operator surface
 
 Every operator capability is a member of `JobIntrospection` (`driver.introspect`), implemented by
@@ -1177,7 +1206,7 @@ the pg and the memory driver alike. `x jobs`, `/_x` and a dashboard read nothing
 | `promote(id)` | a job waiting on its `runAt` — delayed, or backing off — is due now | — |
 | `pauseQueue(q)` / `resumeQueue(q)` / `pausedQueues()` | a paused queue is claimed by no worker; enqueues still land | effective at every worker's next claim |
 | `pauseTask(t)` / `resumeTask(t)` / `pausedTasks()` | a paused task is not dispatched; on resume its own `catchUp` decides what it missed | — |
-| `workers()` | every live worker: id, host, started, queues, slots, in-flight job ids, last heartbeat | rows expire one visibility timeout after the last heartbeat; `MAX_WORKER_IN_FLIGHT` = 100 ids |
+| `workers()` | every live worker: id, host, started, queues, slots, in-flight job ids, last heartbeat, `drainingAt` once it drains | rows expire one visibility timeout after the last heartbeat; `MAX_WORKER_IN_FLIGHT` = 100 ids |
 | `counters({ job, sinceMs })` / `counterTotals(sinceMs)` | per-job history: `done`, `retried`, `failed`, `dead`, `durationMs` per bucket | `COUNTER_TIERS`: 1-minute buckets for 24 h, 5-minute for 7 d, 1-hour for 30 d — 4,176 rows per job name at most |
 | `rollupCounters()` | fold buckets past their tier into the next; the scheduler leader calls it once a minute | idempotent |
 | `recordProgress(id, { workerId, claim }, progress)` | what `run({ progress })` writes | at most one write per `PROGRESS_INTERVAL_MS` (1 s) per run, plus the last before the settle |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/jobs",
-  "version": "29.2.0",
+  "version": "29.3.0",
   "description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
   "license": "MIT",
   "type": "module",
@@ -33,10 +33,10 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "29.2.0",
-    "@ultimat3/db": "29.2.0",
-    "@ultimat3/entity": "29.2.0",
-    "@ultimat3/schema": "29.2.0",
-    "@ultimat3/time": "29.2.0"
+    "@ultimat3/core": "29.3.0",
+    "@ultimat3/db": "29.3.0",
+    "@ultimat3/entity": "29.3.0",
+    "@ultimat3/schema": "29.3.0",
+    "@ultimat3/time": "29.3.0"
   }
 }

package/src/driver-memory-operator.ts

@@ -273,8 +273,13 @@ export function memoryOperator(state: MemoryQueueState): MemoryOperator {
 
     announceWorker(worker, ttlMs) {
       const at = nowMs(clock);
+      // The first draining announce stamps it; later ones keep it — `SQL_WORKER_ANNOUNCE`'s rule.
+      const drainingAt =
+        worker.draining === true ? (workers.get(worker.id)?.drainingAt ?? at) : undefined;
+      const { draining: _draining, ...announced } = worker;
       workers.set(worker.id, {
-        ...worker,
+        ...announced,
+        ...(drainingAt === undefined ? {} : { drainingAt }),
         queues: [...worker.queues],
         inFlight: worker.inFlight.slice(0, MAX_WORKER_IN_FLIGHT),
         heartbeatAt: at,

package/src/driver-pg-ddl.ts

@@ -256,6 +256,12 @@ create table if not exists x_job_workers (
   expires_at   timestamptz not null
 );
 
+-- When the worker began draining (a retire, a SIGTERM, a stop) - set by its first draining
+-- announce and kept by every later one, null while it serves. A queue-depth autoscaler leaves the
+-- jobs such a worker still holds out of its count (SQL_SCALER_DEPTH): its replacement cannot take
+-- them. Added by alter because x_job_workers shipped without it.
+alter table x_job_workers add column if not exists draining_at timestamptz;
+
 -- Per-job history, in buckets. A settle adds to the one-minute bucket in the SAME statement that
 -- moves the row, and the scheduler leader folds old buckets into wider ones, so the key count per
 -- job name is fixed whatever the throughput.

package/src/driver-pg-operator-sql.ts

@@ -148,18 +148,21 @@ select name, (extract(epoch from paused_at) * 1000)::bigint as paused_at
 
 /**
  * A worker's heartbeat: the whole row rewritten, expiry pushed out. $1 id, $2 host, $3 queues,
- * $4 slots, $5 in-flight ids, $6 started (ms), $7 ttl (ms).
+ * $4 slots, $5 in-flight ids, $6 started (ms), $7 ttl (ms), $8 draining. `draining_at` is stamped by
+ * the first draining announce and KEPT by the later ones — when the drain began, not the last beat.
  */
 export const SQL_WORKER_ANNOUNCE = `
 insert into x_job_workers
-  (id, host, queues, concurrency, in_flight, started_at, heartbeat_at, expires_at)
+  (id, host, queues, concurrency, in_flight, started_at, heartbeat_at, expires_at, draining_at)
 values
   ($1, $2, $3::jsonb, $4::int, $5::jsonb, to_timestamp($6::bigint / 1000.0), now(),
-   now() + ($7::bigint * interval '1 millisecond'))
+   now() + ($7::bigint * interval '1 millisecond'), case when $8::boolean then now() end)
 on conflict (id) do update
    set host = excluded.host, queues = excluded.queues, concurrency = excluded.concurrency,
        in_flight = excluded.in_flight, heartbeat_at = excluded.heartbeat_at,
-       expires_at = excluded.expires_at
+       expires_at = excluded.expires_at,
+       draining_at = case when $8::boolean
+                          then coalesce(x_job_workers.draining_at, excluded.draining_at) end
 `.trim();
 
 export const SQL_WORKER_FORGET = `
@@ -176,7 +179,8 @@ with lapsed as (
 )
 select id, host, queues, concurrency, in_flight,
        (extract(epoch from started_at)   * 1000)::bigint as started_at,
-       (extract(epoch from heartbeat_at) * 1000)::bigint as heartbeat_at
+       (extract(epoch from heartbeat_at) * 1000)::bigint as heartbeat_at,
+       (extract(epoch from draining_at)  * 1000)::bigint as draining_at
   from x_job_workers
  where expires_at > now()
  order by started_at, id

package/src/driver-pg-operator.ts

@@ -251,6 +251,7 @@ export function pgOperator(
         JSON.stringify(worker.inFlight.slice(0, MAX_WORKER_IN_FLIGHT)),
         worker.startedAt,
         finiteOption('the pg worker registry', 'ttlMs', ttlMs),
+        worker.draining === true,
       ]);
     },
     async forgetWorker(workerId) {
@@ -265,6 +266,7 @@ export function pgOperator(
         in_flight: unknown;
         started_at: Count;
         heartbeat_at: Count;
+        draining_at: Count | null;
       }>(SQL_WORKERS, []);
       return rows.map((row) => ({
         id: row.id,
@@ -274,6 +276,7 @@ export function pgOperator(
         concurrency: num(row.concurrency),
         inFlight: stringList(row.in_flight),
         heartbeatAt: num(row.heartbeat_at),
+        ...(row.draining_at === null ? {} : { drainingAt: num(row.draining_at) }),
       }));
     },
 

package/src/heartbeat.ts

@@ -6,13 +6,20 @@
 import type { Clock } from '@ultimat3/core';
 import { logger, recordLeaseLost, renderThrowable } from '@ultimat3/core';
 import { nowMs } from './clock';
-import type { ClaimedJob, JobDriver } from './driver';
+import type { ClaimedJob, JobDriver, JobRecord } from './driver';
 import { LeaseLostError } from './errors';
+import type { JobIntrospection } from './introspection';
 import { type IntervalScheduler, startRenewalTimer } from './renewal-timer';
 
 export interface LeaseHeartbeatOptions {
-  /** Only `heartbeat` is used — a lease renews itself and settles nothing. */
-  readonly driver: Pick<JobDriver, 'heartbeat'>;
+  /**
+   * `heartbeat` renews; `introspect.job` is read ONLY when a renewal is refused, to tell a cancel
+   * (the designed stop) from a lease another worker now holds. A lease renews itself and settles
+   * nothing.
+   */
+  readonly driver: Pick<JobDriver, 'heartbeat'> & {
+    readonly introspect?: Pick<JobIntrospection, 'job'> | undefined;
+  };
   readonly claimed: ClaimedJob;
   readonly visibilityTimeoutMs: number;
   readonly intervalMs: number;
@@ -22,6 +29,9 @@ export interface LeaseHeartbeatOptions {
   readonly schedule?: IntervalScheduler;
 }
 
+/** The longest a refused renewal waits on the row read that names it a cancel or a loss. */
+export const CLASSIFY_READ_MAX_MS = 5_000;
+
 export interface LeaseHeartbeat {
   /** One renewal. The interval drives it; tests drive it instead of the timer. */
   renew(): Promise<void>;
@@ -70,6 +80,13 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
     if (lost || timer.stopped()) return;
     lost = true;
     timer.stop();
+    logLost(error, reason);
+    // Cancel LAST, so the loss is logged and counted before the body it unwinds starts throwing.
+    gone.abort(new LeaseLostError({ job: claimed.name, jobId: claimed.id }));
+  };
+
+  /** `jobs.lease.lost` at error, counted: this job may now run twice. */
+  const logLost = (error?: unknown, reason?: 'expired' | 'not-ours'): void => {
     logger.error('jobs.lease.lost', {
       workerId,
       job: claimed.name,
@@ -81,8 +98,51 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
       ...(error === undefined ? {} : { error: renderThrowable(error) }),
     });
     recordLeaseLost(claimed.queue);
-    // Cancel LAST, so the loss is logged and counted before the body it unwinds starts throwing.
+  };
+
+  /**
+   * The row said "not yours" — the driver answered, so ownership is GONE whatever the cause: the
+   * attempt is aborted now, before anything else is awaited. Only then is the row read once, to say
+   * which: a cancel (`x jobs cancel`, an app's own stop) is the designed way a stop reaches a
+   * running attempt — `jobs.lease.cancelled` at info, never counted, since `job_leases_lost_total`
+   * means "this job may now run twice" and a cancel cannot. Anything else, an unreadable row
+   * included, is `jobs.lease.lost`. Read after the abort, a stalled read delays a log line, never
+   * the stop (a stalled read used to keep the attempt running until the window lapsed) — and only
+   * for one renewal interval (at most `CLASSIFY_READ_MAX_MS`): a read still out then is unreadable,
+   * so a real loss is never left uncounted behind a read that never answers.
+   */
+  const refused = async (): Promise<void> => {
+    if (lost || timer.stopped()) return;
+    lost = true;
+    timer.stop();
     gone.abort(new LeaseLostError({ job: claimed.name, jobId: claimed.id }));
+    const row = await boundedRead();
+    if (row?.state !== 'cancelled') {
+      logLost(undefined, 'not-ours');
+      return;
+    }
+    logger.info('jobs.lease.cancelled', {
+      workerId,
+      job: claimed.name,
+      jobId: claimed.id,
+      queue: claimed.queue,
+      attempt: claimed.attempt,
+    });
+  };
+
+  /** `introspect.job`, or `undefined` once it rejects or outlasts its bound. */
+  const boundedRead = async (): Promise<JobRecord | undefined> => {
+    const read = options.driver.introspect?.job(claimed.id);
+    if (read === undefined) return undefined;
+    let bound: ReturnType<typeof setTimeout> | undefined;
+    const outlasted = new Promise<undefined>((resolve) => {
+      bound = setTimeout(resolve, Math.min(options.intervalMs, CLASSIFY_READ_MAX_MS), undefined);
+    });
+    try {
+      return await Promise.race([read.catch((): undefined => undefined), outlasted]);
+    } finally {
+      clearTimeout(bound);
+    }
   };
 
   const renew = async (): Promise<void> => {
@@ -121,7 +181,7 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
       // before that — a hand-rolled one, a test double — resolves `undefined`. Reading that as a
       // lost lease would cancel every job on every renewal. Only an explicit "no" is a no.
       if (held === false) {
-        reportLost(undefined, 'not-ours');
+        await refused();
         return;
       }
       // Asked again AFTER the call, because a renewal that SUCCEEDS late is still late: an

package/src/introspection.ts

@@ -139,11 +139,21 @@ export interface WorkerAnnouncement {
   readonly concurrency: number;
   /** Job ids it holds right now — the first `MAX_WORKER_IN_FLIGHT` of them. */
   readonly inFlight: readonly string[];
+  /**
+   * The worker stopped claiming — a retire, a SIGTERM's drain or a `stop()` — and only finishes
+   * what it holds. Absent: serving.
+   */
+  readonly draining?: boolean;
 }
 
 export interface WorkerRecord extends WorkerAnnouncement {
   /** Epoch ms of the announcement that wrote this row. */
   readonly heartbeatAt: number;
+  /**
+   * Epoch ms of the FIRST announcement that said `draining` — kept by every later one. Absent while
+   * the worker serves. What `SQL_SCALER_DEPTH` excludes a worker's held jobs by.
+   */
+  readonly drainingAt?: number;
 }
 
 /**

package/src/worker-registry.ts

@@ -29,6 +29,13 @@ export interface WorkerRegistryOptions {
 }
 
 export interface WorkerRegistration {
+  /**
+   * The worker stopped claiming and only finishes what it holds: every announce from now on says
+   * `draining`, and one goes out NOW rather than at the next heartbeat (a third of the visibility
+   * window later) — the row is what a queue-depth autoscaler leaves held jobs out by. Idempotent in
+   * effect (the row keeps its first `draining_at`). Never rejects.
+   */
+  draining(): Promise<void>;
   /** Stop announcing and hand the row back. Never rejects. */
   stop(): Promise<void>;
 }
@@ -37,9 +44,13 @@ export interface WorkerRegistration {
 export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegistration {
   const registry = options.driver.introspect;
   // A driver with no introspection has no registry to appear in.
-  if (registry === undefined) return { stop: () => Promise.resolve() };
+  if (registry === undefined) {
+    return { draining: () => Promise.resolve(), stop: () => Promise.resolve() };
+  }
   const startedAt = nowMs(options.clock);
   const host = options.host ?? Bun.env['HOSTNAME'] ?? 'unknown';
+  /** Sticky for this registration: a worker that started again registers afresh. */
+  let draining = false;
 
   /**
    * Every announce still on the wire. `stop()` waits them out before it forgets: the row an
@@ -57,6 +68,7 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
           queues: options.queues,
           concurrency: options.concurrency,
           inFlight: options.inFlight().slice(0, MAX_WORKER_IN_FLIGHT),
+          ...(draining ? { draining: true } : {}),
         },
         options.ttlMs,
       )
@@ -86,6 +98,11 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
   );
 
   return {
+    draining() {
+      draining = true;
+      // Not after `stop()`: the row is being handed back, and an announce landing then relists it.
+      return timer.stopped() ? Promise.resolve() : announce();
+    },
     async stop() {
       timer.stop();
       // `send` never rejects (it logs), so neither does this.

package/src/worker-run.ts

@@ -58,6 +58,30 @@ function unknownJob(claimed: ClaimedJob): JobExecution {
   };
 }
 
+/**
+ * The driver `executeJob` settles through, with the lease's heartbeat stopped the moment a settle
+ * is SENT. `ack` / `nack` move the row out of `running`, and `executeJob` still has work after
+ * them — the `onSettled` hook (an app's settle sink, seconds of its own writes) and the step
+ * listing — before the `finally` below stops the heartbeat. A renewal in that window matched no
+ * row and was reported as a lease lost under a job that had finished: `jobs.lease.lost` at error,
+ * `job_leases_lost_total` counted, and the hook's `ctx.signal` aborted under it. Stopped before the
+ * statement, a renewal already on the wire still lands on `heartbeat.ts`'s stopped flag. A settle
+ * that rejects leaves the lease to lapse — what `execute.ts` already says it wants then.
+ */
+function settlingStops(driver: JobDriver, stop: () => void): JobDriver {
+  return {
+    ...driver,
+    ack: (jobId, by) => {
+      stop();
+      return driver.ack(jobId, by);
+    },
+    nack: (jobId, nackOptions) => {
+      stop();
+      return driver.nack(jobId, nackOptions);
+    },
+  };
+}
+
 /** Run one claimed job under its lease, its slot and its span, and settle it with the driver. */
 export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExecution> {
   const { claimed, driver, fleetSlots, workerId, visibilityTimeoutMs } = options;
@@ -156,7 +180,7 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
       () => {
         started = true;
         return executeJob({
-          driver,
+          driver: settlingStops(driver, heartbeat.stop),
           claimed,
           handle,
           ctx,

package/src/worker.ts

@@ -332,6 +332,7 @@ export function jobWorker(options: WorkerOptions): Worker {
     if (state === 'stopped') return;
     state = 'draining';
     loop.stop();
+    void registration?.draining(); // the row says so NOW: a scaler stops counting what it holds
     if (shutdown === undefined) return;
     budget.bind(shutdown.deadlineAt);
     if (cutoff !== undefined) return;