@ultimat3/jobs 27.16.0 → 27.16.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/jobs",
-  "version": "27.16.0",
+  "version": "27.16.1",
   "description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
   "license": "MIT",
   "type": "module",
@@ -33,10 +33,10 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "27.16.0",
-    "@ultimat3/db": "27.16.0",
-    "@ultimat3/entity": "27.16.0",
-    "@ultimat3/schema": "27.16.0",
-    "@ultimat3/time": "27.16.0"
+    "@ultimat3/core": "27.16.1",
+    "@ultimat3/db": "27.16.1",
+    "@ultimat3/entity": "27.16.1",
+    "@ultimat3/schema": "27.16.1",
+    "@ultimat3/time": "27.16.1"
   }
 }

package/src/driver-memory.ts

@@ -29,6 +29,7 @@ import {
   LIVE_STATES,
   nackState,
   REQUEUEABLE_STATES,
+  STATS_STATES,
 } from './driver';
 import { memoryOperator } from './driver-memory-operator';
 import { signalEnqueued } from './enqueue-signal';
@@ -394,6 +395,9 @@ export function memoryJobDriver(options: MemoryJobDriverOptions = {}): MemoryJob
       const at = nowMs(clock);
       const byQueue = new Map<string, QueueStats>();
       for (const record of jobs.values()) {
+        // A finished row counts toward no bucket, so it does not list its queue either — the
+        // `where` `SQL_STATS` reads through `x_jobs_depth_idx`.
+        if (!STATS_STATES.has(record.state)) continue;
         const current = byQueue.get(record.queue) ?? {
           queue: record.queue,
           ready: 0,

package/src/driver-pg-ddl.ts

@@ -217,6 +217,15 @@ create index if not exists x_jobs_finished_idx
   on x_jobs (updated_at)
   where state in ('done', 'failed', 'dead', 'cancelled');
 
+-- What the queue depth read (SQL_STATS) reads, every 15 seconds from every worker. Partial over
+-- the six states it counts, so the read is a range over the counted rows and never a pass over the
+-- finished ones the retention window keeps. Its predicate is the same list SQL_STATS filters on,
+-- which is what lets the planner prove the index applies. Queue, state and run_at are every column
+-- the read uses, so it can be answered from the index alone.
+create index if not exists x_jobs_depth_idx
+  on x_jobs (queue, state, run_at)
+  where state in ('ready', 'delayed', 'running', 'suspended', 'failed', 'dead');
+
 -- The rows of one run, by run id: the retention sweep asks, per deleted row, whether another row of
 -- the run still stands before it deletes the step records of that run, and its follow-up and its
 -- orphan pass ask again. Without it each question is a scan of the whole queue. Not partial: run_id

package/src/driver-pg-sql.ts

@@ -198,6 +198,12 @@ returning id
  * `delayed`, a delayed dead-letter as `dead` AND `delayed`. The memory driver's `if/else if` chain
  * has always been exclusive, so `x jobs` reported one depth under `x dev` and a larger one in
  * production — off the same rows.
+ *
+ * The `where` is the six states a bucket counts (`STATS_STATES`), spelled as `x_jobs_depth_idx`'s
+ * predicate so the planner reads that index and never a `done` row: every worker runs this every
+ * 15 s, and unfiltered it scanned a fortnight of finished jobs to count a handful (prod,
+ * 2026-10-24: 29 ms mean over 48k rows, 22 counted). A queue holding only `done` or `cancelled`
+ * rows is therefore not listed, and the depth publisher zeroes a queue that drops out.
  */
 export const SQL_STATS = `
 select queue,
@@ -210,6 +216,7 @@ select queue,
        coalesce(max(extract(epoch from now() - run_at)) filter
          (where state in ('ready', 'delayed') and run_at <= now()), 0) * 1000   as oldest_ready_ms
   from x_jobs
+ where state in ('ready', 'delayed', 'running', 'suspended', 'failed', 'dead')
  group by queue
  order by queue collate "C"
 `.trim();

package/src/driver-redis-scripts.ts

@@ -193,7 +193,9 @@ return 1
 /**
  * `SQL_STATS`, one consistent read. `ARGV`: namespace, now. Answers one row per queue:
  * `{ queue, ready, delayed, running, suspended, failed, dead, oldest runAt or -1 }` — due is due
- * whatever state the row was WRITTEN in, the split `SQL_STATS` makes.
+ * whatever state the row was WRITTEN in, the split `SQL_STATS` makes. A queue all six counts leave
+ * at zero is not answered: `SQL_STATS` reads only the states it counts, so a queue holding only
+ * finished jobs is absent there and in the memory driver, and `:queues` never forgets a name.
  */
 export const REDIS_STATS = `
 local P = ARGV[1]
@@ -204,7 +206,7 @@ for _, queue in ipairs(redis.call('SMEMBERS', P .. ':queues')) do
   local oldest = redis.call('ZRANGE', wait, 0, 0, 'WITHSCORES')
   local first = -1
   if oldest[2] and tonumber(oldest[2]) <= now then first = oldest[2] end
-  table.insert(out, {
+  local row = {
     queue,
     redis.call('ZCOUNT', wait, '-inf', now),
     redis.call('ZCOUNT', wait, '(' .. now, '+inf'),
@@ -213,7 +215,8 @@ for _, queue in ipairs(redis.call('SMEMBERS', P .. ':queues')) do
     redis.call('ZCARD', P .. ':failed:' .. queue),
     redis.call('ZCARD', P .. ':dead:' .. queue),
     first,
-  })
+  }
+  if row[2] + row[3] + row[4] + row[5] + row[6] + row[7] > 0 then table.insert(out, row) end
 end
 return out
 `.trim();

package/src/driver.ts

@@ -58,6 +58,21 @@ export const LIVE_STATES: ReadonlySet<JobState> = new Set<JobState>([
   'suspended',
 ]);
 
+/**
+ * The states `stats()` counts, one bucket each (`ready` and `delayed` split by `runAt`): the live
+ * four and the two an operator is paged on. `done` and `cancelled` rows count toward nothing, so
+ * neither driver reads them — `SQL_STATS` and `x_jobs_depth_idx` spell the same six, and a queue
+ * holding only finished rows is not listed.
+ */
+export const STATS_STATES: ReadonlySet<JobState> = new Set<JobState>([
+  'ready',
+  'delayed',
+  'running',
+  'suspended',
+  'failed',
+  'dead',
+]);
+
 /**
  * What a row's `lastError` says when the CLAIM buried it: its lease lapsed on the attempt that was
  * its last, so nothing ever settled it and nothing may run it again. Fixed text, written by the

package/src/worker-queue-depth.ts

@@ -36,12 +36,26 @@ export function createQueueDepthPublisher(input: {
   readonly workerId: string;
 }): () => Promise<void> {
   let publishedAt = Number.NEGATIVE_INFINITY;
+  // The queues the last successful round published. A gauge keeps the last value it was given and
+  // `stats()` lists a queue only while it holds a row it counts, so a queue that drained to `done`
+  // would read its final backlog forever — and hold the HPA scaled out — unless it is zeroed on the
+  // round it drops out. Bounded by the queue names, and replaced each round.
+  let published: ReadonlySet<string> = new Set();
   return async (): Promise<void> => {
     const now = nowMs(input.clock);
     if (now - publishedAt < QUEUE_DEPTH_INTERVAL_MS) return;
     publishedAt = now;
     try {
-      for (const stat of await input.driver.stats()) {
+      const stats = await input.driver.stats();
+      const reported = new Set(stats.map((stat) => stat.queue));
+      for (const queue of published) {
+        if (reported.has(queue)) continue;
+        recordQueueDepth(queue, 0);
+        recordQueueOldestReady(queue, 0);
+        recordQueueDeadJobs(queue, 0);
+      }
+      published = reported;
+      for (const stat of stats) {
         recordQueueDepth(stat.queue, stat.ready);
         // Depth alone is not alertable: it cannot tell "10 jobs stuck for an hour" from "10 jobs
         // enqueued a second ago", and `jobs_total{outcome="dead"}` is a rate, so a dead-letter