@ultimat3/jobs 20.2.1 → 22.0.0

package/src/scheduler.ts

@@ -24,7 +24,8 @@ import type { Clock } from '@ultimat3/core';
 import { finiteOption, isUltimateError, logger, onShutdown, renderThrowable } from '@ultimat3/core';
 import { instant, nextCronOccurrence } from '@ultimat3/time';
 import { nowMs } from './clock';
-import { settleAllBy } from './drain-wait';
+import type { DrainBudget } from './drain-wait';
+import { createDrainBudget, settleAllBy } from './drain-wait';
 import type { JobDriver } from './driver';
 import type { TaskHandle, TaskJobResult } from './task';
 import { registeredTasks } from './task';
@@ -258,43 +259,54 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
       // on a wall clock in the middle of this walk, not between rounds. See the file header for
       // what a second dispatcher costs — the occurrence key does NOT absorb it in general.
       if (!(await stillLeading())) break;
-      const last = await schedulerState.lastFiredAt(handle.name);
-      if (last === undefined) {
-        // First sight of this task: arm it, never fire retroactively for all of history.
-        await schedulerState.markFired(handle.name, nextRunFor(handle, new Date(at)).getTime() - 1);
-        continue;
-      }
+      // One task's failure is that task's: its `enqueue` callback, its state read or its dispatch
+      // threw, and the round used to abort there — every task after it stopped firing, in every
+      // round, until a deploy. Logged and skipped; its watermark is untouched, so it retries next
+      // round.
+      try {
+        const last = await schedulerState.lastFiredAt(handle.name);
+        if (last === undefined) {
+          // First sight of this task: arm it, never fire retroactively for all of history.
+          await schedulerState.markFired(
+            handle.name,
+            nextRunFor(handle, new Date(at)).getTime() - 1,
+          );
+          continue;
+        }
 
-      const due = occurrencesSince(handle, last, at);
-      if (due.length === 0) continue;
+        const due = occurrencesSince(handle, last, at);
+        if (due.length === 0) continue;
 
-      if (handle.catchUp === 'skip') {
-        // The real latest occurrence, never `due`'s last element: that one is `maxCatchUp` steps
-        // past the watermark, and dispatching it leaves the watermark there — so the next tick
-        // found the next ten still due and fired again, until the walk reached `at`. The
-        // occurrence key stays honest (this IS the occurrence the payload is for), and the
-        // watermark `dispatch` leaves is that occurrence — nothing at or before `at` is due past it.
-        const latest = latestOccurrenceBy(handle, last, at);
-        dispatched.push(await dispatch(handle, latest, due.length > 1));
-        continue;
-      }
-      if (handle.catchUp === 'run-once') {
-        const first = due[0];
-        if (first !== undefined) {
-          dispatched.push(await dispatch(handle, first, due.length > 1));
-          // `dispatch` leaves the watermark on the occurrence it RAN — the earliest missed one
-          // here — so the next round found occurrences 2..n still due and fired the second, then
-          // the third, one per tick until the backlog drained: 24 nightly digests a second apart
-          // after a day down. "One catch-up" means the rest are DROPPED, and dropping an
-          // occurrence is moving the watermark past it. `at` rather than the last element of
-          // `due`, which `maxCatchUp` truncates: every occurrence at or before `at` is missed by
-          // definition, and this policy fires none of them.
-          if (first !== at) await schedulerState.markFired(handle.name, at);
+        if (handle.catchUp === 'skip') {
+          // The real latest occurrence, never `due`'s last element: that one is `maxCatchUp` steps
+          // past the watermark, and dispatching it leaves the watermark there — so the next tick
+          // found the next ten still due and fired again, until the walk reached `at`. The
+          // occurrence key stays honest (this IS the occurrence the payload is for), and the
+          // watermark `dispatch` leaves is that occurrence — nothing at or before `at` is due past it.
+          const latest = latestOccurrenceBy(handle, last, at);
+          dispatched.push(await dispatch(handle, latest, due.length > 1));
+          continue;
         }
-        continue;
-      }
-      for (const occurrence of due) {
-        dispatched.push(await dispatch(handle, occurrence, occurrence !== due[due.length - 1]));
+        if (handle.catchUp === 'run-once') {
+          const first = due[0];
+          if (first !== undefined) {
+            dispatched.push(await dispatch(handle, first, due.length > 1));
+            // `dispatch` leaves the watermark on the occurrence it RAN — the earliest missed one
+            // here — so the next round found occurrences 2..n still due and fired the second, then
+            // the third, one per tick until the backlog drained: 24 nightly digests a second apart
+            // after a day down. "One catch-up" means the rest are DROPPED, and dropping an
+            // occurrence is moving the watermark past it. `at` rather than the last element of
+            // `due`, which `maxCatchUp` truncates: every occurrence at or before `at` is missed by
+            // definition, and this policy fires none of them.
+            if (first !== at) await schedulerState.markFired(handle.name, at);
+          }
+          continue;
+        }
+        for (const occurrence of due) {
+          dispatched.push(await dispatch(handle, occurrence, occurrence !== due[due.length - 1]));
+        }
+      } catch (error) {
+        logger.error('jobs.task.round_failed', { task: handle.name, ...failureFields(error) });
       }
     }
 
@@ -346,7 +358,7 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     timer = undefined;
   };
 
-  const teardown = async (reason: string, deadlineAt?: number): Promise<void> => {
+  const teardown = async (reason: string, budget: DrainBudget): Promise<void> => {
     stopDispatching();
     logger.info('jobs.scheduler.draining', { reason, dispatching: round !== undefined });
     try {
@@ -360,7 +372,7 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
       // a queue that is not answering cannot be cancelled from here, and an unbounded wait is a
       // teardown that never ends — hooks never handed back, `state` never past 'draining', and the
       // memoised `stopping` every later `stop()` joins never settling.
-      const dispatched = await settleAllBy(round === undefined ? [] : [round], deadlineAt);
+      const dispatched = await settleAllBy(round === undefined ? [] : [round], budget);
       // A round we ABANDONED is a round still enqueueing, so the lock is deliberately NOT handed
       // back: promoting a standby onto an occurrence this process is mid-dispatch for is the
       // double-fire above, delivered by the shutdown. A lease row expires on its own, which is
@@ -385,16 +397,25 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     }
   };
 
+  /** The wait a teardown may spend, tightened by every shutdown that reaches it. */
+  let budget = createDrainBudget();
+
   const stop = async (reason = 'stop', deadlineAt?: number): Promise<void> => {
     // Answered immediately once this scheduler is done: the teardown always REACHES 'stopped', so
     // a caller landing after an abandoned drain gets an answer rather than joining a promise that
     // never settles.
     if (state === 'stopped') return;
+    // Bound on EVERY call, before the join — `createDrainBudget`, the worker's rule. A manual
+    // stop waits with no deadline; a SIGTERM that lands on it joined that teardown without its
+    // own, so a round parked on a queue that is not answering held the close hook past the whole
+    // process budget. Reproduced in `scheduler-drain-budget.test.ts`.
+    if (deadlineAt !== undefined) budget.bind(deadlineAt);
     // One teardown, joined rather than repeated — the worker's rule, for the same reason: a
     // SIGTERM landing on a manual stop must wait out the same round, not release the lock a
     // second time behind it. Cleared as it settles, so a scheduler started again stops again.
-    stopping ??= teardown(reason, deadlineAt).finally(() => {
+    stopping ??= teardown(reason, budget).finally(() => {
       stopping = undefined;
+      budget = createDrainBudget();
     });
     await stopping;
   };

package/src/steps-memory.ts

@@ -20,7 +20,11 @@ export function createMemoryStepStore(): StepStore {
       return Promise.resolve(byRun.get(runId)?.get(name));
     },
     put(record) {
-      runOf(record.runId).set(record.name, record);
+      // Through JSON, exactly as the pg store persists it (`JSON.stringify(output ?? null)`): a
+      // live reference replayed a `Date` here and a string in production, so a step that worked
+      // in every test broke on the first retry against Postgres.
+      const output: unknown = JSON.parse(JSON.stringify(record.output ?? null));
+      runOf(record.runId).set(record.name, { ...record, output });
       return Promise.resolve();
     },
     list(runId) {

package/src/steps.ts

@@ -110,6 +110,25 @@ export class StepSuspension extends Error {
   }
 }
 
+/**
+ * What a timed-out `waitForEvent` persists, and maps back to `undefined` on replay. `undefined`
+ * itself cannot survive a store — the pg driver writes `JSON.stringify(output ?? null)` — so the
+ * timeout replayed as `null` and `evt === undefined` read as "an event arrived". A namespaced key
+ * rather than a bare `{ timedOut: true }`, so no event payload can be mistaken for the marker.
+ */
+const WAIT_TIMED_OUT = Object.freeze<Record<string, true>>({
+  '~ultimate.waitTimedOut': true,
+});
+
+function isWaitTimedOut(output: unknown): boolean {
+  return (
+    typeof output === 'object' &&
+    output !== null &&
+    Object.keys(output).length === 1 &&
+    (output as Record<string, unknown>)['~ultimate.waitTimedOut'] === true
+  );
+}
+
 export function isStepSuspension(error: unknown): error is StepSuspension {
   return error instanceof Error && (error as { brand?: unknown }).brand === StepSuspension.brand;
 }
@@ -366,7 +385,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
     const existing = await load(name);
     if (existing?.status === 'completed') {
       trace(replayed, name);
-      return existing.output as T | undefined;
+      return isWaitTimedOut(existing.output) ? undefined : (existing.output as T | undefined);
     }
 
     const at = now();
@@ -409,7 +428,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
         runId,
         name,
         status: 'completed',
-        output: undefined,
+        output: WAIT_TIMED_OUT,
         startedAt,
         completedAt: at,
         event,

package/src/task.ts

@@ -19,7 +19,7 @@
 //   late, and `catchUp` cannot recover an occurrence that was never an occurrence.
 
 import { assert } from '@ultimat3/core';
-import { isValidTimeZone } from '@ultimat3/time';
+import { isValidTimeZone, parseCron } from '@ultimat3/time';
 import { nowMs } from './clock';
 import type { EnqueueResult } from './driver';
 import { JobNameTakenError } from './errors';
@@ -131,6 +131,11 @@ export function task(definition: TaskDefinition): TaskHandle {
     `use the full zone id on task("${name}"), e.g. tz: 'America/Bogota' — list the valid ones with: bun -e "console.log(Intl.supportedValuesOf('timeZone').join('\\n'))"`,
   );
 
+  // The cron is parsed HERE, at declaration, beside the tz check. Nothing read it until the first
+  // round, where one bad expression threw inside `runRound` and every task in the app stopped.
+  // `parseCron` throws `X_CRON_INVALID` naming the field that is wrong.
+  parseCron(definition.cron);
+
   // `maxCatchUp: 0` is not "no ceiling" — `occurrencesSince` walks
   // `for (let i = 0; i < handle.maxCatchUp; i += 1)`, so zero (and any negative, and any fraction
   // below one) returns an empty list on every round and the task NEVER fires: no error, no log

package/src/webhook-errors.ts

@@ -40,11 +40,14 @@ export class WebhookEndpointUnknownError extends UltimateError {
 
 /** The endpoint exists and cannot be delivered to as declared. */
 export class WebhookEndpointInvalidError extends UltimateError {
-  constructor(input: { webhook: string; endpointId: string; reason: string }) {
+  /** `fix` for a refusal the url-and-secret instruction does not answer — an address screen's. */
+  constructor(input: { webhook: string; endpointId: string; reason: string; fix?: string }) {
     super({
       code: 'X_WEBHOOK_ENDPOINT_INVALID',
       cause: `webhook "${input.webhook}" endpoint ${input.endpointId} ${input.reason}`,
-      fix: `give the endpoint an https:// url and a non-empty secret before webhook("${input.webhook}").endpoint returns it — a delivery with neither is an unsigned POST to wherever the row points`,
+      fix:
+        input.fix ??
+        `give the endpoint an https:// url and a non-empty secret before webhook("${input.webhook}").endpoint returns it — a delivery with neither is an unsigned POST to wherever the row points`,
       meta: { endpointId: input.endpointId },
     });
   }

package/src/webhook-target.ts

@@ -0,0 +1,98 @@
+// Single responsibility: where a webhook delivery is allowed to connect, decided BEFORE the first
+// byte leaves. A tenant-supplied URL was opened as-is: loopback, RFC 1918, link-local — the cloud
+// metadata service at 169.254.169.254 — so anyone who could register an endpoint could make the
+// worker POST into the cluster. The host is resolved, every address it answers is classified by
+// core's one classifier, and the connection is PINNED to the address that was approved.
+
+import type { AddressClass } from '@ultimat3/core';
+import { classifyAddress, isLocal, renderThrowable } from '@ultimat3/core';
+import { WebhookEndpointInvalidError } from './webhook-errors';
+
+/** Every address a hostname answers. Injected in tests: the network is sealed in this repo. */
+export type WebhookResolve = (hostname: string) => Promise<readonly string[]>;
+
+/** `Bun.dns.lookup` — the runtime's own resolver, every family, every address. */
+export const resolveWebhookHost: WebhookResolve = async (hostname) =>
+  (await Bun.dns.lookup(hostname)).map((entry) => entry.address);
+
+/** The connection a delivery makes: an address the screen approved, and the name it proves. */
+export interface WebhookTarget {
+  /** The endpoint url with its host replaced by the approved address. */
+  readonly url: string;
+  /** The original `host[:port]`, sent as the `Host` header. */
+  readonly host: string;
+  /** For `https:` — the certificate is verified against the NAME, not the address. */
+  readonly serverName?: string;
+}
+
+export interface WebhookTargetInput {
+  readonly webhook: string;
+  readonly endpointId: string;
+  readonly url: URL;
+  readonly allowPrivate: boolean;
+  readonly env: Readonly<Record<string, string | undefined>> | undefined;
+  readonly resolve: WebhookResolve;
+}
+
+const refuse = (input: WebhookTargetInput, reason: string, fix: string): never => {
+  throw new WebhookEndpointInvalidError({
+    webhook: input.webhook,
+    endpointId: input.endpointId,
+    reason,
+    fix,
+  });
+};
+
+/**
+ * The approved target, or `{ unresolved }` when the name did not resolve — a TRANSIENT failure
+ * the queue retries, never a refusal. A refusal throws `X_WEBHOOK_ENDPOINT_INVALID`, and its
+ * reason names the address CLASS, never the address: it reaches a durable dead-letter row.
+ */
+export async function webhookTarget(
+  input: WebhookTargetInput,
+): Promise<WebhookTarget | { readonly unresolved: string }> {
+  // `http:` only where a dev receiver lives. A process that names no environment is production —
+  // `fallback: 'production'`, the reading every dev-secret guard in the framework takes.
+  if (input.url.protocol === 'http:' && !isLocal({ env: input.env, fallback: 'production' })) {
+    refuse(
+      input,
+      'has an http: url outside a local environment, so the signed body would cross the network in clear',
+      `give the endpoint an https:// url before webhook("${input.webhook}").endpoint returns it`,
+    );
+  }
+  const hostname = input.url.hostname.replace(/^\[|\]$/g, '');
+  let addresses: readonly string[];
+  if (classifyAddress(hostname) !== undefined) {
+    addresses = [hostname];
+  } else {
+    try {
+      addresses = await input.resolve(hostname);
+    } catch (error) {
+      return { unresolved: `could not resolve the host: ${renderThrowable(error)}` };
+    }
+    if (addresses.length === 0) return { unresolved: 'the host resolved to no address' };
+  }
+  if (!input.allowPrivate) {
+    const classes = addresses.map((address): AddressClass | 'unclassifiable' => {
+      return classifyAddress(address) ?? 'unclassifiable';
+    });
+    const inside = classes.find((kind) => kind !== 'public');
+    if (inside !== undefined) {
+      refuse(
+        input,
+        `resolves to a ${inside} address, and a delivery only opens a public one`,
+        `point the endpoint at a public https:// receiver — or, for a receiver inside your own network, pass allowPrivate: true to webhook("${input.webhook}")`,
+      );
+    }
+  }
+  // Pinned: the screen approved THESE addresses, and connecting by name would resolve again — a
+  // name that answered public to the screen and private to the socket (DNS rebinding) walks past.
+  const pinned = addresses[0] ?? hostname;
+  const target = new URL(input.url.href);
+  target.hostname = pinned.includes(':') ? `[${pinned}]` : pinned;
+  return {
+    url: target.href,
+    host: input.url.host,
+    ...(input.url.protocol === 'https:' ? { serverName: hostname } : {}),
+  };
+}

package/src/webhook.ts

@@ -47,9 +47,18 @@ import {
   WebhookEventUnknownError,
 } from './webhook-errors';
 import type { WebhookLedger } from './webhook-ledger';
+import type { WebhookResolve, WebhookTarget } from './webhook-target';
+import { resolveWebhookHost, webhookTarget } from './webhook-target';
 
 /** Just the call. `typeof fetch` also carries `preconnect`, which no test double should have to. */
-export type WebhookFetch = (url: string, init: RequestInit) => Promise<Response>;
+/**
+ * The transport. `init.tls.serverName` carries the hostname a PINNED connection proves, because
+ * the url it is handed names the approved address (`webhook-target.ts`); Bun's `fetch` honours it.
+ */
+export type WebhookFetch = (
+  url: string,
+  init: RequestInit & { readonly tls?: { readonly serverName?: string } },
+) => Promise<Response>;
 
 /**
  * Consecutive failures before an endpoint stops taking deliveries. Ten is roughly a day of a
@@ -66,7 +75,11 @@ const RETRY_AFTER_SECONDS = /^\d{1,7}$/;
 
 export interface WebhookEndpoint {
   readonly id: string;
-  /** `https://…` in production, `http://…` for a dev receiver. Nothing else is opened. */
+  /**
+   * `https://…` in production, `http://…` only in a local environment. A host that resolves to a
+   * loopback, private, link-local, ULA, CGNAT or unspecified address is refused unless the
+   * definition sets `allowPrivate` (`webhook-target.ts`). Nothing else is opened.
+   */
   readonly url: string;
   /** The shared secret. Never logged, never in a `cause`, never on the ledger row. */
   readonly secret: string;
@@ -151,6 +164,16 @@ export interface WebhookDefinition {
   readonly clock?: Clock;
   /** Injected so a test can drive the transport. The network is sealed in this repo's suites. */
   readonly fetch?: WebhookFetch;
+  /**
+   * The explicit opt-out from the address screen, for a receiver inside your own network — a dev
+   * receiver on `localhost`, a service in the same cluster. Off by default: an endpoint URL is
+   * tenant-supplied, and the screen is what stops it reaching the metadata service.
+   */
+  readonly allowPrivate?: boolean;
+  /** Which environment decides whether `http:` is allowed. Defaults to `process.env`. */
+  readonly env?: Readonly<Record<string, string | undefined>>;
+  /** How a hostname is resolved. Defaults to `Bun.dns.lookup`; injected in tests. */
+  readonly resolve?: WebhookResolve;
   readonly queue?: string;
   readonly retry?: RetryPolicy;
   /**
@@ -203,7 +226,16 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
           endpointId: endpoint.id,
         });
       }
-      assertDeliverable(definition.name, endpoint);
+      const url = assertDeliverable(definition.name, endpoint);
+      // Screened and resolved before the event is read, let alone sent — see `webhook-target.ts`.
+      const target = await webhookTarget({
+        webhook: definition.name,
+        endpointId: endpoint.id,
+        url,
+        allowPrivate: definition.allowPrivate === true,
+        env: definition.env,
+        resolve: definition.resolve ?? resolveWebhookHost,
+      });
 
       const event = await definition.event({ eventId: input.eventId, ctx });
       if (event === null) {
@@ -229,13 +261,15 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
       // than the age of the fact behind it.
       const timestampSeconds = Math.floor(nowMs(clock) / 1_000);
       const startedAt = clock.monotonic();
-      const outcome = await attemptDelivery(send, endpoint, {
-        secret: endpoint.secret,
-        timestampSeconds,
-        eventId: input.eventId,
-        topic: event.topic,
-        body: event.body,
-      });
+      const outcome: Outcome = await ('unresolved' in target
+        ? Promise.resolve({ ok: false as const, status: null, detail: target.unresolved })
+        : attemptDelivery(send, endpoint, target, {
+            secret: endpoint.secret,
+            timestampSeconds,
+            eventId: input.eventId,
+            topic: event.topic,
+            body: event.body,
+          }));
       const durationMs = Math.max(0, clock.monotonic() - startedAt);
 
       // Recorded whatever happened, and BEFORE the throw: a failure that is not on the ledger is a
@@ -279,8 +313,8 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
   });
 }
 
-/** A URL no delivery may open, or a secret that would make the POST unsigned. */
-function assertDeliverable(name: string, endpoint: WebhookEndpoint): void {
+/** A URL no delivery may open, or a secret that would make the POST unsigned. The parsed url. */
+function assertDeliverable(name: string, endpoint: WebhookEndpoint): URL {
   if (endpoint.secret.length === 0) {
     throw new WebhookEndpointInvalidError({
       webhook: name,
@@ -288,9 +322,9 @@ function assertDeliverable(name: string, endpoint: WebhookEndpoint): void {
       reason: 'has an empty secret, so the delivery would carry no proof of who sent it',
     });
   }
-  let protocol: string;
+  let parsed: URL;
   try {
-    protocol = new URL(endpoint.url).protocol;
+    parsed = new URL(endpoint.url);
   } catch {
     // Never the caught value and never the url: an unparseable value is exactly the one whose
     // shape is unknown, and this reason reaches a durable dead-letter row.
@@ -302,13 +336,14 @@ function assertDeliverable(name: string, endpoint: WebhookEndpoint): void {
   }
   // `file:`, `data:` and the rest are refused by NAME rather than by a blocklist: a delivery opens
   // an HTTP conversation, and anything else is a row in a table reaching the worker's filesystem.
-  if (protocol !== 'https:' && protocol !== 'http:') {
+  if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
     throw new WebhookEndpointInvalidError({
       webhook: name,
       endpointId: endpoint.id,
-      reason: `has a ${protocol} url, and a delivery only ever opens http: or https:`,
+      reason: `has a ${parsed.protocol} url, and a delivery only ever opens http: or https:`,
     });
   }
+  return parsed;
 }
 
 type Outcome =
@@ -323,18 +358,23 @@ type Outcome =
 async function attemptDelivery(
   send: WebhookFetch,
   endpoint: WebhookEndpoint,
+  target: WebhookTarget,
   signing: Parameters<typeof webhookHeaders>[0],
 ): Promise<Outcome> {
   let response: Response;
   try {
-    response = await send(endpoint.url, {
+    response = await send(target.url, {
       method: 'POST',
       headers: {
         ...endpoint.headers,
+        // The name the pinned address is reached AS — after the row's own headers, so an endpoint
+        // cannot redirect the request to a different virtual host than the one it registered.
+        host: target.host,
         'content-type': WEBHOOK_CONTENT_TYPE,
         // LAST, so an endpoint row's own headers can never overwrite the signature it is proved by.
         ...webhookHeaders(signing),
       },
+      ...(target.serverName === undefined ? {} : { tls: { serverName: target.serverName } }),
       body: signing.body,
       // Never followed: a 3xx would re-POST a body signed for one host to whatever the receiver
       // named, and the signature would travel with it.

package/src/worker-hand-back.ts

@@ -0,0 +1,30 @@
+// Single responsibility: returning the rest of a claimed batch a round cannot start. Split from
+// `worker.ts` at the file-size ceiling. A fleet-slot write that rejected mid-batch rethrew out of
+// `claimRound` and stranded every job behind it in `running`, each with an attempt burned.
+
+import { logger, renderThrowable } from '@ultimat3/core';
+import type { ClaimedJob, JobDriver } from './driver';
+
+/**
+ * Each job returned exactly as a shed returns one — no attempt burned, no `lastError` — so
+ * another worker takes it at once. Best-effort per job: the failure that got us here is usually
+ * the database, one nack failing must not stop the rest being tried, and a job whose nack also
+ * fails comes back when its visibility timeout lapses.
+ */
+export async function handBack(
+  driver: JobDriver,
+  jobs: readonly ClaimedJob[],
+  options: { readonly delayMs: number; readonly workerId: string },
+): Promise<void> {
+  for (const claimed of jobs) {
+    await driver
+      .nack(claimed.id, { delayMs: options.delayMs, countsAsAttempt: false })
+      .catch((error: unknown) => {
+        logger.debug('jobs.worker.hand_back_failed', {
+          workerId: options.workerId,
+          jobId: claimed.id,
+          error: renderThrowable(error),
+        });
+      });
+  }
+}

package/src/worker-queue-depth.ts

@@ -0,0 +1,61 @@
+// Single responsibility: the worker's queue metrics — `queue_depth` and the two gauges beside it —
+// republished on their own interval. Split from `worker.ts` at the file-size ceiling; nothing here
+// closes over the claim loop, only over the driver, the clock and the worker's id.
+
+import type { Clock } from '@ultimat3/core';
+import { logger, recordQueueDepth, renderThrowable } from '@ultimat3/core';
+import { nowMs } from './clock';
+import type { JobDriver } from './driver';
+import { recordQueueDeadJobs, recordQueueOldestReady } from './metrics';
+
+/**
+ * How often the claim loop republishes `queue_depth`. Its own interval, not `pollIntervalMs`:
+ * `driver.stats()` is an aggregate over the whole jobs table and a scrape reads the gauge every
+ * ~15s, so publishing at the poll rate would multiply the queue's read load by sixty to write the
+ * same number sixty times.
+ */
+const QUEUE_DEPTH_INTERVAL_MS = 15_000;
+
+/**
+ * This package's ONE metrics call site: the `queue_depth` series `docker/helm`'s worker HPA
+ * scales on. `ready` and not `ready + delayed` — the gauge means "waiting to be picked up", and
+ * a job parked until Tuesday is not backlog no matter how many workers are added. `ready` counts
+ * every DUE row, a delayed one past its `run_at` included (`SQL_STATS`). Every queue the driver
+ * reports, not only the ones this process serves, because depth is the queue's fact and a queue
+ * no pod published is a queue no autoscaler can see.
+ *
+ * So every worker pod publishes the SAME number, and a reader must treat it as one series for
+ * the whole fleet: deduplicate with `max`, never `sum`. The chart's worker HPA reads it as an
+ * `External` metric with an `AverageValue` target, which divides it by the replica count; read
+ * as a `Pods` metric it was averaged as if each pod held the whole backlog, and the autoscaler
+ * asked for N times the workers the queue needed until it hit `maxReplicas`.
+ */
+export function createQueueDepthPublisher(input: {
+  readonly driver: JobDriver;
+  readonly clock: Clock | undefined;
+  readonly workerId: string;
+}): () => Promise<void> {
+  let publishedAt = Number.NEGATIVE_INFINITY;
+  return async (): Promise<void> => {
+    const now = nowMs(input.clock);
+    if (now - publishedAt < QUEUE_DEPTH_INTERVAL_MS) return;
+    publishedAt = now;
+    try {
+      for (const stat of await input.driver.stats()) {
+        recordQueueDepth(stat.queue, stat.ready);
+        // Depth alone is not alertable: it cannot tell "10 jobs stuck for an hour" from "10 jobs
+        // enqueued a second ago", and `jobs_total{outcome="dead"}` is a rate, so a dead-letter
+        // queue that filled overnight and stopped growing pages nobody. Both numbers are already
+        // in `stats()` — this queries nothing new.
+        recordQueueOldestReady(stat.queue, stat.oldestReadyMs);
+        recordQueueDeadJobs(stat.queue, stat.dead);
+      }
+    } catch (error) {
+      // Instrumentation never costs a tick: a queue that cannot be measured must still be worked.
+      logger.warn('jobs.worker.depth-failed', {
+        workerId: input.workerId,
+        error: renderThrowable(error),
+      });
+    }
+  };
+}