@ultimat3/jobs 29.3.2 → 30.0.0

package/src/webhook-receiver-types.ts

@@ -0,0 +1,97 @@
+// The shapes `webhookReceiver()` speaks: what `verify` is handed and answers, what the queue row
+// holds, and what `handle` receives. Apart from the factory so each file keeps one job.
+
+import type { Ctx } from '@ultimat3/core';
+import type { Bucket, Route } from '@ultimat3/http';
+import type { JsonValue } from '@ultimat3/schema';
+import type { JobHandle } from './job';
+import type { RetryPolicy } from './retry';
+import type { StepApi } from './steps-api';
+
+/** What `verify` and `handshake` are handed: the request as bytes, nothing parsed. */
+export interface WebhookInbound {
+  /** The body exactly as sent, read under `maxBytes`. Empty for the GET handshake. */
+  readonly bytes: Uint8Array;
+  readonly headers: Headers;
+  /** The `:param` segments of `path` — a per-credential token, an account id. */
+  readonly params: Readonly<Record<string, string>>;
+  readonly url: URL;
+  /** A fresh `Request` over the same bytes, for `verifyWebhookSignature` or a mail receiver. */
+  request(): Request;
+}
+
+/** Who the event is for, as `verify` read it — what `tenant` decides from. */
+export interface ReceivedEventFacts {
+  /**
+   * The dedupe key, and therefore the replay key: read it out of VERIFIED data — the signed bytes
+   * `verifyBodyHmac` answers (`.text`), or a header the mac covers — after the signature check,
+   * never from an unsigned header. An unsigned id lets one captured signature replay forever under
+   * fresh ids. 1-256 printable ASCII characters, else `X_WEBHOOK_EVENT_ID_INVALID` (422).
+   */
+  readonly eventId: string;
+  readonly orgId?: string | undefined;
+  readonly topic?: string | undefined;
+}
+
+/**
+ * `verify` accepted the bytes. `payload` is what it read out of them AFTER the signature check —
+ * the job stores it and `handle` receives it, so no retry re-verifies against a window that has
+ * closed. JSON, because it is a queue row.
+ */
+export interface ReceivedEvent<P extends JsonValue = JsonValue> extends ReceivedEventFacts {
+  readonly payload: P;
+}
+
+/** `verify` answered the request itself — a subscription confirm, a challenge. Nothing queues. */
+export interface WebhookAnswer {
+  readonly answer: Response;
+}
+
+/** The queue row. `payload` is `.sensitive()`, so an operator never reads it. */
+export interface WebhookReceiverInput {
+  readonly eventId: string;
+  readonly orgId?: string | undefined;
+  readonly topic?: string | undefined;
+  readonly payload: JsonValue;
+}
+
+export interface WebhookHandleArgs<P extends JsonValue = JsonValue> {
+  readonly eventId: string;
+  readonly orgId: string | undefined;
+  readonly topic: string | undefined;
+  /** What `verify` answered, already verified. Never re-verify here: the vendor's window closes. */
+  readonly payload: P;
+  readonly ctx: Ctx;
+  readonly step: StepApi;
+  readonly attempt: number;
+}
+
+/** Who a receiver's run acts as: `'none'`, or the org `verify` named. */
+export type WebhookReceiverTenant = 'none' | ((event: ReceivedEventFacts) => string | undefined);
+
+export interface WebhookReceiverDefinition<P extends JsonValue = JsonValue> {
+  /** The job's durable name and the route's `webhook.<name>`. Required: it is a queue key. */
+  readonly name: string;
+  /** Absolute, may hold `:param`; never under `/api` or `/_x`, which the framework serves. */
+  readonly path: string;
+  readonly tenant: WebhookReceiverTenant;
+  /** Throws to refuse (`X_WEBHOOK_SIGNATURE_INVALID`); runs before anything is parsed. */
+  verify(
+    inbound: WebhookInbound,
+  ): Promise<ReceivedEvent<P> | WebhookAnswer> | ReceivedEvent<P> | WebhookAnswer;
+  /** The job's body. At least once per `eventId`. */
+  handle(args: WebhookHandleArgs<P>): Promise<unknown>;
+  /** Mounts `GET path` answering what this returns. Unset, no GET route exists. */
+  handshake?(inbound: WebhookInbound): Promise<Response> | Response;
+  readonly retry?: RetryPolicy;
+  readonly queue?: string;
+  /** Defaults to `DEFAULT_WEBHOOK_BODY_LIMIT` (1 MiB), enforced while the body streams. */
+  readonly maxBytes?: number;
+  /** A bucket name from `rateLimit.buckets`, or the route's own numbers. */
+  readonly rateLimit?: string | Bucket;
+}
+
+/** The job handle, plus its routes — mounted by the boot for every receiver the app declares. */
+export interface WebhookReceiver extends JobHandle<WebhookReceiverInput> {
+  routes(): readonly Route[];
+}

package/src/webhook-receiver.ts

@@ -0,0 +1,132 @@
+// `webhookReceiver()` — one inbound webhook, declared as a `job` plus the plain route that feeds
+// it: the twin of `webhook()`, and a factory over `job()`, never a ninth primitive. The route reads
+// the exact bytes, asks the app's `verify`, enqueues durably and only then acknowledges; the job's
+// body is the app's `handle`. Which vendors exist and what a payload means stay the app's (axiom 8).
+//
+// DEDUPE is the queue's live window, keyed by `eventId`: a redelivery while the job is live answers
+// `deduped: true`. After it settles, `handle` is AT LEAST ONCE and receives `eventId` — the seen-set
+// is the app's table. `verify` answers the payload it read AFTER the signature check; the row
+// stores it `.sensitive()` (`x jobs show` masks it) and `handle` receives it, so no retry
+// re-verifies against a vendor's window that has since closed.
+
+import { ACTION_PATH_PREFIX } from '@ultimat3/core';
+import type { Route } from '@ultimat3/http';
+import { DEFAULT_WEBHOOK_BODY_LIMIT } from '@ultimat3/http';
+import type { JsonValue } from '@ultimat3/schema';
+import { t } from '@ultimat3/schema';
+import { JobTenantMismatchError } from './errors-tenant';
+import { job, registeredJobs } from './job';
+import { DEFAULT_RETRY } from './retry';
+import type { JobTenant } from './tenant';
+import { WebhookReceiverInvalidError } from './webhook-receiver-errors';
+import { receiverRoutes } from './webhook-receiver-route';
+import type {
+  WebhookReceiver,
+  WebhookReceiverDefinition,
+  WebhookReceiverInput,
+} from './webhook-receiver-types';
+
+const input = t.object({
+  eventId: t.string,
+  orgId: t.optional(t.string),
+  topic: t.optional(t.string),
+  payload: t.json().sensitive(),
+});
+
+/** Every handle `webhookReceiver()` built — so the boot can mount them without a second list. */
+const receivers = new WeakSet<object>();
+
+export function webhookReceiver<P extends JsonValue = JsonValue>(
+  definition: WebhookReceiverDefinition<P>,
+): WebhookReceiver {
+  const maxBytes = assertReceiver(definition);
+  const handle = job<WebhookReceiverInput>({
+    name: definition.name,
+    input,
+    // `eventId` alone: the namespace is already (name, tenant, key), so one vendor event is one row.
+    idempotencyKey: ({ eventId }) => eventId,
+    tenant: tenantOf(definition),
+    retry: definition.retry ?? DEFAULT_RETRY,
+    ...(definition.queue === undefined ? {} : { queue: definition.queue }),
+    run: ({ input: row, ctx, step, attempt }) =>
+      definition.handle({
+        eventId: row.eventId,
+        orgId: row.orgId,
+        topic: row.topic,
+        // `verify` answered a `P` and the row holds exactly that JSON: the type it was stored as.
+        payload: row.payload as P,
+        ctx,
+        step,
+        attempt,
+      }),
+  });
+  const routes = receiverRoutes({ definition, maxBytes, handle });
+  // In place, never a copy: `isJobHandle` proves the object `job()` built, so a spread would stop
+  // registering.
+  const receiver = Object.assign(handle, { routes: () => routes });
+  receivers.add(receiver);
+  return receiver;
+}
+
+/**
+ * The routes of every receiver declared in this process, in job-name order. The boot mounts these
+ * beside `runtime.routes`: declaring a receiver is the one registration, never a second list.
+ */
+export function webhookReceiverRoutes(): readonly Route[] {
+  return registeredJobs().flatMap((handle) =>
+    receivers.has(handle) ? (handle as WebhookReceiver).routes() : [],
+  );
+}
+
+function tenantOf<P extends JsonValue>(
+  definition: WebhookReceiverDefinition<P>,
+): JobTenant<WebhookReceiverInput> {
+  const declared = definition.tenant;
+  if (typeof declared !== 'function') return declared;
+  return (row) => {
+    const orgId = declared(row);
+    if (typeof orgId === 'string' && orgId.length > 0) return orgId;
+    throw new JobTenantMismatchError({
+      job: definition.name,
+      reason: 'declares an org tenant, and verify named no org for this delivery',
+      fix: `webhookReceiver({ name: '${definition.name}', verify: (inbound) => ({ eventId, orgId }) }) — answer the org the delivery belongs to from verify`,
+    });
+  };
+}
+
+const RESERVED = [ACTION_PATH_PREFIX, '/_x'];
+
+/** Every declaration refusal, at `webhookReceiver()`. Answers the resolved `maxBytes`. */
+function assertReceiver<P extends JsonValue>(definition: WebhookReceiverDefinition<P>): number {
+  const name = typeof definition.name === 'string' ? definition.name : '';
+  const refuse = (field: string, reason: string, fix: string): never => {
+    throw new WebhookReceiverInvalidError({ receiver: name, field, reason, fix });
+  };
+  if (name === '') refuse('name', 'is empty', "name: 'billing.events'");
+  const path: unknown = definition.path;
+  if (typeof path !== 'string' || !path.startsWith('/') || path.length < 2 || path.endsWith('/')) {
+    refuse('path', 'is not an absolute path like /hooks/billing', "path: '/hooks/<vendor>'");
+  }
+  for (const reserved of RESERVED) {
+    if (path === reserved || String(path).startsWith(`${reserved}/`)) {
+      refuse(
+        'path',
+        `sits under ${reserved}, which the framework serves`,
+        "path: '/hooks/<vendor>'",
+      );
+    }
+  }
+  for (const field of ['verify', 'handle'] as const) {
+    if (typeof definition[field] !== 'function') {
+      refuse(field, 'is not a function', `${field}: (…) => …`);
+    }
+  }
+  if (definition.handshake !== undefined && typeof definition.handshake !== 'function') {
+    refuse('handshake', 'is set and is not a function', 'handshake: (inbound) => new Response(…)');
+  }
+  const maxBytes = definition.maxBytes ?? DEFAULT_WEBHOOK_BODY_LIMIT;
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
+    refuse('maxBytes', 'is not a whole number of bytes, at least 1', 'maxBytes: 1_048_576');
+  }
+  return maxBytes;
+}

package/src/webhook-target.ts

@@ -1,19 +1,16 @@
 // Single responsibility: where a webhook delivery is allowed to connect, decided BEFORE the first
 // byte leaves. A tenant-supplied URL was opened as-is: loopback, RFC 1918, link-local — the cloud
 // metadata service at 169.254.169.254 — so anyone who could register an endpoint could make the
-// worker POST into the cluster. The host is resolved, every address it answers is classified by
-// core's one classifier, and the connection is PINNED to the address that was approved.
+// worker POST into the cluster. The screen is core's `screenDial` — resolve, classify every address,
+// pin — and this file keeps only what is a webhook's: the `http:` rule and its own refusal code.
 
-import type { AddressClass } from '@ultimat3/core';
-import { classifyAddress, isLocal, renderThrowable } from '@ultimat3/core';
+import { isLocal } from '@ultimat3/core';
+import type { DialResolve } from '@ultimat3/core/outbound';
+import { screenDial } from '@ultimat3/core/outbound';
 import { WebhookEndpointInvalidError } from './webhook-errors';
 
 /** Every address a hostname answers. Injected in tests: the network is sealed in this repo. */
-export type WebhookResolve = (hostname: string) => Promise<readonly string[]>;
-
-/** `Bun.dns.lookup` — the runtime's own resolver, every family, every address. */
-export const resolveWebhookHost: WebhookResolve = async (hostname) =>
-  (await Bun.dns.lookup(hostname)).map((entry) => entry.address);
+export type WebhookResolve = DialResolve;
 
 /** The connection a delivery makes: an address the screen approved, and the name it proves. */
 export interface WebhookTarget {
@@ -60,39 +57,19 @@ export async function webhookTarget(
       `give the endpoint an https:// url before webhook("${input.webhook}").endpoint returns it`,
     );
   }
-  const hostname = input.url.hostname.replace(/^\[|\]$/g, '');
-  let addresses: readonly string[];
-  if (classifyAddress(hostname) !== undefined) {
-    addresses = [hostname];
-  } else {
-    try {
-      addresses = await input.resolve(hostname);
-    } catch (error) {
-      return { unresolved: `could not resolve the host: ${renderThrowable(error)}` };
-    }
-    if (addresses.length === 0) return { unresolved: 'the host resolved to no address' };
-  }
-  if (!input.allowPrivate) {
-    const classes = addresses.map((address): AddressClass | 'unclassifiable' => {
-      return classifyAddress(address) ?? 'unclassifiable';
-    });
-    const inside = classes.find((kind) => kind !== 'public');
-    if (inside !== undefined) {
-      refuse(
-        input,
-        `resolves to a ${inside} address, and a delivery only opens a public one`,
-        `point the endpoint at a public https:// receiver — or, for a receiver inside your own network, pass allowPrivate: true to webhook("${input.webhook}")`,
-      );
-    }
+  const verdict = await screenDial(input.url, {
+    resolve: input.resolve,
+    allowPrivate: input.allowPrivate,
+  });
+  if ('unresolved' in verdict) return { unresolved: verdict.unresolved };
+  if ('refused' in verdict) {
+    return refuse(
+      input,
+      `resolves to a ${verdict.refused} address, and a delivery only opens a public one`,
+      `point the endpoint at a public https:// receiver — or, for a receiver inside your own network, pass allowPrivate: true to webhook("${input.webhook}")`,
+    );
   }
-  // Pinned: the screen approved THESE addresses, and connecting by name would resolve again — a
-  // name that answered public to the screen and private to the socket (DNS rebinding) walks past.
-  const pinned = addresses[0] ?? hostname;
-  const target = new URL(input.url.href);
-  target.hostname = pinned.includes(':') ? `[${pinned}]` : pinned;
-  return {
-    url: target.href,
-    host: input.url.host,
-    ...(input.url.protocol === 'https:' ? { serverName: hostname } : {}),
-  };
+  // Pinned by the screen: connecting by name would resolve again, and a name that answered public
+  // to the screen and private to the socket (DNS rebinding) would walk past.
+  return verdict.pin;
 }

package/src/webhook.ts

@@ -24,6 +24,7 @@ import {
   throwIfAborted,
   WEBHOOK_FIELD_MAX,
 } from '@ultimat3/core';
+import { resolveDialHost } from '@ultimat3/core/outbound';
 import { t } from '@ultimat3/schema';
 import type { DurationInput } from './clock';
 import { nowMs } from './clock';
@@ -44,7 +45,7 @@ import {
 } from './webhook-errors';
 import type { WebhookLedger } from './webhook-ledger';
 import type { WebhookResolve } from './webhook-target';
-import { resolveWebhookHost, webhookTarget } from './webhook-target';
+import { webhookTarget } from './webhook-target';
 
 /**
  * Consecutive failures before an endpoint stops taking deliveries. Ten is roughly a day of a
@@ -175,7 +176,7 @@ export interface WebhookDefinition {
   readonly allowPrivate?: boolean;
   /** Which environment decides whether `http:` is allowed. Defaults to `process.env`. */
   readonly env?: Readonly<Record<string, string | undefined>>;
-  /** How a hostname is resolved. Defaults to `Bun.dns.lookup`; injected in tests. */
+  /** How a hostname is resolved. Defaults to core's `resolveDialHost`; injected in tests. */
   readonly resolve?: WebhookResolve;
   readonly queue?: string;
   readonly retry?: RetryPolicy;
@@ -194,7 +195,9 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
     'disableAfter',
     definition.disableAfter ?? DEFAULT_WEBHOOK_DISABLE_AFTER,
   );
-  const send = definition.fetch ?? ((url, init) => fetch(url, init));
+  // The attempt's signal, spelled: `webhook-attempt.ts` always sets it (the job's `ctx.signal`).
+  const send =
+    definition.fetch ?? ((url, init) => fetch(url, { ...init, signal: init.signal ?? null }));
 
   // Named so `run` can read the deadline `job()` resolved — one parse of `timeout`, never two.
   const handle: JobHandle<WebhookDeliveryInput> = job<WebhookDeliveryInput>({
@@ -249,7 +252,7 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
         url,
         allowPrivate: definition.allowPrivate === true,
         env: definition.env,
-        resolve: definition.resolve ?? resolveWebhookHost,
+        resolve: definition.resolve ?? resolveDialHost,
       });
 
       const event = await definition.event({ eventId: input.eventId, orgId: input.orgId, ctx });

package/src/worker-queue-demand.ts

@@ -0,0 +1,57 @@
+// Single responsibility: the `queue_demand` gauge one worker process publishes. Split from
+// `worker-queue-depth.ts` because this gauge alone asks the driver a second question, and alone must
+// go quiet when the worker drains.
+
+import type { JobDriver, QueueDemand, QueueStats } from './driver';
+import { recordQueueDemand } from './metrics';
+
+export interface DemandGauge {
+  /** Taken BEFORE the round's `stats()`: a drain landing during that read moves it too. */
+  fence(): number;
+  /** One round, off the `stats()` it read under `fence`. A failed read rejects, zeroing nothing. */
+  publish(stats: readonly QueueStats[], fence: number): Promise<void>;
+  retract(): void;
+}
+
+/**
+ * `stats()` bounds the answer, so an idle fleet is not asked: a queue with nothing ready, running or
+ * suspended has no demand, and `idle-cost.test.ts` holds what an idle worker may cost.
+ */
+function readDemand(
+  driver: JobDriver,
+  stats: readonly QueueStats[],
+): Promise<readonly QueueDemand[]> {
+  const live = stats.some((stat) => stat.ready + stat.running + stat.suspended > 0);
+  return live ? driver.demand() : Promise.resolve([]);
+}
+
+/**
+ * Every worker publishes the same fleet-wide number, so a reader takes `max by (queue)`. That is why
+ * `retract` exists: the claim loop is what publishes, and it stops at the drain, so a retiring pod's
+ * last value would otherwise stand for as long as its job runs, up to the retire, and `max` would
+ * read it over every live worker's. Zero is honest: the pods still serving carry the true number.
+ */
+export function createDemandGauge(driver: JobDriver): DemandGauge {
+  // The queues the last round listed: a gauge keeps its last value, so one that drops out is zeroed.
+  let published: ReadonlySet<string> = new Set();
+  // Moved by `retract`, so a round whose read was in flight at the drain records nothing.
+  let generation = 0;
+
+  return {
+    fence: () => generation,
+    async publish(stats, fence) {
+      if (fence !== generation) return;
+      const rows = await readDemand(driver, stats);
+      if (fence !== generation) return;
+      const listed = new Set(rows.map((row) => row.queue));
+      for (const queue of published) if (!listed.has(queue)) recordQueueDemand(queue, 0);
+      published = listed;
+      for (const row of rows) recordQueueDemand(row.queue, row.demand);
+    },
+    retract() {
+      generation += 1;
+      for (const queue of published) recordQueueDemand(queue, 0);
+      published = new Set();
+    },
+  };
+}

package/src/worker-queue-depth.ts

@@ -1,4 +1,4 @@
-// Single responsibility: the worker's queue metrics — `queue_depth` and the two gauges beside it —
+// Single responsibility: the worker's queue metrics — `queue_depth` and the gauges beside it —
 // republished on their own interval. Split from `worker.ts` at the file-size ceiling; nothing here
 // closes over the claim loop, only over the driver, the clock and the worker's id.
 
@@ -7,6 +7,7 @@ import { logger, recordQueueDepth, renderThrowable } from '@ultimat3/core';
 import { nowMs } from './clock';
 import type { JobDriver } from './driver';
 import { recordQueueDeadJobs, recordQueueOldestReady } from './metrics';
+import { createDemandGauge } from './worker-queue-demand';
 
 /**
  * How often the claim loop republishes `queue_depth`. Its own interval, not `pollIntervalMs`:
@@ -16,6 +17,13 @@ import { recordQueueDeadJobs, recordQueueOldestReady } from './metrics';
  */
 const QUEUE_DEPTH_INTERVAL_MS = 15_000;
 
+/** One publishing round, and what a worker calls the moment it stops serving. */
+export interface QueueDepthPublisher {
+  (): Promise<void>;
+  /** A draining worker's `queue_demand` goes to zero (`worker-queue-demand.ts`). */
+  retractDemand(): void;
+}
+
 /**
  * This package's ONE metrics call site: the `queue_depth` series `docker/helm`'s worker HPA
  * scales on. `ready` and not `ready + delayed` — the gauge means "waiting to be picked up", and
@@ -34,18 +42,20 @@ export function createQueueDepthPublisher(input: {
   readonly driver: JobDriver;
   readonly clock: Clock | undefined;
   readonly workerId: string;
-}): () => Promise<void> {
+}): QueueDepthPublisher {
+  const demand = createDemandGauge(input.driver);
   let publishedAt = Number.NEGATIVE_INFINITY;
   // The queues the last successful round published. A gauge keeps the last value it was given and
   // `stats()` lists a queue only while it holds a row it counts, so a queue that drained to `done`
   // would read its final backlog forever — and hold the HPA scaled out — unless it is zeroed on the
   // round it drops out. Bounded by the queue names, and replaced each round.
   let published: ReadonlySet<string> = new Set();
-  return async (): Promise<void> => {
+  const publish = async (): Promise<void> => {
     const now = nowMs(input.clock);
     if (now - publishedAt < QUEUE_DEPTH_INTERVAL_MS) return;
     publishedAt = now;
     try {
+      const fence = demand.fence();
       const stats = await input.driver.stats();
       const reported = new Set(stats.map((stat) => stat.queue));
       for (const queue of published) {
@@ -64,6 +74,7 @@ export function createQueueDepthPublisher(input: {
         recordQueueOldestReady(stat.queue, stat.oldestReadyMs);
         recordQueueDeadJobs(stat.queue, stat.dead);
       }
+      await demand.publish(stats, fence);
     } catch (error) {
       // Instrumentation never costs a tick: a queue that cannot be measured must still be worked.
       logger.warn('jobs.worker.depth-failed', {
@@ -72,4 +83,5 @@ export function createQueueDepthPublisher(input: {
       });
     }
   };
+  return Object.assign(publish, { retractDemand: demand.retract });
 }

package/src/worker-registry.ts

@@ -16,6 +16,8 @@ export interface WorkerRegistryOptions {
   readonly workerId: string;
   /** Where this worker runs. A pod name under an orchestrator; `HOSTNAME` when nothing says. */
   readonly host: string | undefined;
+  /** The build this process runs, stamped on its row. Absent: the row names none. */
+  readonly build?: string | undefined;
   readonly queues: readonly string[];
   /** Slots across every queue served. */
   readonly concurrency: number;
@@ -32,7 +34,7 @@ export interface WorkerRegistration {
   /**
    * The worker stopped claiming and only finishes what it holds: every announce from now on says
    * `draining`, and one goes out NOW rather than at the next heartbeat (a third of the visibility
-   * window later) — the row is what a queue-depth autoscaler leaves held jobs out by. Idempotent in
+   * window later) — the row is what a queue-demand autoscaler leaves held jobs out by. Idempotent in
    * effect (the row keeps its first `draining_at`). Never rejects.
    */
   draining(): Promise<void>;
@@ -69,6 +71,7 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
           concurrency: options.concurrency,
           inFlight: options.inFlight().slice(0, MAX_WORKER_IN_FLIGHT),
           ...(draining ? { draining: true } : {}),
+          ...(options.build === undefined ? {} : { build: options.build }),
         },
         options.ttlMs,
       )

package/src/worker-types.ts

@@ -50,6 +50,8 @@ export interface WorkerOptions {
   readonly workerId?: string;
   /** Where this worker runs, for the registry. Default `HOSTNAME`, which a pod sets to its name. */
   readonly host?: string;
+  /** The build this process runs, for the registry: what `x deploy verify` grades a queue by. */
+  readonly build?: string;
   /** Default true. Registers a SIGTERM drain via `onShutdown`. */
   readonly drainOnShutdown?: boolean;
 }

package/src/worker.ts

@@ -108,7 +108,6 @@ export function jobWorker(options: WorkerOptions): Worker {
   let stopping: Promise<void> | undefined;
   const tally = createWorkerTally();
   let registration: WorkerRegistration | undefined;
-  /** `queue_depth` and its two siblings, republished on their own interval (`worker-queue-depth.ts`). */
   const publishQueueDepth = createQueueDepthPublisher({
     driver: options.driver,
     clock: options.clock,
@@ -133,7 +132,6 @@ export function jobWorker(options: WorkerOptions): Worker {
       ...(options.events === undefined ? {} : { events: options.events }),
     });
 
-  /** The drain's one question: may this worker still take work off the queue? */
   const claiming = (): boolean => state !== 'draining' && state !== 'stopped';
 
   /**
@@ -333,6 +331,7 @@ export function jobWorker(options: WorkerOptions): Worker {
     state = 'draining';
     loop.stop();
     void registration?.draining(); // the row says so NOW: a scaler stops counting what it holds
+    publishQueueDepth.retractDemand(); // the loop just stopped: nothing would refresh this gauge
     if (shutdown === undefined) return;
     budget.bind(shutdown.deadlineAt);
     if (cutoff !== undefined) return;
@@ -450,6 +449,7 @@ export function jobWorker(options: WorkerOptions): Worker {
         driver: options.driver,
         workerId,
         host: options.host,
+        build: options.build,
         queues,
         concurrency: Math.min(slots, cap.max ?? slots),
         inFlight: () => inFlight.ids(),