@ultimat3/jobs 23.0.0 → 25.0.0

package/src/webhook-attempt.ts

@@ -0,0 +1,206 @@
+// One delivery attempt on the wire: the headers it carries, the POST, what the answer means, and
+// which of the three failure codes it becomes. Everything BEFORE the socket — the endpoint, the
+// screen, the event, the gates — and everything AFTER it — the ledger, disable-after-N — is
+// `webhook.ts`'s; this file only ever sees an approved target and a signed body.
+
+import {
+  // Core's ONE table, never a fifth copy of it. The copy that lived here omitted 409, so a
+  // receiver saying "a concurrent writer won this round" dead-lettered on attempt 1 as a refusal
+  // no retry could change — the exact divergence `retryable-status.ts` was extracted to end.
+  isRetryableStatus,
+  isUltimateError,
+  renderThrowable,
+  retryAfterSecondsOf,
+  webhookHeaders,
+} from '@ultimat3/core';
+import { JobTimeoutError } from './errors';
+import {
+  WebhookDeliveryFailedError,
+  WebhookDeliveryRejectedError,
+  WebhookDeliveryThrottledError,
+} from './webhook-errors';
+import type { WebhookTarget } from './webhook-target';
+
+/**
+ * The transport — just the call: `typeof fetch` also carries `preconnect`, which no test double
+ * should have to. `init.tls.serverName` carries the hostname a PINNED connection proves, because
+ * the url it is handed names the approved address (`webhook-target.ts`); Bun's `fetch` honours it.
+ */
+export type WebhookFetch = (
+  url: string,
+  init: RequestInit & { readonly tls?: { readonly serverName?: string } },
+) => Promise<Response>;
+
+/** The one content type a delivery announces. The BODY is the app's; how it is framed is not. */
+export const WEBHOOK_CONTENT_TYPE = 'application/json';
+
+/** What of a `WebhookEndpoint` an attempt reads. Never the secret: that is inside `signing`. */
+export interface DeliveryEndpoint {
+  readonly id: string;
+  readonly url: string;
+  readonly headers?: Readonly<Record<string, string>>;
+}
+
+export type Outcome =
+  | { readonly ok: true; readonly status: number }
+  | {
+      readonly ok: false;
+      readonly status: number | null;
+      readonly detail: string;
+      readonly retryAfterSeconds?: number;
+    };
+
+/**
+ * The receiver's `Retry-After`, through core's one reader: delta-seconds as before, and now the
+ * HTTP-date form too, measured against the SAME response's `Date` — the receiver's clock against
+ * the receiver's clock, which is what made the date safe to read without trusting ours. Capped at
+ * a day, and the nack clamps it to the policy's `maxDelay` on top.
+ */
+const retryAfterSeconds = (response: Response): number | undefined =>
+  retryAfterSecondsOf(response.headers.get('retry-after'), response.headers.get('date'));
+
+/**
+ * The row's own headers, then the framework's — each SET, case-insensitively, over whatever the row
+ * said. An object spread cannot do this: `{ Host: 'evil.test', host: <pinned> }` keeps both keys and
+ * `fetch` joins them, so a row would reach another virtual host, change the content type, and ship
+ * a forged value inside the signature list. Going through `Headers` collapses every spelling of a
+ * name to one entry before the framework's value replaces it. Returned as a plain lowercase record,
+ * because an injected `WebhookFetch` has always been handed one.
+ *
+ * Inside the caller's `try`: a row header the platform refuses (a newline in a value) is the same
+ * failed attempt it always was, never a bare `TypeError` out of the job.
+ */
+function deliveryHeaders(
+  endpoint: DeliveryEndpoint,
+  target: WebhookTarget,
+  signing: Parameters<typeof webhookHeaders>[0],
+): Record<string, string> {
+  const headers = new Headers(endpoint.headers);
+  // The name the pinned address is reached AS, so an endpoint cannot redirect the request to a
+  // different virtual host than the one it registered.
+  headers.set('host', target.host);
+  headers.set('content-type', WEBHOOK_CONTENT_TYPE);
+  // The signature and the event identity it covers: a row can never overwrite what it is proved by.
+  for (const [name, value] of Object.entries(webhookHeaders(signing))) headers.set(name, value);
+  return Object.fromEntries(headers.entries());
+}
+
+/** The reason `JobDrainedError` carries — the one cancellation a request on the wire outlives. */
+const DRAINING = 'X_DRAINING';
+
+const isDrain = (reason: unknown): boolean => isUltimateError(reason) && reason.code === DRAINING;
+
+/** The request's own signal, and the listener it hangs on the attempt's, handed back after. */
+export interface DeliverySignal {
+  readonly signal: AbortSignal;
+  dispose(): void;
+}
+
+/** Where this attempt's own deadline falls, on `performance.now()`'s clock. */
+export interface AttemptDeadline {
+  readonly atMs: number;
+  readonly job: string;
+  readonly timeoutMs: number;
+}
+
+/**
+ * The attempt's signal for every reason but the DRAIN. A deadline or a lost lease is followed by a
+ * nack that hands the job to another worker at once, so a request still open is the receiver
+ * getting the event twice: those tear it down. The drain is the opposite case — tearing a POST
+ * already on the wire down hands the job to the next pod, which re-POSTs the same event on every
+ * deploy — so the request finishes inside the drain budget instead, and its outcome is recorded.
+ *
+ * Not past the attempt's DEADLINE, though. The attempt's signal keeps its FIRST reason, so once the
+ * drain has aborted it the deadline behind it never arrives there — while `executeJob` still hands
+ * the job on when the deadline passes. So a drained request arms the deadline itself.
+ */
+export function deliverySignal(attempt: AbortSignal, deadline?: AttemptDeadline): DeliverySignal {
+  const request = new AbortController();
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const follow = (): void => {
+    if (!isDrain(attempt.reason)) {
+      request.abort(attempt.reason);
+      return;
+    }
+    if (deadline === undefined) return;
+    timer = setTimeout(
+      () =>
+        request.abort(new JobTimeoutError({ job: deadline.job, timeoutMs: deadline.timeoutMs })),
+      Math.max(0, deadline.atMs - performance.now()),
+    );
+  };
+  if (attempt.aborted) follow();
+  else attempt.addEventListener('abort', follow, { once: true });
+  return {
+    signal: request.signal,
+    dispose: () => {
+      attempt.removeEventListener('abort', follow);
+      if (timer !== undefined) clearTimeout(timer);
+    },
+  };
+}
+
+/**
+ * One POST, on `deliverySignal`'s signal: torn down by the deadline or a lost lease, never by the
+ * drain.
+ */
+export async function attemptDelivery(
+  send: WebhookFetch,
+  endpoint: DeliveryEndpoint,
+  target: WebhookTarget,
+  signing: Parameters<typeof webhookHeaders>[0],
+  signal: AbortSignal,
+): Promise<Outcome> {
+  let response: Response;
+  try {
+    response = await send(target.url, {
+      method: 'POST',
+      headers: deliveryHeaders(endpoint, target, signing),
+      ...(target.serverName === undefined ? {} : { tls: { serverName: target.serverName } }),
+      body: signing.body,
+      // Never followed: a 3xx would re-POST a body signed for one host to whatever the receiver
+      // named, and the signature would travel with it.
+      redirect: 'manual',
+      signal,
+    });
+  } catch (error) {
+    // `renderThrowable`, never `String(error)` or `${error}`: this string lands in a `cause` and on
+    // a durable ledger row, and a null-prototype throwable makes both of those a `TypeError`.
+    return { ok: false, status: null, detail: renderThrowable(error) };
+  }
+  if (response.ok) return { ok: true, status: response.status };
+  const stated = retryAfterSeconds(response);
+  return {
+    ok: false,
+    status: response.status,
+    detail: `status ${response.status}`,
+    ...(stated === undefined ? {} : { retryAfterSeconds: stated }),
+  };
+}
+
+/** Which of the three failure codes this outcome is. The split is "can the same request land?". */
+export function deliveryError(
+  name: string,
+  endpoint: DeliveryEndpoint,
+  outcome: Extract<Outcome, { ok: false }>,
+): Error {
+  const base = { webhook: name, endpointId: endpoint.id, url: endpoint.url };
+  if (outcome.status === null) {
+    return new WebhookDeliveryFailedError({ ...base, status: null, detail: outcome.detail });
+  }
+  if (!isRetryableStatus(outcome.status)) {
+    return new WebhookDeliveryRejectedError({ ...base, status: outcome.status });
+  }
+  if (outcome.retryAfterSeconds !== undefined) {
+    return new WebhookDeliveryThrottledError({
+      ...base,
+      status: outcome.status,
+      retryAfterSeconds: outcome.retryAfterSeconds,
+    });
+  }
+  return new WebhookDeliveryFailedError({
+    ...base,
+    status: outcome.status,
+    detail: outcome.detail,
+  });
+}

package/src/webhook-ledger.ts

@@ -39,11 +39,19 @@ export interface WebhookLedger {
    */
   record(attempt: WebhookAttempt): Promise<number>;
   /**
-   * Stop delivering to this endpoint. Called once, on the attempt whose count reached
-   * `disableAfter`. Re-enabling is the app's — a dead endpoint that heals itself is a retry loop
-   * with no end.
+   * Stop delivering to this endpoint. Called by the attempt whose count reached `disableAfter` —
+   * and, rarely, by a second attempt that was already in flight when the first one crossed it, so
+   * an implementation is IDEMPOTENT: disabling a disabled endpoint changes nothing. Re-enabling is
+   * the app's — a dead endpoint that heals itself is a retry loop with no end.
    */
   disable(endpointId: string, reason: string): Promise<void>;
+  /**
+   * Whether `disable` has switched this endpoint off. Asked before every attempt opens a socket,
+   * because `disable` is the only thing the mechanism writes: a ledger that recorded the verdict
+   * and was never asked for it again is an endpoint that keeps receiving POSTs. An app whose
+   * `disable` sets its endpoint row's `disabled` column answers from that same column.
+   */
+  isDisabled(endpointId: string): Promise<boolean>;
 }
 
 /** Attempts one process keeps before the oldest go. A dev ledger that grows forever is a leak. */
@@ -88,9 +96,12 @@ export function memoryWebhookLedger(
       return Promise.resolve(failures);
     },
     disable(endpointId: string, reason: string): Promise<void> {
-      off.set(endpointId, reason);
+      // The FIRST reason stays: it names the run that switched the endpoint off, and a late
+      // in-flight attempt repeating the verdict must not overwrite it.
+      if (!off.has(endpointId)) off.set(endpointId, reason);
       return Promise.resolve();
     },
+    isDisabled: (endpointId: string): Promise<boolean> => Promise.resolve(off.has(endpointId)),
     attempts: (): readonly WebhookAttempt[] => [...attempts],
     disabled: (): ReadonlyMap<string, string> => new Map(off),
     reset(): void {

package/src/webhook.ts

@@ -19,14 +19,10 @@ import type { Clock, Ctx } from '@ultimat3/core';
 import {
   finiteOption,
   isCanonicalWebhookField,
-  // Core's ONE table, never a fifth copy of it. The copy that lived here omitted 409, so a
-  // receiver saying "a concurrent writer won this round" dead-lettered on attempt 1 as a refusal
-  // no retry could change — the exact divergence `retryable-status.ts` was extracted to end.
-  isRetryableStatus,
-  renderThrowable,
+  isUltimateError,
   systemClock,
+  throwIfAborted,
   WEBHOOK_FIELD_MAX,
-  webhookHeaders,
 } from '@ultimat3/core';
 import { t } from '@ultimat3/schema';
 import type { DurationInput } from './clock';
@@ -36,10 +32,9 @@ import { job } from './job';
 import type { RetryPolicy } from './retry';
 import { DEFAULT_RETRY } from './retry';
 import type { JobTenant } from './tenant';
+import type { Outcome, WebhookFetch } from './webhook-attempt';
+import { attemptDelivery, deliveryError, deliverySignal } from './webhook-attempt';
 import {
-  WebhookDeliveryFailedError,
-  WebhookDeliveryRejectedError,
-  WebhookDeliveryThrottledError,
   WebhookEndpointDisabledError,
   WebhookEndpointInvalidError,
   WebhookEndpointUnknownError,
@@ -47,19 +42,9 @@ import {
   WebhookEventUnknownError,
 } from './webhook-errors';
 import type { WebhookLedger } from './webhook-ledger';
-import type { WebhookResolve, WebhookTarget } from './webhook-target';
+import type { WebhookResolve } from './webhook-target';
 import { resolveWebhookHost, webhookTarget } from './webhook-target';
 
-/** Just the call. `typeof fetch` also carries `preconnect`, which no test double should have to. */
-/**
- * The transport. `init.tls.serverName` carries the hostname a PINNED connection proves, because
- * the url it is handed names the approved address (`webhook-target.ts`); Bun's `fetch` honours it.
- */
-export type WebhookFetch = (
-  url: string,
-  init: RequestInit & { readonly tls?: { readonly serverName?: string } },
-) => Promise<Response>;
-
 /**
  * Consecutive failures before an endpoint stops taking deliveries. Ten is roughly a day of a
  * retrying queue against a dead receiver, which is long enough for an outage and short enough that
@@ -67,12 +52,6 @@ export type WebhookFetch = (
  */
 export const DEFAULT_WEBHOOK_DISABLE_AFTER = 10;
 
-/** The one content type a delivery announces. The BODY is the app's; how it is framed is not. */
-export const WEBHOOK_CONTENT_TYPE = 'application/json';
-
-/** `Retry-After` in seconds. The HTTP-date form is ignored on purpose — see `retryAfterSeconds`. */
-const RETRY_AFTER_SECONDS = /^\d{1,7}$/;
-
 export interface WebhookEndpoint {
   readonly id: string;
   /**
@@ -184,13 +163,6 @@ export interface WebhookDefinition {
   readonly timeout?: DurationInput;
 }
 
-/** `Retry-After: 120`. The HTTP-date form needs the receiver's clock, which is what we do not trust. */
-const retryAfterSeconds = (response: Response): number | undefined => {
-  const header = response.headers.get('retry-after');
-  if (header === null || !RETRY_AFTER_SECONDS.test(header.trim())) return undefined;
-  return Number(header.trim());
-};
-
 export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliveryInput> {
   const clock = definition.clock ?? systemClock;
   const disableAfter = finiteOption(
@@ -200,7 +172,8 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
   );
   const send = definition.fetch ?? ((url, init) => fetch(url, init));
 
-  return job<WebhookDeliveryInput>({
+  // Named so `run` can read the deadline `job()` resolved — one parse of `timeout`, never two.
+  const handle: JobHandle<WebhookDeliveryInput> = job<WebhookDeliveryInput>({
     name: definition.name,
     input: t.object({ endpointId: t.string, eventId: t.string }),
     // Endpoint AND event: the same event fans out to every subscribed endpoint, so a key on the
@@ -211,6 +184,16 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
     ...(definition.queue === undefined ? {} : { queue: definition.queue }),
     ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
     async run({ input, ctx, attempt }): Promise<WebhookReport> {
+      // On the clock `executeJob`'s deadline timer runs on, read as the body starts — which is
+      // when that timer is armed.
+      const deadline =
+        handle.timeoutMs === undefined
+          ? undefined
+          : {
+              atMs: performance.now() + handle.timeoutMs,
+              job: definition.name,
+              timeoutMs: handle.timeoutMs,
+            };
       const endpoint = await definition.endpoint({ endpointId: input.endpointId, ctx });
       if (endpoint === null) {
         throw new WebhookEndpointUnknownError({
@@ -220,7 +203,9 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
       }
       // Before the ledger and before the socket: a disabled endpoint costs nothing, which is the
       // whole point of disabling one.
-      if (endpoint.disabled === true) {
+      // Asked of BOTH sources, because `disableAfter` writes only the ledger: an endpoint the
+      // mechanism switched off must stop here whether or not the app mirrored it onto its row.
+      if (endpoint.disabled === true || (await definition.ledger.isDisabled(endpoint.id))) {
         throw new WebhookEndpointDisabledError({
           webhook: definition.name,
           endpointId: endpoint.id,
@@ -260,17 +245,41 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
       // again now, so a receiver's freshness window measures the request in front of it rather
       // than the age of the fact behind it.
       const timestampSeconds = Math.floor(nowMs(clock) / 1_000);
+      // The last moment a cancelled attempt can stop without the receiver hearing anything — the
+      // drain included: a POST not yet started is handed back to the next pod at no cost.
+      stopIfCancelled(ctx);
       const startedAt = clock.monotonic();
-      const outcome: Outcome = await ('unresolved' in target
-        ? Promise.resolve({ ok: false as const, status: null, detail: target.unresolved })
-        : attemptDelivery(send, endpoint, target, {
-            secret: endpoint.secret,
-            timestampSeconds,
-            eventId: input.eventId,
-            topic: event.topic,
-            body: event.body,
-          }));
+      // Past this line the drain no longer cancels: see `deliverySignal`.
+      const request = deliverySignal(ctx.signal, deadline);
+      let outcome: Outcome;
+      try {
+        outcome = await ('unresolved' in target
+          ? Promise.resolve({ ok: false as const, status: null, detail: target.unresolved })
+          : attemptDelivery(
+              send,
+              endpoint,
+              target,
+              {
+                secret: endpoint.secret,
+                timestampSeconds,
+                eventId: input.eventId,
+                topic: event.topic,
+                body: event.body,
+              },
+              request.signal,
+            ));
+      } finally {
+        request.dispose();
+      }
       const durationMs = Math.max(0, clock.monotonic() - startedAt);
+      // A failure the REQUEST's cancellation caused is not the receiver's, and is not this
+      // attempt's to record: the queue has already handed the delivery to another worker, whose
+      // outcome is the one the consecutive count must hear. Recorded, it was a failure nobody saw
+      // counting toward `disableAfter` — an abandoned attempt switching off a healthy endpoint.
+      // Read off the request's signal, never the attempt's: a receiver that really failed while
+      // the worker drained is a failure like any other, and the count hears it. A delivery that
+      // LANDED is a fact, and is always recorded.
+      if (!outcome.ok && request.signal.aborted) stopIfCancelled(ctx);
 
       // Recorded whatever happened, and BEFORE the throw: a failure that is not on the ledger is a
       // failure the consecutive count cannot see, which is an endpoint that never gets disabled.
@@ -311,6 +320,7 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
       throw deliveryError(definition.name, endpoint, outcome);
     },
   });
+  return handle;
 }
 
 /** A URL no delivery may open, or a secret that would make the POST unsigned. The parsed url. */
@@ -346,78 +356,14 @@ function assertDeliverable(name: string, endpoint: WebhookEndpoint): URL {
   return parsed;
 }
 
-type Outcome =
-  | { readonly ok: true; readonly status: number }
-  | {
-      readonly ok: false;
-      readonly status: number | null;
-      readonly detail: string;
-      readonly retryAfterSeconds?: number;
-    };
-
-async function attemptDelivery(
-  send: WebhookFetch,
-  endpoint: WebhookEndpoint,
-  target: WebhookTarget,
-  signing: Parameters<typeof webhookHeaders>[0],
-): Promise<Outcome> {
-  let response: Response;
-  try {
-    response = await send(target.url, {
-      method: 'POST',
-      headers: {
-        ...endpoint.headers,
-        // The name the pinned address is reached AS — after the row's own headers, so an endpoint
-        // cannot redirect the request to a different virtual host than the one it registered.
-        host: target.host,
-        'content-type': WEBHOOK_CONTENT_TYPE,
-        // LAST, so an endpoint row's own headers can never overwrite the signature it is proved by.
-        ...webhookHeaders(signing),
-      },
-      ...(target.serverName === undefined ? {} : { tls: { serverName: target.serverName } }),
-      body: signing.body,
-      // Never followed: a 3xx would re-POST a body signed for one host to whatever the receiver
-      // named, and the signature would travel with it.
-      redirect: 'manual',
-    });
-  } catch (error) {
-    // `renderThrowable`, never `String(error)` or `${error}`: this string lands in a `cause` and on
-    // a durable ledger row, and a null-prototype throwable makes both of those a `TypeError`.
-    return { ok: false, status: null, detail: renderThrowable(error) };
-  }
-  if (response.ok) return { ok: true, status: response.status };
-  const stated = retryAfterSeconds(response);
-  return {
-    ok: false,
-    status: response.status,
-    detail: `status ${response.status}`,
-    ...(stated === undefined ? {} : { retryAfterSeconds: stated }),
-  };
-}
-
-/** Which of the three failure codes this outcome is. The split is "can the same request land?". */
-function deliveryError(
-  name: string,
-  endpoint: WebhookEndpoint,
-  outcome: Extract<Outcome, { ok: false }>,
-): Error {
-  const base = { webhook: name, endpointId: endpoint.id, url: endpoint.url };
-  if (outcome.status === null) {
-    return new WebhookDeliveryFailedError({ ...base, status: null, detail: outcome.detail });
-  }
-  if (!isRetryableStatus(outcome.status)) {
-    return new WebhookDeliveryRejectedError({ ...base, status: outcome.status });
-  }
-  if (outcome.retryAfterSeconds !== undefined) {
-    return new WebhookDeliveryThrottledError({
-      ...base,
-      status: outcome.status,
-      retryAfterSeconds: outcome.retryAfterSeconds,
-    });
-  }
-  return new WebhookDeliveryFailedError({
-    ...base,
-    status: outcome.status,
-    detail: outcome.detail,
-  });
+/**
+ * Throw the attempt's cancellation as itself — the deadline's `X_JOB_TIMEOUT`, the drain's
+ * `X_DRAINING`, a lost lease — so `executeJob` reads the stop it caused, never a delivery failure
+ * naming a receiver that did nothing wrong. Core's `X_ABORTED` when the reason carries no code.
+ */
+function stopIfCancelled(ctx: Ctx): void {
+  if (!ctx.signal.aborted) return;
+  const reason: unknown = ctx.signal.reason;
+  if (isUltimateError(reason)) throw reason;
+  throwIfAborted(ctx);
 }

package/src/worker-admit.ts

@@ -76,8 +76,10 @@ export function createAdmission(options: AdmissionOptions): Admit {
       try {
         await shed(job, queue, limiter.blockedBy(key) ?? 'unknown');
       } catch (error) {
-        // The nack itself failed: the jobs BEHIND it go back before the round reports.
-        await handBack(driver, behind, back);
+        // The nack itself failed, so THIS job went nowhere either: it goes back with the jobs
+        // behind it, one more best-effort try, before the round reports. Left out, it sat
+        // `running` until its lease lapsed — an attempt spent on a body that never started.
+        await handBack(driver, [job, ...behind], back);
         throw error;
       }
       return { kind: 'waiting' };
@@ -95,7 +97,8 @@ export function createAdmission(options: AdmissionOptions): Admit {
     try {
       grant = await fleetSlots.acquire(job);
     } catch (error) {
-      lease.release();
+      // Never started: the slot AND the rate stamp go back (`Lease.abandon`).
+      lease.abandon();
       // This job and every one behind it go BACK, unburned — rethrowing alone stranded them in
       // `running` with an attempt spent on work that never started.
       await handBack(driver, [job, ...behind], back);
@@ -106,7 +109,9 @@ export function createAdmission(options: AdmissionOptions): Admit {
     // refusal before the body: it fails as an attempt, never as a round.
     if (grant.outcome === 'undecidable') return { kind: 'run', lease, refusal: grant.error };
 
-    lease.release();
+    // Over the fleet cap, so this run does not start: `abandon`, never `release`, or a tenant
+    // waiting behind a full `job.concurrency` pays one `ratePerTenant` start per poll interval.
+    lease.abandon();
     try {
       // The two `whenBusy` answers. `'fail'` settles the run here, its body never started.
       if (grant.outcome === 'fail') {
@@ -119,12 +124,15 @@ export function createAdmission(options: AdmissionOptions): Admit {
           context: options.context,
           ...(options.clock === undefined ? {} : { clock: options.clock }),
         });
-        return { kind: 'refused', execution };
+        // No execution: the nack did not land, so the row is not this worker's to have refused.
+        return execution === undefined ? { kind: 'waiting' } : { kind: 'refused', execution };
       }
       const keyed = grant.key === undefined ? '' : `, key ${grant.key}`;
       await shed(job, queue, `job concurrency (${grant.limit}${keyed})`);
     } catch (error) {
-      await handBack(driver, behind, back);
+      // This job too, for the reason above. A refusal that DID settle before throwing is safe to
+      // repeat: every nack is fenced on `running` and the claim, so a settled row ignores it.
+      await handBack(driver, [job, ...behind], back);
       throw error;
     }
     return { kind: 'waiting' };

package/src/worker-drain-cutoff.ts

@@ -0,0 +1,110 @@
+// When a draining worker stops waiting for the jobs it holds. SIGTERM stops the CLAIMING; the jobs
+// already running keep going, because a deploy that cancels work in flight re-runs every side
+// effect it interrupted. Only near the end of the drain budget is what is still running told to
+// stop (`cancel`), and a beat later what is still held goes back to the queue (`handBack`).
+
+import { logger, renderThrowable, systemClock } from '@ultimat3/core';
+import type { DrainBudget } from './drain-wait';
+import { JobDrainedError } from './errors';
+import type { HeldRuns } from './worker-held';
+
+/**
+ * The most of the budget's tail kept for unwinding: the cancel lands this far before the deadline,
+ * the hand-back half as far, and the teardown — the registry row, the driver — gets the rest. Two
+ * seconds of a 25 s default, and never more than HALF of a small budget, so a budget of 300 ms
+ * still spends 150 ms letting its jobs finish before anything is cut.
+ */
+export const DRAIN_CANCEL_MARGIN_MS = 2_000;
+
+export interface CutoffTimes {
+  /** Real monotonic ms: the held runs' `ctx.signal` aborts with `X_DRAINING`. */
+  readonly cancelAt: number;
+  /** Real monotonic ms: every claim still held is handed back uncounted. */
+  readonly handBackAt: number;
+}
+
+/** The cut-off for a deadline, read against the instant it was bound. */
+export function cutoffTimes(deadlineAt: number, now: number): CutoffTimes {
+  const remaining = Math.max(0, deadlineAt - now);
+  const margin = Math.min(DRAIN_CANCEL_MARGIN_MS, remaining / 2);
+  return { cancelAt: deadlineAt - margin, handBackAt: deadlineAt - margin / 2 };
+}
+
+export interface DrainCutoff {
+  /** Stop listening to the budget and clear both timers — the teardown is over. */
+  dispose(): void;
+}
+
+/**
+ * Arm `cancel` and `handBack` on `budget`, re-armed each time it tightens (the earliest deadline
+ * wins — `drain-wait.ts`). Each fires at most once. The timers are `unref`ed: a drained process is
+ * never kept alive by its own cut-off.
+ */
+export function armDrainCutoff(
+  budget: DrainBudget,
+  steps: { readonly cancel: () => void; readonly handBack: () => Promise<void> },
+): DrainCutoff {
+  const timers: ReturnType<typeof setTimeout>[] = [];
+  let cancelled = false;
+  let handedBack = false;
+  const clear = (): void => {
+    for (const timer of timers.splice(0)) clearTimeout(timer);
+  };
+  const at = (instant: number, fire: () => void): void => {
+    const timer = setTimeout(fire, Math.max(0, instant - systemClock.monotonic()));
+    timer.unref?.();
+    timers.push(timer);
+  };
+  const unwatch = budget.watch((deadlineAt) => {
+    clear();
+    const times = cutoffTimes(deadlineAt, systemClock.monotonic());
+    if (!cancelled) {
+      at(times.cancelAt, () => {
+        cancelled = true;
+        steps.cancel();
+      });
+    }
+    if (!handedBack) {
+      at(times.handBackAt, () => {
+        handedBack = true;
+        // Each nack's failure is logged where it happens (`worker-hand-back.ts`); this is the one
+        // place anything else could still escape a timer, so it is observed here.
+        void steps.handBack().catch((error: unknown) => {
+          logger.error('jobs.worker.drain-hand-back-failed', { error: renderThrowable(error) });
+        });
+      });
+    }
+  });
+  return {
+    dispose() {
+      unwatch();
+      clear();
+    },
+  };
+}
+
+/**
+ * The worker's cut-off: `drain` aborted with `X_DRAINING` at `cancelAt`, every run `held` still
+ * holds handed back at `handBackAt`. Aborted even with nothing held: a round still inside its
+ * `claim()` starts its runs under this controller, and they must hear the cut-off too.
+ */
+export function armWorkerCutoff(input: {
+  readonly budget: DrainBudget;
+  readonly workerId: string;
+  /** The shutdown's signal name, carried on the reason the bodies read. */
+  readonly signal: string;
+  readonly drain: AbortController;
+  readonly held: HeldRuns;
+}): DrainCutoff {
+  const { workerId, held } = input;
+  return armDrainCutoff(input.budget, {
+    cancel: () => {
+      if (held.size > 0) logger.warn('jobs.worker.drain-cancelling', { workerId, held: held.size });
+      input.drain.abort(new JobDrainedError({ workerId, signal: input.signal }));
+    },
+    handBack: async () => {
+      const handed = await held.handBackAll();
+      if (handed > 0) logger.warn('jobs.worker.drain-handed-back', { workerId, handed });
+    },
+  });
+}