@ultimat3/jobs 24.0.0 → 25.1.0

package/src/webhook.ts

@@ -19,27 +19,23 @@ import type { Clock, Ctx } from '@ultimat3/core';
 import {
   finiteOption,
   isCanonicalWebhookField,
-  // Core's ONE table, never a fifth copy of it. The copy that lived here omitted 409, so a
-  // receiver saying "a concurrent writer won this round" dead-lettered on attempt 1 as a refusal
-  // no retry could change — the exact divergence `retryable-status.ts` was extracted to end.
-  isRetryableStatus,
-  renderThrowable,
+  isUltimateError,
   systemClock,
+  throwIfAborted,
   WEBHOOK_FIELD_MAX,
-  webhookHeaders,
 } from '@ultimat3/core';
 import { t } from '@ultimat3/schema';
 import type { DurationInput } from './clock';
 import { nowMs } from './clock';
+import { JobTenantMismatchError } from './errors-tenant';
 import type { JobHandle } from './job';
 import { job } from './job';
 import type { RetryPolicy } from './retry';
 import { DEFAULT_RETRY } from './retry';
 import type { JobTenant } from './tenant';
+import type { Outcome, WebhookFetch } from './webhook-attempt';
+import { attemptDelivery, deliveryError, deliverySignal } from './webhook-attempt';
 import {
-  WebhookDeliveryFailedError,
-  WebhookDeliveryRejectedError,
-  WebhookDeliveryThrottledError,
   WebhookEndpointDisabledError,
   WebhookEndpointInvalidError,
   WebhookEndpointUnknownError,
@@ -47,19 +43,9 @@ import {
   WebhookEventUnknownError,
 } from './webhook-errors';
 import type { WebhookLedger } from './webhook-ledger';
-import type { WebhookResolve, WebhookTarget } from './webhook-target';
+import type { WebhookResolve } from './webhook-target';
 import { resolveWebhookHost, webhookTarget } from './webhook-target';
 
-/** Just the call. `typeof fetch` also carries `preconnect`, which no test double should have to. */
-/**
- * The transport. `init.tls.serverName` carries the hostname a PINNED connection proves, because
- * the url it is handed names the approved address (`webhook-target.ts`); Bun's `fetch` honours it.
- */
-export type WebhookFetch = (
-  url: string,
-  init: RequestInit & { readonly tls?: { readonly serverName?: string } },
-) => Promise<Response>;
-
 /**
  * Consecutive failures before an endpoint stops taking deliveries. Ten is roughly a day of a
  * retrying queue against a dead receiver, which is long enough for an outage and short enough that
@@ -67,12 +53,6 @@ export type WebhookFetch = (
  */
 export const DEFAULT_WEBHOOK_DISABLE_AFTER = 10;
 
-/** The one content type a delivery announces. The BODY is the app's; how it is framed is not. */
-export const WEBHOOK_CONTENT_TYPE = 'application/json';
-
-/** `Retry-After` in seconds. The HTTP-date form is ignored on purpose — see `retryAfterSeconds`. */
-const RETRY_AFTER_SECONDS = /^\d{1,7}$/;
-
 export interface WebhookEndpoint {
   readonly id: string;
   /**
@@ -115,8 +95,23 @@ export interface WebhookDeliveryInput {
   readonly endpointId: string;
   /** Also the id the receiver dedupes on — it is signed, so it cannot be moved in transit. */
   readonly eventId: string;
+  /**
+   * The org that owns the endpoint, for a delivery declared `tenant: ({ orgId }) => orgId`. On the
+   * input because `tenant` is a synchronous function of the input alone — derived from the
+   * endpoint id it would be a read before the run has an org to read under. Neither signed nor
+   * sent: the receiver learns nothing it did not already know.
+   */
+  readonly orgId?: string | undefined;
 }
 
+/**
+ * Who a delivery runs as: `'none'`, or the org derived from its payload. May answer `undefined` —
+ * `({ orgId }) => orgId` over a row that carries none — and that answer is refused, TERMINALLY,
+ * before either seam reads (`X_JOB_TENANT_MISMATCH`). One signature for both shapes on purpose: a
+ * 25.0.0 `({ endpointId }) => …` that never reads `orgId` keeps enqueuing without one.
+ */
+export type WebhookTenant = 'none' | ((input: WebhookDeliveryInput) => string | undefined);
+
 /** What one landed delivery reports. Bounded, so `x jobs show` can print it. */
 export interface WebhookReport {
   readonly endpointId: string;
@@ -132,12 +127,13 @@ export interface WebhookDefinition {
    */
   readonly name: string;
   /**
-   * REQUIRED, exactly as on `job()`: `tenant: ({ endpointId }) => …` for a delivery scoped to the
-   * org that owns the endpoint, or the explicit `tenant: 'none'`. A delivery reads the app's own
-   * endpoint and event rows through the seams below, so the org those reads run under is a fact
-   * about the WORK and is declared here rather than inherited from whichever worker claimed it.
+   * REQUIRED, exactly as on `job()`: `tenant: ({ orgId }) => orgId` for a delivery scoped to the
+   * org that owns the endpoint — every enqueue then carries `orgId` — or the explicit
+   * `tenant: 'none'`. A delivery reads the app's own endpoint and event rows
+   * through the seams below, so the org those reads run under is a fact about the WORK and is
+   * declared here rather than inherited from whichever worker claimed it.
    */
-  readonly tenant: JobTenant<WebhookDeliveryInput>;
+  readonly tenant: WebhookTenant;
   /**
    * The endpoint this delivery is for. Read once PER ATTEMPT and never checkpointed: it carries a
    * secret, and a `step.run` output is written to `x_job_steps` — a credential in a durable table
@@ -145,11 +141,18 @@ export interface WebhookDefinition {
    */
   endpoint(args: {
     readonly endpointId: string;
+    /**
+     * The delivery's own `orgId`, off the input — by the time a seam runs, the org the run is
+     * under, or absent. Handed over so a seam names its tenant from the work rather than reading it
+     * back off `ctx.actor`.
+     */
+    readonly orgId: string | undefined;
     readonly ctx: Ctx;
   }): Promise<WebhookEndpoint | null> | WebhookEndpoint | null;
   /** The event's bytes. Read per attempt as well, out of the app's own table. */
   event(args: {
     readonly eventId: string;
+    readonly orgId: string | undefined;
     readonly ctx: Ctx;
   }): Promise<WebhookEvent | null> | WebhookEvent | null;
   /** Where every attempt is recorded, and where the consecutive-failure count comes from. */
@@ -184,13 +187,6 @@ export interface WebhookDefinition {
   readonly timeout?: DurationInput;
 }
 
-/** `Retry-After: 120`. The HTTP-date form needs the receiver's clock, which is what we do not trust. */
-const retryAfterSeconds = (response: Response): number | undefined => {
-  const header = response.headers.get('retry-after');
-  if (header === null || !RETRY_AFTER_SECONDS.test(header.trim())) return undefined;
-  return Number(header.trim());
-};
-
 export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliveryInput> {
   const clock = definition.clock ?? systemClock;
   const disableAfter = finiteOption(
@@ -200,18 +196,35 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
   );
   const send = definition.fetch ?? ((url, init) => fetch(url, init));
 
-  return job<WebhookDeliveryInput>({
+  // Named so `run` can read the deadline `job()` resolved — one parse of `timeout`, never two.
+  const handle: JobHandle<WebhookDeliveryInput> = job<WebhookDeliveryInput>({
     name: definition.name,
-    input: t.object({ endpointId: t.string, eventId: t.string }),
+    input: t.object({ endpointId: t.string, eventId: t.string, orgId: t.optional(t.string) }),
     // Endpoint AND event: the same event fans out to every subscribed endpoint, so a key on the
     // event alone would dedupe every one of those deliveries into the first endpoint's row.
     idempotencyKey: ({ endpointId, eventId }) => `${definition.name}:${endpointId}:${eventId}`,
-    tenant: definition.tenant,
+    tenant: tenantOf(definition),
     retry: definition.retry ?? DEFAULT_RETRY,
     ...(definition.queue === undefined ? {} : { queue: definition.queue }),
     ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
     async run({ input, ctx, attempt }): Promise<WebhookReport> {
-      const endpoint = await definition.endpoint({ endpointId: input.endpointId, ctx });
+      // On the clock `executeJob`'s deadline timer runs on, read as the body starts — which is
+      // when that timer is armed.
+      const deadline =
+        handle.timeoutMs === undefined
+          ? undefined
+          : {
+              atMs: performance.now() + handle.timeoutMs,
+              job: definition.name,
+              timeoutMs: handle.timeoutMs,
+            };
+      // Before either seam: the org they are handed must be the org every read runs under.
+      assertInputOrg(definition.name, input.orgId, ctx);
+      const endpoint = await definition.endpoint({
+        endpointId: input.endpointId,
+        orgId: input.orgId,
+        ctx,
+      });
       if (endpoint === null) {
         throw new WebhookEndpointUnknownError({
           webhook: definition.name,
@@ -220,7 +233,9 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
       }
       // Before the ledger and before the socket: a disabled endpoint costs nothing, which is the
       // whole point of disabling one.
-      if (endpoint.disabled === true) {
+      // Asked of BOTH sources, because `disableAfter` writes only the ledger: an endpoint the
+      // mechanism switched off must stop here whether or not the app mirrored it onto its row.
+      if (endpoint.disabled === true || (await definition.ledger.isDisabled(endpoint.id))) {
         throw new WebhookEndpointDisabledError({
           webhook: definition.name,
           endpointId: endpoint.id,
@@ -237,7 +252,7 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
         resolve: definition.resolve ?? resolveWebhookHost,
       });
 
-      const event = await definition.event({ eventId: input.eventId, ctx });
+      const event = await definition.event({ eventId: input.eventId, orgId: input.orgId, ctx });
       if (event === null) {
         throw new WebhookEventUnknownError({ webhook: definition.name, eventId: input.eventId });
       }
@@ -260,17 +275,41 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
       // again now, so a receiver's freshness window measures the request in front of it rather
       // than the age of the fact behind it.
       const timestampSeconds = Math.floor(nowMs(clock) / 1_000);
+      // The last moment a cancelled attempt can stop without the receiver hearing anything — the
+      // drain included: a POST not yet started is handed back to the next pod at no cost.
+      stopIfCancelled(ctx);
       const startedAt = clock.monotonic();
-      const outcome: Outcome = await ('unresolved' in target
-        ? Promise.resolve({ ok: false as const, status: null, detail: target.unresolved })
-        : attemptDelivery(send, endpoint, target, {
-            secret: endpoint.secret,
-            timestampSeconds,
-            eventId: input.eventId,
-            topic: event.topic,
-            body: event.body,
-          }));
+      // Past this line the drain no longer cancels: see `deliverySignal`.
+      const request = deliverySignal(ctx.signal, deadline);
+      let outcome: Outcome;
+      try {
+        outcome = await ('unresolved' in target
+          ? Promise.resolve({ ok: false as const, status: null, detail: target.unresolved })
+          : attemptDelivery(
+              send,
+              endpoint,
+              target,
+              {
+                secret: endpoint.secret,
+                timestampSeconds,
+                eventId: input.eventId,
+                topic: event.topic,
+                body: event.body,
+              },
+              request.signal,
+            ));
+      } finally {
+        request.dispose();
+      }
       const durationMs = Math.max(0, clock.monotonic() - startedAt);
+      // A failure the REQUEST's cancellation caused is not the receiver's, and is not this
+      // attempt's to record: the queue has already handed the delivery to another worker, whose
+      // outcome is the one the consecutive count must hear. Recorded, it was a failure nobody saw
+      // counting toward `disableAfter` — an abandoned attempt switching off a healthy endpoint.
+      // Read off the request's signal, never the attempt's: a receiver that really failed while
+      // the worker drained is a failure like any other, and the count hears it. A delivery that
+      // LANDED is a fact, and is always recorded.
+      if (!outcome.ok && request.signal.aborted) stopIfCancelled(ctx);
 
       // Recorded whatever happened, and BEFORE the throw: a failure that is not on the ledger is a
       // failure the consecutive count cannot see, which is an endpoint that never gets disabled.
@@ -311,6 +350,45 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
       throw deliveryError(definition.name, endpoint, outcome);
     },
   });
+  return handle;
+}
+
+/**
+ * The declared tenant, asked as declared, with one refusal added: an answer that names NO org. Only
+ * then — a 25.0.0 `tenant: ({ endpointId }) => …` never reads `orgId` and needs none, so requiring
+ * one would refuse every delivery it queued. The usual cause is a row queued before `orgId` existed
+ * under a `({ orgId }) => orgId` declaration. Coded and TERMINAL rather than `jobTenantFor`'s
+ * `assert`: an unclassified refusal spent the whole retry policy, and `x jobs retry`, re-proving it.
+ */
+function tenantOf(definition: WebhookDefinition): JobTenant<WebhookDeliveryInput> {
+  const declared = definition.tenant;
+  if (typeof declared !== 'function') return declared;
+  return (input) => {
+    const orgId = declared(input);
+    if (typeof orgId === 'string' && orgId.length > 0) return orgId;
+    throw new JobTenantMismatchError({
+      job: definition.name,
+      reason: `declares an org tenant, and this delivery's payload names no org for it${input.orgId === undefined ? ' (it carries no orgId)' : ''}`,
+      fix: `${definition.name}.enqueue({ endpointId, eventId, orgId }) — re-enqueue it with the org that owns the endpoint; a queued row cannot be retried into having one`,
+    });
+  };
+}
+
+/**
+ * The org a seam is HANDED is the org the run is UNDER, or the delivery is refused before either
+ * seam reads: a lookup filtered by `orgId` would otherwise read another org's endpoint, secret and
+ * all, inside this org's run. With `tenant: 'none'` nothing would check it at all, so any `orgId`
+ * is refused there. Terminal for `tenantOf`'s reason — the payload is the same on every attempt.
+ */
+function assertInputOrg(name: string, orgId: string | undefined, ctx: Ctx): void {
+  if (orgId === undefined) return;
+  const runOrg = ctx.actor.orgId ?? undefined;
+  if (orgId === runOrg) return;
+  throw new JobTenantMismatchError({
+    job: name,
+    reason: `was handed orgId ${orgId}, but the run is under ${runOrg === undefined ? "tenant 'none'" : `org ${runOrg}`}`,
+    fix: `webhook({ name: '${name}', tenant: ({ orgId }) => orgId, … }) — derive the tenant from the payload's own org, or enqueue ${name} with the orgId its tenant resolves to`,
+  });
 }
 
 /** A URL no delivery may open, or a secret that would make the POST unsigned. The parsed url. */
@@ -346,78 +424,14 @@ function assertDeliverable(name: string, endpoint: WebhookEndpoint): URL {
   return parsed;
 }
 
-type Outcome =
-  | { readonly ok: true; readonly status: number }
-  | {
-      readonly ok: false;
-      readonly status: number | null;
-      readonly detail: string;
-      readonly retryAfterSeconds?: number;
-    };
-
-async function attemptDelivery(
-  send: WebhookFetch,
-  endpoint: WebhookEndpoint,
-  target: WebhookTarget,
-  signing: Parameters<typeof webhookHeaders>[0],
-): Promise<Outcome> {
-  let response: Response;
-  try {
-    response = await send(target.url, {
-      method: 'POST',
-      headers: {
-        ...endpoint.headers,
-        // The name the pinned address is reached AS — after the row's own headers, so an endpoint
-        // cannot redirect the request to a different virtual host than the one it registered.
-        host: target.host,
-        'content-type': WEBHOOK_CONTENT_TYPE,
-        // LAST, so an endpoint row's own headers can never overwrite the signature it is proved by.
-        ...webhookHeaders(signing),
-      },
-      ...(target.serverName === undefined ? {} : { tls: { serverName: target.serverName } }),
-      body: signing.body,
-      // Never followed: a 3xx would re-POST a body signed for one host to whatever the receiver
-      // named, and the signature would travel with it.
-      redirect: 'manual',
-    });
-  } catch (error) {
-    // `renderThrowable`, never `String(error)` or `${error}`: this string lands in a `cause` and on
-    // a durable ledger row, and a null-prototype throwable makes both of those a `TypeError`.
-    return { ok: false, status: null, detail: renderThrowable(error) };
-  }
-  if (response.ok) return { ok: true, status: response.status };
-  const stated = retryAfterSeconds(response);
-  return {
-    ok: false,
-    status: response.status,
-    detail: `status ${response.status}`,
-    ...(stated === undefined ? {} : { retryAfterSeconds: stated }),
-  };
-}
-
-/** Which of the three failure codes this outcome is. The split is "can the same request land?". */
-function deliveryError(
-  name: string,
-  endpoint: WebhookEndpoint,
-  outcome: Extract<Outcome, { ok: false }>,
-): Error {
-  const base = { webhook: name, endpointId: endpoint.id, url: endpoint.url };
-  if (outcome.status === null) {
-    return new WebhookDeliveryFailedError({ ...base, status: null, detail: outcome.detail });
-  }
-  if (!isRetryableStatus(outcome.status)) {
-    return new WebhookDeliveryRejectedError({ ...base, status: outcome.status });
-  }
-  if (outcome.retryAfterSeconds !== undefined) {
-    return new WebhookDeliveryThrottledError({
-      ...base,
-      status: outcome.status,
-      retryAfterSeconds: outcome.retryAfterSeconds,
-    });
-  }
-  return new WebhookDeliveryFailedError({
-    ...base,
-    status: outcome.status,
-    detail: outcome.detail,
-  });
+/**
+ * Throw the attempt's cancellation as itself — the deadline's `X_JOB_TIMEOUT`, the drain's
+ * `X_DRAINING`, a lost lease — so `executeJob` reads the stop it caused, never a delivery failure
+ * naming a receiver that did nothing wrong. Core's `X_ABORTED` when the reason carries no code.
+ */
+function stopIfCancelled(ctx: Ctx): void {
+  if (!ctx.signal.aborted) return;
+  const reason: unknown = ctx.signal.reason;
+  if (isUltimateError(reason)) throw reason;
+  throwIfAborted(ctx);
 }

package/src/worker-admit.ts

@@ -76,8 +76,10 @@ export function createAdmission(options: AdmissionOptions): Admit {
       try {
         await shed(job, queue, limiter.blockedBy(key) ?? 'unknown');
       } catch (error) {
-        // The nack itself failed: the jobs BEHIND it go back before the round reports.
-        await handBack(driver, behind, back);
+        // The nack itself failed, so THIS job went nowhere either: it goes back with the jobs
+        // behind it, one more best-effort try, before the round reports. Left out, it sat
+        // `running` until its lease lapsed — an attempt spent on a body that never started.
+        await handBack(driver, [job, ...behind], back);
         throw error;
       }
       return { kind: 'waiting' };
@@ -128,7 +130,9 @@ export function createAdmission(options: AdmissionOptions): Admit {
       const keyed = grant.key === undefined ? '' : `, key ${grant.key}`;
       await shed(job, queue, `job concurrency (${grant.limit}${keyed})`);
     } catch (error) {
-      await handBack(driver, behind, back);
+      // This job too, for the reason above. A refusal that DID settle before throwing is safe to
+      // repeat: every nack is fenced on `running` and the claim, so a settled row ignores it.
+      await handBack(driver, [job, ...behind], back);
       throw error;
     }
     return { kind: 'waiting' };

package/src/worker-drain-cutoff.ts

@@ -0,0 +1,110 @@
+// When a draining worker stops waiting for the jobs it holds. SIGTERM stops the CLAIMING; the jobs
+// already running keep going, because a deploy that cancels work in flight re-runs every side
+// effect it interrupted. Only near the end of the drain budget is what is still running told to
+// stop (`cancel`), and a beat later what is still held goes back to the queue (`handBack`).
+
+import { logger, renderThrowable, systemClock } from '@ultimat3/core';
+import type { DrainBudget } from './drain-wait';
+import { JobDrainedError } from './errors';
+import type { HeldRuns } from './worker-held';
+
+/**
+ * The most of the budget's tail kept for unwinding: the cancel lands this far before the deadline,
+ * the hand-back half as far, and the teardown — the registry row, the driver — gets the rest. Two
+ * seconds of a 25 s default, and never more than HALF of a small budget, so a budget of 300 ms
+ * still spends 150 ms letting its jobs finish before anything is cut.
+ */
+export const DRAIN_CANCEL_MARGIN_MS = 2_000;
+
+export interface CutoffTimes {
+  /** Real monotonic ms: the held runs' `ctx.signal` aborts with `X_DRAINING`. */
+  readonly cancelAt: number;
+  /** Real monotonic ms: every claim still held is handed back uncounted. */
+  readonly handBackAt: number;
+}
+
+/** The cut-off for a deadline, read against the instant it was bound. */
+export function cutoffTimes(deadlineAt: number, now: number): CutoffTimes {
+  const remaining = Math.max(0, deadlineAt - now);
+  const margin = Math.min(DRAIN_CANCEL_MARGIN_MS, remaining / 2);
+  return { cancelAt: deadlineAt - margin, handBackAt: deadlineAt - margin / 2 };
+}
+
+export interface DrainCutoff {
+  /** Stop listening to the budget and clear both timers — the teardown is over. */
+  dispose(): void;
+}
+
+/**
+ * Arm `cancel` and `handBack` on `budget`, re-armed each time it tightens (the earliest deadline
+ * wins — `drain-wait.ts`). Each fires at most once. The timers are `unref`ed: a drained process is
+ * never kept alive by its own cut-off.
+ */
+export function armDrainCutoff(
+  budget: DrainBudget,
+  steps: { readonly cancel: () => void; readonly handBack: () => Promise<void> },
+): DrainCutoff {
+  const timers: ReturnType<typeof setTimeout>[] = [];
+  let cancelled = false;
+  let handedBack = false;
+  const clear = (): void => {
+    for (const timer of timers.splice(0)) clearTimeout(timer);
+  };
+  const at = (instant: number, fire: () => void): void => {
+    const timer = setTimeout(fire, Math.max(0, instant - systemClock.monotonic()));
+    timer.unref?.();
+    timers.push(timer);
+  };
+  const unwatch = budget.watch((deadlineAt) => {
+    clear();
+    const times = cutoffTimes(deadlineAt, systemClock.monotonic());
+    if (!cancelled) {
+      at(times.cancelAt, () => {
+        cancelled = true;
+        steps.cancel();
+      });
+    }
+    if (!handedBack) {
+      at(times.handBackAt, () => {
+        handedBack = true;
+        // Each nack's failure is logged where it happens (`worker-hand-back.ts`); this is the one
+        // place anything else could still escape a timer, so it is observed here.
+        void steps.handBack().catch((error: unknown) => {
+          logger.error('jobs.worker.drain-hand-back-failed', { error: renderThrowable(error) });
+        });
+      });
+    }
+  });
+  return {
+    dispose() {
+      unwatch();
+      clear();
+    },
+  };
+}
+
+/**
+ * The worker's cut-off: `drain` aborted with `X_DRAINING` at `cancelAt`, every run `held` still
+ * holds handed back at `handBackAt`. Aborted even with nothing held: a round still inside its
+ * `claim()` starts its runs under this controller, and they must hear the cut-off too.
+ */
+export function armWorkerCutoff(input: {
+  readonly budget: DrainBudget;
+  readonly workerId: string;
+  /** The shutdown's signal name, carried on the reason the bodies read. */
+  readonly signal: string;
+  readonly drain: AbortController;
+  readonly held: HeldRuns;
+}): DrainCutoff {
+  const { workerId, held } = input;
+  return armDrainCutoff(input.budget, {
+    cancel: () => {
+      if (held.size > 0) logger.warn('jobs.worker.drain-cancelling', { workerId, held: held.size });
+      input.drain.abort(new JobDrainedError({ workerId, signal: input.signal }));
+    },
+    handBack: async () => {
+      const handed = await held.handBackAll();
+      if (handed > 0) logger.warn('jobs.worker.drain-handed-back', { workerId, handed });
+    },
+  });
+}

package/src/worker-fleet-slots.ts

@@ -73,7 +73,7 @@ export interface FleetSlots {
    * to do without one. `granted` is also the answer for "no cap declared": a job with no
    * `concurrency` never touches the lease table.
    *
-   * A driver with no lease store cannot reach here: `createWorker().start()` refuses to boot when
+   * A driver with no lease store cannot reach here: `jobWorker().start()` refuses to boot when
    * a registered job declares `concurrency` and the driver has none, because a cap that silently
    * holds per process is the documented-guarantee-that-does-nothing axiom 3 exists to make
    * impossible.
@@ -186,6 +186,9 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
         // One in flight at a time, on the connection that is already the thing failing.
         if (renewing) return;
         renewing = true;
+        // Stamped BEFORE the call, as `heartbeat.ts` does: the store dates the new expiry at the
+        // statement, so a slow reply must not stretch this worker's view past it.
+        const sentAt = now();
         try {
           const renewed = await options.leases.renew(slot, options.ttlMs);
           if (timer.stopped()) return;
@@ -194,7 +197,7 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
           if (renewed === false) return reportLost('not-ours');
           // A renewal that lands LATE is still late: the slot may already be another worker's.
           if (lapsed()) return reportLost('expired');
-          renewedAt = now();
+          renewedAt = sentAt;
         } catch (error) {
           // One failed renewal is not a lost slot: the TTL gives the interval several tries.
           logger.warn('jobs.worker.slot-renewal-failed', {

package/src/worker-held.ts

@@ -0,0 +1,100 @@
+// The runs one worker holds right now, each with the claim it runs under — so a drain that reaches
+// its cut-off can hand back every row still held instead of leaving it under a lease another
+// worker waits a whole visibility timeout to see lapse. Split from `worker.ts`, whose job is the
+// claim loop; this one is "what am I holding, and give it back".
+
+import type { ClaimedJob, JobDriver } from './driver';
+import { handBack } from './worker-hand-back';
+
+export interface HeldRuns {
+  /** How many runs are held — what `stats().inFlight` reports. */
+  readonly size: number;
+  /**
+   * The job ids held, each once — what the registry row reports. Per RUN underneath, because one
+   * worker can hold a job twice (a claim that lapsed under a body still running, then its own
+   * re-claim): keyed by id, the first run to end dropped an id the second still held.
+   */
+  ids(): string[];
+  /**
+   * A run started. `release` is what the run's own `finally` would do for the process — the
+   * in-flight count core's drain waits on, the fleet slot — run early when the run is handed back.
+   */
+  add(run: Promise<unknown>, claimed: ClaimedJob, release: () => Promise<void>): void;
+  /** The run settled on its own: no longer held. */
+  settle(run: Promise<unknown>): void;
+  /** Resolves the moment nothing is held — settled or handed back. */
+  empty(): Promise<void>;
+  /**
+   * Every run still held goes back to the queue — `countsAsAttempt: false`, no delay — and stops
+   * being this worker's. Answers how many. The bodies themselves cannot be stopped; what they
+   * write after this is fenced out by the claim that just moved (`StepFence`, `SQL_ACK`).
+   */
+  handBackAll(): Promise<number>;
+}
+
+interface Holding {
+  readonly claimed: ClaimedJob;
+  readonly release: () => Promise<void>;
+}
+
+export function createHeldRuns(options: {
+  readonly driver: JobDriver;
+  readonly workerId: string;
+}): HeldRuns {
+  const held = new Map<Promise<unknown>, Holding>();
+  let waiters: (() => void)[] = [];
+
+  /** Pending hand-backs: `empty()` answers only once their nacks have LANDED, never mid-wire. */
+  let handingBack = 0;
+  const wakeIfEmpty = (): void => {
+    if (held.size > 0 || handingBack > 0 || waiters.length === 0) return;
+    const woken = waiters;
+    waiters = [];
+    for (const wake of woken) wake();
+  };
+
+  return {
+    get size() {
+      return held.size;
+    },
+    ids() {
+      return [...new Set([...held.values()].map((holding) => holding.claimed.id))];
+    },
+    add(run, claimed, release) {
+      held.set(run, { claimed, release });
+    },
+    settle(run) {
+      held.delete(run);
+      wakeIfEmpty();
+    },
+    empty() {
+      if (held.size === 0 && handingBack === 0) return Promise.resolve();
+      return new Promise<void>((resolve) => {
+        waiters.push(resolve);
+      });
+    },
+    async handBackAll() {
+      // Taken out of the set BEFORE the first await: a run settling while the nacks are on the
+      // wire is a run the hand-back already owns, and must not be counted twice. And `empty()`
+      // stays unanswered until the nacks land — the teardown waiting on it closes the driver next.
+      const stranded = [...held.values()];
+      held.clear();
+      if (stranded.length === 0) return 0;
+      handingBack += 1;
+      try {
+        await handBack(
+          options.driver,
+          stranded.map((holding) => holding.claimed),
+          { delayMs: 0, workerId: options.workerId },
+        );
+        // After the nacks, never before: the slot is what lets the replacement run a
+        // `concurrency: 1` job, and freed first it could be taken while the row was still ours.
+        for (const holding of stranded) await holding.release();
+      } finally {
+        handingBack -= 1;
+        wakeIfEmpty();
+      }
+      return stranded.length;
+    },
+  };
+}