@ultimat3/jobs 24.0.0 → 25.0.0

package/src/scheduler-leader.ts

@@ -15,8 +15,8 @@ export interface LeaderElection {
   readonly renewEveryMs: number;
 }
 
-/** Single-node default: always the leader. Multi-node uses `createPgLeaseLeader()` — never
- * `createPgLeader()`, whose advisory lock is owned by a pooled session this process cannot name. */
+/** Single-node default: always the leader. Multi-node uses `postgresLeaseLeader()` — never
+ * `postgresLeader()`, whose advisory lock is owned by a pooled session this process cannot name. */
 export function soleLeader(): LeaderElection {
   return {
     acquire: () => Promise.resolve(true),

package/src/scheduler-pg.ts

@@ -4,7 +4,7 @@
 // pod it replaced dropped, and two pods in a rolling update both dispatch every task.
 
 import type { Clock, PgExecutor } from '@ultimat3/core';
-import { finiteOption, uuid } from '@ultimat3/core';
+import { finiteOption, uuidV7 } from '@ultimat3/core';
 import { nowMs } from './clock';
 import { DEFAULT_QUEUE } from './driver';
 import { SQL_SCHEDULER_FIRE } from './driver-pg-operator-sql';
@@ -24,7 +24,7 @@ import { fireThroughDriver } from './scheduler-state';
  * whose watermark dies with its process has none of them — it takes the arming branch on every
  * boot and drops every occurrence between the two processes with nothing logged.
  */
-export function pgSchedulerState(executor: PgExecutor): SchedulerState {
+export function postgresSchedulerState(executor: PgExecutor): SchedulerState {
   const state: SchedulerState = {
     async lastFiredAt(taskName) {
       const rows = await executor.query<{ last_fired_at: number | string | null }>(
@@ -41,12 +41,11 @@ export function pgSchedulerState(executor: PgExecutor): SchedulerState {
       // One statement only when the queue IS this database. A host that swapped the driver keeps
       // the watermark here and its jobs elsewhere, and no statement spans the two.
       if (driver.name !== 'pg') return fireThroughDriver(state, driver, fire);
-      const ids = fire.jobs.map(() => uuid());
+      const ids = fire.jobs.map(() => uuidV7());
       const rows = await executor.query<{
         fired: number | string;
         id: string | null;
         run_id: string | null;
-        idempotency_key: string | null;
       }>(SQL_SCHEDULER_FIRE, [
         fire.task,
         fire.occurrenceMs,
@@ -57,7 +56,7 @@ export function pgSchedulerState(executor: PgExecutor): SchedulerState {
             queue: request.queue || DEFAULT_QUEUE,
             input: request.input ?? null,
             idempotency_key: request.idempotencyKey,
-            run_id: request.runId ?? uuid(),
+            run_id: request.runId ?? uuidV7(),
             max_attempts: request.maxAttempts,
           })),
         ),
@@ -65,17 +64,20 @@ export function pgSchedulerState(executor: PgExecutor): SchedulerState {
         fire.watermarkMs ?? null,
       ]);
       if (Number(rows[0]?.fired ?? 0) === 0) return undefined;
+      // Keyed on the id THIS statement minted (`ids[index]`), never on the idempotency key: the
+      // key is unique per (name, tenant, key), so two jobs of one occurrence may share it, and a
+      // map by key told both callers the second one's id — where the memory store tells each its own.
       const queued = new Map(
         rows.flatMap((row) =>
-          row.id === null || row.run_id === null || row.idempotency_key === null
+          row.id === null || row.run_id === null
             ? []
-            : [[row.idempotency_key, { id: row.id, runId: row.run_id }] as const],
+            : [[row.id.toLowerCase(), { id: row.id, runId: row.run_id }] as const],
         ),
       );
       // A job absent from the answer met a LIVE row of its key — a fire from before this
       // statement existed, still in flight. It is that row's work, so it is reported deduped.
-      return fire.jobs.map((request) => {
-        const inserted = queued.get(request.idempotencyKey);
+      return fire.jobs.map((_, index) => {
+        const inserted = queued.get(String(ids[index]).toLowerCase());
         return inserted === undefined
           ? { id: '', runId: '', deduped: true }
           : { ...inserted, deduped: false };
@@ -85,7 +87,7 @@ export function pgSchedulerState(executor: PgExecutor): SchedulerState {
   return state;
 }
 
-export interface PgLeaseLeaderOptions {
+export interface PostgresLeaseLeaderOptions {
   readonly executor: PgExecutor;
   /** One key per elected role. Default `'scheduler'`. */
   readonly lockKey?: string;
@@ -107,9 +109,9 @@ export const LEASE_RENEWALS_PER_TTL = 3;
 
 /**
  * Leader election as an expiring row, which is what makes it correct on the executor this package
- * is actually handed: a POOL. `createPgLeader`'s `pg_try_advisory_lock` is session-scoped, and a
+ * is actually handed: a POOL. `postgresLeader`'s `pg_try_advisory_lock` is session-scoped, and a
  * session on a pool ends the moment the connection goes back — so every node reads itself as
- * leader and a rolling update double-fires every task. `@ultimat3/realtime`'s `PgAdvisoryLock`
+ * leader and a rolling update double-fires every task. `@ultimat3/realtime`'s `postgresAdvisoryLock()`
  * solves the same problem by owning its connection; this package holds no wire protocol, so it
  * solves it with a row instead.
  *
@@ -118,9 +120,9 @@ export const LEASE_RENEWALS_PER_TTL = 3;
  * with nothing to clean up, which is the one property the advisory lock had and a plain
  * `insert ... on conflict do nothing` would not.
  */
-export function createPgLeaseLeader(options: PgLeaseLeaderOptions): LeaderElection {
+export function postgresLeaseLeader(options: PostgresLeaseLeaderOptions): LeaderElection {
   const lockKey = options.lockKey ?? 'scheduler';
-  const holder = options.holder ?? `scheduler-${uuid()}`;
+  const holder = options.holder ?? `scheduler-${uuidV7()}`;
   const ttlMs = finiteOption(
     'the pg scheduler lease',
     'ttlMs',

package/src/scheduler-state.ts

@@ -39,7 +39,7 @@ export interface SchedulerState {
 /**
  * The fire for a store that cannot share a statement with the queue: the watermark is checked,
  * the jobs go through `driver.enqueue`, and the watermark moves last. Not atomic across a crash —
- * which is why `pgSchedulerState` does not use it against the pg driver — but exact for the
+ * which is why `postgresSchedulerState` does not use it against the pg driver — but exact for the
  * memory pair, where nothing can fail between the steps and no second process exists.
  */
 export async function fireThroughDriver(
@@ -58,7 +58,7 @@ export async function fireThroughDriver(
   return results;
 }
 
-export function createMemorySchedulerState(): SchedulerState {
+export function memorySchedulerState(): SchedulerState {
   const fired = new Map<string, number>();
   const state: SchedulerState = {
     lastFiredAt: (taskName) => Promise.resolve(fired.get(taskName)),

package/src/scheduler.ts

@@ -2,7 +2,7 @@
 // it owes and enqueues their jobs. The `task` primitive it reads lives in `task.ts`.
 //
 // Exactly one node dispatches per tick, enforced by leader election. Multi-node that is an
-// EXPIRING LEASE ROW in `x_scheduler_leader` (`createPgLeaseLeader`), never an advisory lock: an
+// EXPIRING LEASE ROW in `x_scheduler_leader` (`postgresLeaseLeader`), never an advisory lock: an
 // advisory lock is held by the SESSION, not by this process — it outlives every transaction and is
 // released only by an explicit unlock, the pool's reset on release, or the connection dying, and
 // the next round may run on a different connection. So a node can neither renew it nor prove it
@@ -31,7 +31,7 @@ import type { LeaderElection } from './scheduler-leader';
 import { soleLeader } from './scheduler-leader';
 import { defaultCronResolver, latestOccurrence, occurrencesIn } from './scheduler-occurrences';
 import type { SchedulerState } from './scheduler-state';
-import { createMemorySchedulerState } from './scheduler-state';
+import { memorySchedulerState } from './scheduler-state';
 import type { TaskHandle, TaskJobResult } from './task';
 import { registeredTasks } from './task';
 
@@ -89,11 +89,11 @@ export interface Scheduler {
   nextRunFor(handle: TaskHandle, from?: Date): Date;
 }
 
-export function createScheduler(options: SchedulerOptions): Scheduler {
-  const schedulerState = options.state ?? createMemorySchedulerState();
+export function jobScheduler(options: SchedulerOptions): Scheduler {
+  const schedulerState = options.state ?? memorySchedulerState();
   const resolveCron = options.cron ?? defaultCronResolver;
   const tickIntervalMs = finiteOption(
-    'createScheduler',
+    'jobScheduler',
     'tickIntervalMs',
     options.tickIntervalMs ?? 1_000,
   );

package/src/steps-memory.ts

@@ -1,11 +1,11 @@
-// The in-memory `StepStore`: what `createMemoryDriver` and every runner test persist steps into.
+// The in-memory `StepStore`: what `memoryJobDriver` and every runner test persist steps into.
 // Split from `steps.ts` at the file-size ceiling, along the seam the pg driver already draws —
 // `driver-pg.ts` holds the Postgres store, this file the map-backed one, `steps.ts` the runner
 // both are handed to. Same contract, and `steps.test.ts` is where the contract is pinned.
 
 import type { StepRecord, StepStore } from './steps';
 
-export function createMemoryStepStore(): StepStore {
+export function memoryStepStore(): StepStore {
   const byRun = new Map<string, Map<string, StepRecord>>();
   const runOf = (runId: string): Map<string, StepRecord> => {
     let run = byRun.get(runId);

package/src/steps.ts

@@ -7,7 +7,7 @@
 // catches it and re-queues the job for `resumeAt` instead of holding a process for three days.
 
 import type { Clock } from '@ultimat3/core';
-import { finiteOption, logger, renderThrowable } from '@ultimat3/core';
+import { finiteOption, isUltimateError, logger, renderThrowable } from '@ultimat3/core';
 import { expectedQueryLoop } from '@ultimat3/db';
 import type { DurationInput } from './clock';
 import { finiteDurationMs, nowMs } from './clock';
@@ -138,6 +138,13 @@ export interface StepRunnerOptions {
    * store belongs to whichever attempt has it now.
    */
   readonly signal?: AbortSignal;
+  /**
+   * Aborted the moment the ATTEMPT is over — settled, or past its job deadline. `signal` alone
+   * cannot say so once the drain has aborted it first (a controller keeps its first reason), and
+   * the drain's reason writes through `put` (`writesThrough`): this is what still refuses a step
+   * left in flight by a body that returned without awaiting it. `executeJob` passes it.
+   */
+  readonly ended?: AbortSignal;
 }
 
 export interface StepRunner {
@@ -168,7 +175,7 @@ function trace(into: string[], name: string): void {
   if (into.length > MAX_TRACE_NAMES) into.shift();
 }
 
-export function createStepRunner(options: StepRunnerOptions): StepRunner {
+export function stepRunner(options: StepRunnerOptions): StepRunner {
   const { runId, jobName, store } = options;
   /** Every name this attempt has claimed. Membership only — the trace is `used`. */
   const claimed = new Set<string>();
@@ -176,6 +183,13 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
   const replayed: string[] = [];
   const clock = options.clock;
   const pollMs = finiteOption('step.waitForEvent', 'eventPollMs', options.eventPollMs ?? 30_000);
+  // `job()` refuses a non-finite `stepTimeout` at declaration (`job.ts`); screened again here, where
+  // it is read, because this runner is also built directly and `withStepTimeout` reads a `NaN`
+  // ceiling as none at all — the backstop `pollMs` above already is.
+  const stepTimeoutMs =
+    options.stepTimeoutMs === undefined
+      ? undefined
+      : finiteOption('step.run', 'stepTimeoutMs', options.stepTimeoutMs);
   const runSignal = options.signal ?? NEVER_ABORTED;
 
   /**
@@ -233,6 +247,19 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
 
   const cancelled = (): boolean => runSignal.aborted;
 
+  /**
+   * A cancelled run whose step result is still this attempt's to record: the worker's drain cut
+   * it short (`X_DRAINING`, `worker-drain-cutoff.ts`), the write is fenced on the claim, and the
+   * attempt has not ended. The side effect behind the record HAPPENED — refused, the next worker
+   * runs it again. The fence is what makes this safe and not the in-process signal: a row the
+   * drain has already handed back answers the write `X_JOB_LEASE_LOST` in the store.
+   */
+  const writesThrough = (): boolean =>
+    options.fence !== undefined &&
+    options.ended?.aborted !== true &&
+    isUltimateError(runSignal.reason) &&
+    runSignal.reason.code === 'X_DRAINING';
+
   /**
    * The statement itself, declared deliberate to the N+1 detector. One write per step IS the
    * design this file's header states — each step completes at its own instant and its output has
@@ -255,7 +282,9 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
    * never ran — or overwrite one it did. The write is refused, and refusing it unwinds the body.
    */
   const put = async (record: StepRecord): Promise<void> => {
-    if (cancelled()) throw new JobAbortedError({ job: jobName, step: record.name });
+    if (cancelled() && !writesThrough()) {
+      throw new JobAbortedError({ job: jobName, step: record.name });
+    }
     await persist(record);
     remember(record);
   };
@@ -283,15 +312,14 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
     // to compose, and DISPOSED in the `finally` below, which is the whole reason `run-signal.ts`
     // exists — `worker-run.ts` disposes the run's own the same way.
     const composed =
-      options.stepTimeoutMs === undefined ? null : createRunSignal([runSignal, deadline.signal]);
+      stepTimeoutMs === undefined ? null : createRunSignal([runSignal, deadline.signal]);
     const signal = composed?.signal ?? runSignal;
     try {
       const output = await withStepTimeout(
         fn(signal),
-        options.stepTimeoutMs,
+        stepTimeoutMs,
         deadline,
-        () =>
-          new JobTimeoutError({ job: jobName, step: name, timeoutMs: options.stepTimeoutMs ?? 0 }),
+        () => new JobTimeoutError({ job: jobName, step: name, timeoutMs: stepTimeoutMs ?? 0 }),
       );
       // Persist BEFORE returning: a crash one line later must not re-run this step.
       await put({

package/src/task.ts

@@ -141,7 +141,7 @@ export function task(definition: TaskDefinition): TaskHandle {
   // `for (let i = 0; i < handle.maxCatchUp; i += 1)`, so zero (and any negative, and any fraction
   // below one) returns an empty list on every round and the task NEVER fires: no error, no log
   // line, no queue row, forever. Refused where it is written, exactly as `job()` refuses
-  // `concurrency: 0` and `createPacer` refuses `rate: 0`. `Number.isInteger` covers `NaN` and
+  // `concurrency: 0` and `backfillPacer` refuses `rate: 0`. `Number.isInteger` covers `NaN` and
   // `Infinity` in the same predicate — an unbounded catch-up is a burst nobody declared.
   assert(
     definition.maxCatchUp === undefined ||

package/src/webhook-attempt.ts

@@ -0,0 +1,206 @@
+// One delivery attempt on the wire: the headers it carries, the POST, what the answer means, and
+// which of the three failure codes it becomes. Everything BEFORE the socket — the endpoint, the
+// screen, the event, the gates — and everything AFTER it — the ledger, disable-after-N — is
+// `webhook.ts`'s; this file only ever sees an approved target and a signed body.
+
+import {
+  // Core's ONE table, never a fifth copy of it. The copy that lived here omitted 409, so a
+  // receiver saying "a concurrent writer won this round" dead-lettered on attempt 1 as a refusal
+  // no retry could change — the exact divergence `retryable-status.ts` was extracted to end.
+  isRetryableStatus,
+  isUltimateError,
+  renderThrowable,
+  retryAfterSecondsOf,
+  webhookHeaders,
+} from '@ultimat3/core';
+import { JobTimeoutError } from './errors';
+import {
+  WebhookDeliveryFailedError,
+  WebhookDeliveryRejectedError,
+  WebhookDeliveryThrottledError,
+} from './webhook-errors';
+import type { WebhookTarget } from './webhook-target';
+
+/**
+ * The transport — just the call: `typeof fetch` also carries `preconnect`, which no test double
+ * should have to. `init.tls.serverName` carries the hostname a PINNED connection proves, because
+ * the url it is handed names the approved address (`webhook-target.ts`); Bun's `fetch` honours it.
+ */
+export type WebhookFetch = (
+  url: string,
+  init: RequestInit & { readonly tls?: { readonly serverName?: string } },
+) => Promise<Response>;
+
+/** The one content type a delivery announces. The BODY is the app's; how it is framed is not. */
+export const WEBHOOK_CONTENT_TYPE = 'application/json';
+
+/** What of a `WebhookEndpoint` an attempt reads. Never the secret: that is inside `signing`. */
+export interface DeliveryEndpoint {
+  readonly id: string;
+  readonly url: string;
+  readonly headers?: Readonly<Record<string, string>>;
+}
+
+export type Outcome =
+  | { readonly ok: true; readonly status: number }
+  | {
+      readonly ok: false;
+      readonly status: number | null;
+      readonly detail: string;
+      readonly retryAfterSeconds?: number;
+    };
+
+/**
+ * The receiver's `Retry-After`, through core's one reader: delta-seconds as before, and now the
+ * HTTP-date form too, measured against the SAME response's `Date` — the receiver's clock against
+ * the receiver's clock, which is what made the date safe to read without trusting ours. Capped at
+ * a day, and the nack clamps it to the policy's `maxDelay` on top.
+ */
+const retryAfterSeconds = (response: Response): number | undefined =>
+  retryAfterSecondsOf(response.headers.get('retry-after'), response.headers.get('date'));
+
+/**
+ * The row's own headers, then the framework's — each SET, case-insensitively, over whatever the row
+ * said. An object spread cannot do this: `{ Host: 'evil.test', host: <pinned> }` keeps both keys and
+ * `fetch` joins them, so a row would reach another virtual host, change the content type, and ship
+ * a forged value inside the signature list. Going through `Headers` collapses every spelling of a
+ * name to one entry before the framework's value replaces it. Returned as a plain lowercase record,
+ * because an injected `WebhookFetch` has always been handed one.
+ *
+ * Inside the caller's `try`: a row header the platform refuses (a newline in a value) is the same
+ * failed attempt it always was, never a bare `TypeError` out of the job.
+ */
+function deliveryHeaders(
+  endpoint: DeliveryEndpoint,
+  target: WebhookTarget,
+  signing: Parameters<typeof webhookHeaders>[0],
+): Record<string, string> {
+  const headers = new Headers(endpoint.headers);
+  // The name the pinned address is reached AS, so an endpoint cannot redirect the request to a
+  // different virtual host than the one it registered.
+  headers.set('host', target.host);
+  headers.set('content-type', WEBHOOK_CONTENT_TYPE);
+  // The signature and the event identity it covers: a row can never overwrite what it is proved by.
+  for (const [name, value] of Object.entries(webhookHeaders(signing))) headers.set(name, value);
+  return Object.fromEntries(headers.entries());
+}
+
+/** The reason `JobDrainedError` carries — the one cancellation a request on the wire outlives. */
+const DRAINING = 'X_DRAINING';
+
+const isDrain = (reason: unknown): boolean => isUltimateError(reason) && reason.code === DRAINING;
+
+/** The request's own signal, and the listener it hangs on the attempt's, handed back after. */
+export interface DeliverySignal {
+  readonly signal: AbortSignal;
+  dispose(): void;
+}
+
+/** Where this attempt's own deadline falls, on `performance.now()`'s clock. */
+export interface AttemptDeadline {
+  readonly atMs: number;
+  readonly job: string;
+  readonly timeoutMs: number;
+}
+
+/**
+ * The attempt's signal for every reason but the DRAIN. A deadline or a lost lease is followed by a
+ * nack that hands the job to another worker at once, so a request still open is the receiver
+ * getting the event twice: those tear it down. The drain is the opposite case — tearing a POST
+ * already on the wire down hands the job to the next pod, which re-POSTs the same event on every
+ * deploy — so the request finishes inside the drain budget instead, and its outcome is recorded.
+ *
+ * Not past the attempt's DEADLINE, though. The attempt's signal keeps its FIRST reason, so once the
+ * drain has aborted it the deadline behind it never arrives there — while `executeJob` still hands
+ * the job on when the deadline passes. So a drained request arms the deadline itself.
+ */
+export function deliverySignal(attempt: AbortSignal, deadline?: AttemptDeadline): DeliverySignal {
+  const request = new AbortController();
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const follow = (): void => {
+    if (!isDrain(attempt.reason)) {
+      request.abort(attempt.reason);
+      return;
+    }
+    if (deadline === undefined) return;
+    timer = setTimeout(
+      () =>
+        request.abort(new JobTimeoutError({ job: deadline.job, timeoutMs: deadline.timeoutMs })),
+      Math.max(0, deadline.atMs - performance.now()),
+    );
+  };
+  if (attempt.aborted) follow();
+  else attempt.addEventListener('abort', follow, { once: true });
+  return {
+    signal: request.signal,
+    dispose: () => {
+      attempt.removeEventListener('abort', follow);
+      if (timer !== undefined) clearTimeout(timer);
+    },
+  };
+}
+
+/**
+ * One POST, on `deliverySignal`'s signal: torn down by the deadline or a lost lease, never by the
+ * drain.
+ */
+export async function attemptDelivery(
+  send: WebhookFetch,
+  endpoint: DeliveryEndpoint,
+  target: WebhookTarget,
+  signing: Parameters<typeof webhookHeaders>[0],
+  signal: AbortSignal,
+): Promise<Outcome> {
+  let response: Response;
+  try {
+    response = await send(target.url, {
+      method: 'POST',
+      headers: deliveryHeaders(endpoint, target, signing),
+      ...(target.serverName === undefined ? {} : { tls: { serverName: target.serverName } }),
+      body: signing.body,
+      // Never followed: a 3xx would re-POST a body signed for one host to whatever the receiver
+      // named, and the signature would travel with it.
+      redirect: 'manual',
+      signal,
+    });
+  } catch (error) {
+    // `renderThrowable`, never `String(error)` or `${error}`: this string lands in a `cause` and on
+    // a durable ledger row, and a null-prototype throwable makes both of those a `TypeError`.
+    return { ok: false, status: null, detail: renderThrowable(error) };
+  }
+  if (response.ok) return { ok: true, status: response.status };
+  const stated = retryAfterSeconds(response);
+  return {
+    ok: false,
+    status: response.status,
+    detail: `status ${response.status}`,
+    ...(stated === undefined ? {} : { retryAfterSeconds: stated }),
+  };
+}
+
+/** Which of the three failure codes this outcome is. The split is "can the same request land?". */
+export function deliveryError(
+  name: string,
+  endpoint: DeliveryEndpoint,
+  outcome: Extract<Outcome, { ok: false }>,
+): Error {
+  const base = { webhook: name, endpointId: endpoint.id, url: endpoint.url };
+  if (outcome.status === null) {
+    return new WebhookDeliveryFailedError({ ...base, status: null, detail: outcome.detail });
+  }
+  if (!isRetryableStatus(outcome.status)) {
+    return new WebhookDeliveryRejectedError({ ...base, status: outcome.status });
+  }
+  if (outcome.retryAfterSeconds !== undefined) {
+    return new WebhookDeliveryThrottledError({
+      ...base,
+      status: outcome.status,
+      retryAfterSeconds: outcome.retryAfterSeconds,
+    });
+  }
+  return new WebhookDeliveryFailedError({
+    ...base,
+    status: outcome.status,
+    detail: outcome.detail,
+  });
+}

package/src/webhook-ledger.ts

@@ -39,11 +39,19 @@ export interface WebhookLedger {
    */
   record(attempt: WebhookAttempt): Promise<number>;
   /**
-   * Stop delivering to this endpoint. Called once, on the attempt whose count reached
-   * `disableAfter`. Re-enabling is the app's — a dead endpoint that heals itself is a retry loop
-   * with no end.
+   * Stop delivering to this endpoint. Called by the attempt whose count reached `disableAfter` —
+   * and, rarely, by a second attempt that was already in flight when the first one crossed it, so
+   * an implementation is IDEMPOTENT: disabling a disabled endpoint changes nothing. Re-enabling is
+   * the app's — a dead endpoint that heals itself is a retry loop with no end.
    */
   disable(endpointId: string, reason: string): Promise<void>;
+  /**
+   * Whether `disable` has switched this endpoint off. Asked before every attempt opens a socket,
+   * because `disable` is the only thing the mechanism writes: a ledger that recorded the verdict
+   * and was never asked for it again is an endpoint that keeps receiving POSTs. An app whose
+   * `disable` sets its endpoint row's `disabled` column answers from that same column.
+   */
+  isDisabled(endpointId: string): Promise<boolean>;
 }
 
 /** Attempts one process keeps before the oldest go. A dev ledger that grows forever is a leak. */
@@ -88,9 +96,12 @@ export function memoryWebhookLedger(
       return Promise.resolve(failures);
     },
     disable(endpointId: string, reason: string): Promise<void> {
-      off.set(endpointId, reason);
+      // The FIRST reason stays: it names the run that switched the endpoint off, and a late
+      // in-flight attempt repeating the verdict must not overwrite it.
+      if (!off.has(endpointId)) off.set(endpointId, reason);
       return Promise.resolve();
     },
+    isDisabled: (endpointId: string): Promise<boolean> => Promise.resolve(off.has(endpointId)),
     attempts: (): readonly WebhookAttempt[] => [...attempts],
     disabled: (): ReadonlyMap<string, string> => new Map(off),
     reset(): void {