@ultimat3/jobs 11.3.0 → 13.0.0

package/src/webhook-errors.ts

@@ -0,0 +1,170 @@
+// The eight `X_WEBHOOK_*` codes an outbound delivery can end on, apart from `errors.ts` for the
+// reason `backfill-errors.ts` is: one file, one job, and `errors.ts` holds the registry. The codes,
+// the titles and the single `registerErrorCodes()` call stay there; only the classes live here.
+//
+// THE RULE THIS FILE EXISTS TO KEEP: no secret and no signature reaches a `cause`, a `fix` or a
+// `meta`. A refusal is read by the caller, the log store and the dead-letter row, and a credential
+// in any of the three is a leak wearing a diagnostic's clothes. The URL is redacted for the same
+// reason one step down — several senders in the wild put a token in the query string, so only the
+// origin and path travel.
+
+import { UltimateError } from '@ultimat3/core';
+
+/**
+ * Origin + path, never the query and never any userinfo. A webhook URL is an operator's fact and
+ * a `cause` must carry enough of it to act on, but `https://hooks.example/x?token=…` is a
+ * credential in a field that reaches the log store unredactable.
+ */
+export function webhookTarget(url: string): string {
+  try {
+    const parsed = new URL(url);
+    return `${parsed.origin}${parsed.pathname}`;
+  } catch {
+    // Never `${error}` and never the URL itself: an unparseable value is exactly the one whose
+    // shape is unknown, and this string is going into a durable row.
+    return '<unparseable url>';
+  }
+}
+
+/** The endpoint seam answered nothing. Terminal: a queued delivery cannot invent a destination. */
+export class WebhookEndpointUnknownError extends UltimateError {
+  constructor(input: { webhook: string; endpointId: string }) {
+    super({
+      code: 'X_WEBHOOK_ENDPOINT_UNKNOWN',
+      cause: `webhook "${input.webhook}" has no endpoint ${input.endpointId}`,
+      fix: `return the endpoint row for ${input.endpointId} from webhook("${input.webhook}").endpoint, or stop enqueueing deliveries for endpoints your app has deleted`,
+      meta: { endpointId: input.endpointId },
+    });
+  }
+}
+
+/** The endpoint exists and cannot be delivered to as declared. */
+export class WebhookEndpointInvalidError extends UltimateError {
+  constructor(input: { webhook: string; endpointId: string; reason: string }) {
+    super({
+      code: 'X_WEBHOOK_ENDPOINT_INVALID',
+      cause: `webhook "${input.webhook}" endpoint ${input.endpointId} ${input.reason}`,
+      fix: `give the endpoint an https:// url and a non-empty secret before webhook("${input.webhook}").endpoint returns it — a delivery with neither is an unsigned POST to wherever the row points`,
+      meta: { endpointId: input.endpointId },
+    });
+  }
+}
+
+/**
+ * The endpoint takes nothing, either because the app disabled it or because this delivery is the
+ * one that reached `disableAfter`. Terminal by construction: the next attempt reads the same row.
+ */
+export class WebhookEndpointDisabledError extends UltimateError {
+  constructor(input: {
+    webhook: string;
+    endpointId: string;
+    consecutiveFailures?: number;
+    disableAfter?: number;
+  }) {
+    const why =
+      input.consecutiveFailures === undefined
+        ? 'is disabled'
+        : `failed ${input.consecutiveFailures} deliveries in a row and disableAfter is ${String(input.disableAfter)}`;
+    super({
+      code: 'X_WEBHOOK_ENDPOINT_DISABLED',
+      cause: `webhook "${input.webhook}" endpoint ${input.endpointId} ${why}`,
+      fix: `fix the receiver, then re-enable the endpoint in your own table and enqueue the deliveries it missed — the framework never re-enables one on its own, because a dead endpoint that heals itself is a retry loop with no end`,
+      meta: { endpointId: input.endpointId },
+    });
+  }
+}
+
+/** The event seam answered nothing for a queued id. */
+export class WebhookEventUnknownError extends UltimateError {
+  constructor(input: { webhook: string; eventId: string }) {
+    super({
+      code: 'X_WEBHOOK_EVENT_UNKNOWN',
+      cause: `webhook "${input.webhook}" has no event ${input.eventId}`,
+      fix: `write the event row before enqueueing its delivery — inside the same transaction, so webhook("${input.webhook}").event can never be asked for a row that is not committed yet`,
+      meta: { eventId: input.eventId },
+    });
+  }
+}
+
+/**
+ * The event's id or topic cannot be put in the canonical string without making it ambiguous. This
+ * is refused BEFORE the mac is taken, because a mac over an ambiguous string is a valid signature
+ * for a delivery the sender did not write.
+ */
+export class WebhookEventInvalidError extends UltimateError {
+  constructor(input: { webhook: string; eventId: string; field: string; max: number }) {
+    super({
+      code: 'X_WEBHOOK_EVENT_INVALID',
+      cause: `webhook "${input.webhook}" event ${input.eventId} has a ${input.field} that cannot be signed: 1-${input.max} characters, no ":" and no control characters`,
+      fix: `change how your app mints a ${input.field} — one mac over "v1:<t>:<id>:<topic>:<body>" would authenticate two different id/topic splits, so the separator can never appear in either`,
+      meta: { eventId: input.eventId, field: input.field },
+    });
+  }
+}
+
+/**
+ * The delivery did not land and the same request, unchanged, might: a 5xx, a throttle that named
+ * no delay, or a connection that never opened. Retryable — the job's own policy spends the
+ * backoff, which is core's one curve.
+ */
+export class WebhookDeliveryFailedError extends UltimateError {
+  constructor(input: {
+    webhook: string;
+    endpointId: string;
+    url: string;
+    status: number | null;
+    detail: string;
+  }) {
+    const answer = input.status === null ? 'never answered' : `answered ${input.status}`;
+    super({
+      code: 'X_WEBHOOK_DELIVERY_FAILED',
+      cause: `webhook "${input.webhook}" delivery to ${webhookTarget(input.url)} ${answer}: ${input.detail}`,
+      fix: 'no edit here — the job retries on its declared backoff; watch the endpoint with x jobs ls --json and fix the receiver before it reaches disableAfter',
+      meta: { endpointId: input.endpointId, status: input.status },
+    });
+  }
+}
+
+/**
+ * The receiver said "later" AND said when. Its own code because the answer is different work: the
+ * nack waits the delay the receiver named (`statedDelayMs` reads `meta.retryAfterSeconds`, clamped
+ * by the policy's `maxDelay`) instead of spending a curve against a number it was already given.
+ */
+export class WebhookDeliveryThrottledError extends UltimateError {
+  constructor(input: {
+    webhook: string;
+    endpointId: string;
+    url: string;
+    status: number;
+    retryAfterSeconds: number;
+  }) {
+    super({
+      code: 'X_WEBHOOK_DELIVERY_THROTTLED',
+      cause: `webhook "${input.webhook}" delivery to ${webhookTarget(input.url)} answered ${input.status} and asked for ${input.retryAfterSeconds}s`,
+      fix: 'no edit here — the next attempt waits the delay the receiver named; if this endpoint throttles constantly, raise its retry.maxDelay or fan out fewer events to it',
+      meta: {
+        endpointId: input.endpointId,
+        status: input.status,
+        // The one spelling `statedDelayMs` reads. Never a second key.
+        retryAfterSeconds: input.retryAfterSeconds,
+      },
+    });
+  }
+}
+
+/**
+ * The receiver refused in a way a retry cannot change — a 4xx that is not a throttle, or a
+ * redirect. Its own code rather than an instance flag on the one above: `classifyThrown` reads a
+ * per-instance `terminal` on an unregistered code as UNCLASSIFIED, so the distinction has to be a
+ * registered code or it does not exist.
+ */
+export class WebhookDeliveryRejectedError extends UltimateError {
+  constructor(input: { webhook: string; endpointId: string; url: string; status: number }) {
+    super({
+      code: 'X_WEBHOOK_DELIVERY_REJECTED',
+      cause: `webhook "${input.webhook}" delivery to ${webhookTarget(input.url)} answered ${input.status}, which no retry changes`,
+      fix: 'x jobs show <jobId> --json   # meta.status is the answer; then correct the url or the secret on this endpoint row in your own table — a 401 or 403 is a secret the two sides disagree about, a 404 is a path that moved',
+      meta: { endpointId: input.endpointId, status: input.status },
+    });
+  }
+}

package/src/webhook-ledger.ts

@@ -0,0 +1,102 @@
+// The delivery ledger: what every outbound attempt is recorded in, and the one fact the mechanism
+// reads back out of it — how many times in a row this endpoint has failed.
+//
+// A SEAM and not a table, exactly like `PurgeTarget`: an app already has a place for this and the
+// retention on it is seven years for one business and thirty days for the next, so shipping a
+// schema would be shipping one of those answers (axiom 8). What the framework owns is that every
+// attempt IS recorded and that the count is what disables an endpoint.
+
+import { assert } from '@ultimat3/core';
+
+/** One outbound attempt, as it happened. Bounded and JSON-safe, so a row can hold it verbatim. */
+export interface WebhookAttempt {
+  /** The `webhook()` declaration's name — the durable queue key deliveries are routed under. */
+  readonly webhook: string;
+  readonly endpointId: string;
+  readonly eventId: string;
+  readonly topic: string;
+  /** The JOB's attempt, 1-based: attempt 3 is the third time the queue handed this delivery out. */
+  readonly attempt: number;
+  readonly ok: boolean;
+  /** The response status, or `null` when there was no response at all. */
+  readonly status: number | null;
+  /** When the attempt finished, in ms on the declaration's clock. */
+  readonly at: number;
+  readonly durationMs: number;
+  /**
+   * Why it failed, rendered by the framework and never by the receiver: the response BODY is not
+   * here, because an endpoint that echoes the request has just written the payload into a second
+   * table with a different retention. Absent on a success.
+   */
+  readonly error?: string;
+}
+
+export interface WebhookLedger {
+  /**
+   * Append this attempt and answer how many CONSECUTIVE failures the endpoint now has — 0 when
+   * `attempt.ok`. One method rather than an append plus a separate count, so the number the
+   * mechanism disables on cannot be read from before the row it is deciding about.
+   */
+  record(attempt: WebhookAttempt): Promise<number>;
+  /**
+   * Stop delivering to this endpoint. Called once, on the attempt whose count reached
+   * `disableAfter`. Re-enabling is the app's — a dead endpoint that heals itself is a retry loop
+   * with no end.
+   */
+  disable(endpointId: string, reason: string): Promise<void>;
+}
+
+/** Attempts one process keeps before the oldest go. A dev ledger that grows forever is a leak. */
+export const DEFAULT_MAX_WEBHOOK_ATTEMPTS = 1_000;
+
+export interface MemoryWebhookLedger extends WebhookLedger {
+  /** Newest last, bounded by `maxAttempts`. */
+  attempts(): readonly WebhookAttempt[];
+  /** Endpoint id -> the reason it was disabled. */
+  disabled(): ReadonlyMap<string, string>;
+  reset(): void;
+}
+
+/**
+ * The dev and test ledger, and honest about being one: a bounded ring in one heap, so nothing here
+ * survives a restart and nothing here is shared between replicas. The consecutive count is derived
+ * from a counter rather than by scanning the ring — a ring that evicted the failures would answer
+ * a smaller number the longer an outage ran, which is the direction that never disables anything.
+ */
+export function memoryWebhookLedger(
+  options: { readonly maxAttempts?: number } = {},
+): MemoryWebhookLedger {
+  const maxAttempts = options.maxAttempts ?? DEFAULT_MAX_WEBHOOK_ATTEMPTS;
+  assert(
+    Number.isSafeInteger(maxAttempts) && maxAttempts >= 1,
+    `memoryWebhookLedger maxAttempts is ${String(maxAttempts)}, which keeps no attempt at all`,
+    `pass memoryWebhookLedger({ maxAttempts: ${DEFAULT_MAX_WEBHOOK_ATTEMPTS} }) or leave it out — a ledger that holds nothing cannot disable an endpoint`,
+  );
+  const attempts: WebhookAttempt[] = [];
+  const consecutive = new Map<string, number>();
+  const off = new Map<string, string>();
+
+  return {
+    record(attempt: WebhookAttempt): Promise<number> {
+      attempts.push(attempt);
+      if (attempts.length > maxAttempts) attempts.splice(0, attempts.length - maxAttempts);
+      const failures = attempt.ok ? 0 : (consecutive.get(attempt.endpointId) ?? 0) + 1;
+      // Deleted at zero rather than written as one, the bound `limits.ts` keeps for the same
+      // reason: a healthy endpoint must not cost a permanent map entry per org that creates one.
+      if (failures === 0) consecutive.delete(attempt.endpointId);
+      else consecutive.set(attempt.endpointId, failures);
+      return Promise.resolve(failures);
+    },
+    disable(endpointId: string, reason: string): Promise<void> {
+      off.set(endpointId, reason);
+      return Promise.resolve();
+    },
+    attempts: (): readonly WebhookAttempt[] => [...attempts],
+    disabled: (): ReadonlyMap<string, string> => new Map(off),
+    reset(): void {
+      attempts.length = 0;
+      consecutive.clear();
+      off.clear();
+    },
+  };
+}

package/src/webhook.ts

@@ -0,0 +1,378 @@
+// `webhook()` — one outbound delivery to one endpoint, declared as a `job` and NOT as a ninth
+// primitive. A delivery is durable background work with an input schema, a retry policy, an
+// idempotency key and a queue, which is the definition of a `job` — so this file is a FACTORY over
+// `job()`, exactly as `backfill()` and `purge()` are and as `llm()` is over `action()`. That gives
+// it `.enqueue()`, the retry backoff, the worker's cancellation, the dead-letter path,
+// `x jobs show` and a manifest row without a line here.
+//
+// ONE ENDPOINT PER JOB, deliberately. Retry, backoff and disable-after-N are all per-endpoint
+// facts, and a job that fanned out inside one body would retry every endpoint because one of them
+// was down — the same defect `docs/architecture/15-adding-a-feature.md` names for a mail loop
+// under a single step. WHICH endpoints exist is the app's (axiom 8), so the fan-out is the app's
+// `for` loop over its own subscription table, one `enqueue` per endpoint.
+//
+// WHAT THIS MECHANISM OWNS: signing, a timestamped signature, retry with core's backoff,
+// disable-after-N-consecutive-failures, and a ledger row per attempt. WHAT IT NEVER OWNS: the
+// event taxonomy, which endpoints exist, and what a payload means.
+
+import type { Clock, Ctx } from '@ultimat3/core';
+import {
+  isCanonicalWebhookField,
+  renderThrowable,
+  systemClock,
+  WEBHOOK_FIELD_MAX,
+  webhookHeaders,
+} from '@ultimat3/core';
+import { t } from '@ultimat3/schema';
+import type { DurationInput } from './clock';
+import { nowMs } from './clock';
+import type { JobHandle } from './job';
+import { job } from './job';
+import type { RetryPolicy } from './retry';
+import { DEFAULT_RETRY } from './retry';
+import type { JobTenant } from './tenant';
+import {
+  WebhookDeliveryFailedError,
+  WebhookDeliveryRejectedError,
+  WebhookDeliveryThrottledError,
+  WebhookEndpointDisabledError,
+  WebhookEndpointInvalidError,
+  WebhookEndpointUnknownError,
+  WebhookEventInvalidError,
+  WebhookEventUnknownError,
+} from './webhook-errors';
+import type { WebhookLedger } from './webhook-ledger';
+
+/** Just the call. `typeof fetch` also carries `preconnect`, which no test double should have to. */
+export type WebhookFetch = (url: string, init: RequestInit) => Promise<Response>;
+
+/**
+ * Consecutive failures before an endpoint stops taking deliveries. Ten is roughly a day of a
+ * retrying queue against a dead receiver, which is long enough for an outage and short enough that
+ * a decommissioned endpoint does not cost the fleet forever.
+ */
+export const DEFAULT_WEBHOOK_DISABLE_AFTER = 10;
+
+/** The one content type a delivery announces. The BODY is the app's; how it is framed is not. */
+export const WEBHOOK_CONTENT_TYPE = 'application/json';
+
+/** `Retry-After` in seconds. The HTTP-date form is ignored on purpose — see `retryAfterSeconds`. */
+const RETRY_AFTER_SECONDS = /^\d{1,7}$/;
+
+export interface WebhookEndpoint {
+  readonly id: string;
+  /** `https://…` in production, `http://…` for a dev receiver. Nothing else is opened. */
+  readonly url: string;
+  /** The shared secret. Never logged, never in a `cause`, never on the ledger row. */
+  readonly secret: string;
+  /** True stops every delivery before the socket opens. Set by the app, or by `disableAfter`. */
+  readonly disabled?: boolean;
+  /**
+   * Extra headers this receiver asked for. Merged UNDER the framework's, so nothing here can
+   * overwrite the signature, the id or the topic — an endpoint row that could set its own
+   * `x-ultimate-webhook-signature` is an endpoint row that can forge one.
+   */
+  readonly headers?: Readonly<Record<string, string>>;
+}
+
+export interface WebhookEvent {
+  /**
+   * The sender's routing label, signed and carried and never interpreted. Which topics exist is
+   * the app's taxonomy (axiom 8); that a delivery HAS one is the mechanism.
+   */
+  readonly topic: string;
+  /**
+   * The exact text to sign and send. Serialised by the app, because what a payload means is the
+   * app's — and byte-stable within one attempt, which is all the signature needs.
+   */
+  readonly body: string;
+}
+
+/**
+ * The queue row's payload: a POINTER, never a record. The event's bytes live in the app's own
+ * table and are read once per attempt, so nothing durable here holds a payload and nothing here
+ * holds a secret.
+ */
+export interface WebhookDeliveryInput {
+  readonly endpointId: string;
+  /** Also the id the receiver dedupes on — it is signed, so it cannot be moved in transit. */
+  readonly eventId: string;
+}
+
+/** What one landed delivery reports. Bounded, so `x jobs show` can print it. */
+export interface WebhookReport {
+  readonly endpointId: string;
+  readonly eventId: string;
+  readonly status: number;
+  readonly durationMs: number;
+}
+
+export interface WebhookDefinition {
+  /**
+   * REQUIRED, unlike a job's. A delivery's name is a durable queue key — every queued, retrying
+   * and dead-lettered row carries it — so it is never left to whichever export name a module used.
+   */
+  readonly name: string;
+  /**
+   * REQUIRED, exactly as on `job()`: `tenant: ({ endpointId }) => …` for a delivery scoped to the
+   * org that owns the endpoint, or the explicit `tenant: 'none'`. A delivery reads the app's own
+   * endpoint and event rows through the seams below, so the org those reads run under is a fact
+   * about the WORK and is declared here rather than inherited from whichever worker claimed it.
+   */
+  readonly tenant: JobTenant<WebhookDeliveryInput>;
+  /**
+   * The endpoint this delivery is for. Read once PER ATTEMPT and never checkpointed: it carries a
+   * secret, and a `step.run` output is written to `x_job_steps` — a credential in a durable table
+   * the queue keeps for the life of the run.
+   */
+  endpoint(args: {
+    readonly endpointId: string;
+    readonly ctx: Ctx;
+  }): Promise<WebhookEndpoint | null> | WebhookEndpoint | null;
+  /** The event's bytes. Read per attempt as well, out of the app's own table. */
+  event(args: {
+    readonly eventId: string;
+    readonly ctx: Ctx;
+  }): Promise<WebhookEvent | null> | WebhookEvent | null;
+  /** Where every attempt is recorded, and where the consecutive-failure count comes from. */
+  readonly ledger: WebhookLedger;
+  /**
+   * Consecutive failures before the endpoint is disabled. Defaults to
+   * `DEFAULT_WEBHOOK_DISABLE_AFTER`. Re-enabling is always the app's — an endpoint the framework
+   * un-disabled on its own is a retry loop with no end.
+   */
+  readonly disableAfter?: number;
+  /** The clock the signature's timestamp and the ledger's instants are read from. */
+  readonly clock?: Clock;
+  /** Injected so a test can drive the transport. The network is sealed in this repo's suites. */
+  readonly fetch?: WebhookFetch;
+  readonly queue?: string;
+  readonly retry?: RetryPolicy;
+  /**
+   * The ceiling for ONE attempt, and therefore for the request inside it. Deliberately not a
+   * second per-request timeout: `ctx.signal` already carries this deadline into `fetch`, and two
+   * numbers for one wait is the ambiguity axiom 1 refuses.
+   */
+  readonly timeout?: DurationInput;
+}
+
+/** `Retry-After: 120`. The HTTP-date form needs the receiver's clock, which is what we do not trust. */
+const retryAfterSeconds = (response: Response): number | undefined => {
+  const header = response.headers.get('retry-after');
+  if (header === null || !RETRY_AFTER_SECONDS.test(header.trim())) return undefined;
+  return Number(header.trim());
+};
+
+/** A status the same request, unchanged, can still land on. Everything else is somebody's edit. */
+const isRetryableStatus = (status: number): boolean =>
+  status >= 500 || status === 408 || status === 425 || status === 429;
+
+export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliveryInput> {
+  const clock = definition.clock ?? systemClock;
+  const disableAfter = definition.disableAfter ?? DEFAULT_WEBHOOK_DISABLE_AFTER;
+  const send = definition.fetch ?? ((url, init) => fetch(url, init));
+
+  return job<WebhookDeliveryInput>({
+    name: definition.name,
+    input: t.object({ endpointId: t.string, eventId: t.string }),
+    // Endpoint AND event: the same event fans out to every subscribed endpoint, so a key on the
+    // event alone would dedupe every one of those deliveries into the first endpoint's row.
+    idempotencyKey: ({ endpointId, eventId }) => `${definition.name}:${endpointId}:${eventId}`,
+    tenant: definition.tenant,
+    retry: definition.retry ?? DEFAULT_RETRY,
+    ...(definition.queue === undefined ? {} : { queue: definition.queue }),
+    ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
+    async run({ input, ctx, attempt }): Promise<WebhookReport> {
+      const endpoint = await definition.endpoint({ endpointId: input.endpointId, ctx });
+      if (endpoint === null) {
+        throw new WebhookEndpointUnknownError({
+          webhook: definition.name,
+          endpointId: input.endpointId,
+        });
+      }
+      // Before the ledger and before the socket: a disabled endpoint costs nothing, which is the
+      // whole point of disabling one.
+      if (endpoint.disabled === true) {
+        throw new WebhookEndpointDisabledError({
+          webhook: definition.name,
+          endpointId: endpoint.id,
+        });
+      }
+      assertDeliverable(definition.name, endpoint);
+
+      const event = await definition.event({ eventId: input.eventId, ctx });
+      if (event === null) {
+        throw new WebhookEventUnknownError({ webhook: definition.name, eventId: input.eventId });
+      }
+      // Refused BEFORE the mac is taken. A mac over an ambiguous canonical string is a valid
+      // signature for a delivery the sender never wrote — see `isCanonicalWebhookField`.
+      for (const [field, value] of [
+        ['event id', input.eventId],
+        ['topic', event.topic],
+      ] as const) {
+        if (isCanonicalWebhookField(value)) continue;
+        throw new WebhookEventInvalidError({
+          webhook: definition.name,
+          eventId: input.eventId,
+          field,
+          max: WEBHOOK_FIELD_MAX,
+        });
+      }
+
+      // Seconds, at SEND time and not at event time: a delivery retried three days later is signed
+      // again now, so a receiver's freshness window measures the request in front of it rather
+      // than the age of the fact behind it.
+      const timestampSeconds = Math.floor(nowMs(clock) / 1_000);
+      const startedAt = clock.monotonic();
+      const outcome = await attemptDelivery(send, endpoint, {
+        secret: endpoint.secret,
+        timestampSeconds,
+        eventId: input.eventId,
+        topic: event.topic,
+        body: event.body,
+      });
+      const durationMs = Math.max(0, clock.monotonic() - startedAt);
+
+      // Recorded whatever happened, and BEFORE the throw: a failure that is not on the ledger is a
+      // failure the consecutive count cannot see, which is an endpoint that never gets disabled.
+      const failures = await definition.ledger.record({
+        webhook: definition.name,
+        endpointId: endpoint.id,
+        eventId: input.eventId,
+        topic: event.topic,
+        attempt,
+        ok: outcome.ok,
+        status: outcome.status,
+        at: nowMs(clock),
+        durationMs,
+        ...(outcome.ok ? {} : { error: outcome.detail }),
+      });
+
+      if (outcome.ok) {
+        return {
+          endpointId: endpoint.id,
+          eventId: input.eventId,
+          status: outcome.status,
+          durationMs,
+        };
+      }
+
+      if (failures >= disableAfter) {
+        const reason = `${failures} consecutive failed deliveries, disableAfter is ${disableAfter}`;
+        await definition.ledger.disable(endpoint.id, reason);
+        // The endpoint's own code and not this attempt's: what an operator needs to see is that
+        // deliveries have STOPPED, not the status of the one that tipped it over.
+        throw new WebhookEndpointDisabledError({
+          webhook: definition.name,
+          endpointId: endpoint.id,
+          consecutiveFailures: failures,
+          disableAfter,
+        });
+      }
+      throw deliveryError(definition.name, endpoint, outcome);
+    },
+  });
+}
+
+/** A URL no delivery may open, or a secret that would make the POST unsigned. */
+function assertDeliverable(name: string, endpoint: WebhookEndpoint): void {
+  if (endpoint.secret.length === 0) {
+    throw new WebhookEndpointInvalidError({
+      webhook: name,
+      endpointId: endpoint.id,
+      reason: 'has an empty secret, so the delivery would carry no proof of who sent it',
+    });
+  }
+  let protocol: string;
+  try {
+    protocol = new URL(endpoint.url).protocol;
+  } catch {
+    // Never the caught value and never the url: an unparseable value is exactly the one whose
+    // shape is unknown, and this reason reaches a durable dead-letter row.
+    throw new WebhookEndpointInvalidError({
+      webhook: name,
+      endpointId: endpoint.id,
+      reason: 'has a url that is not a url',
+    });
+  }
+  // `file:`, `data:` and the rest are refused by NAME rather than by a blocklist: a delivery opens
+  // an HTTP conversation, and anything else is a row in a table reaching the worker's filesystem.
+  if (protocol !== 'https:' && protocol !== 'http:') {
+    throw new WebhookEndpointInvalidError({
+      webhook: name,
+      endpointId: endpoint.id,
+      reason: `has a ${protocol} url, and a delivery only ever opens http: or https:`,
+    });
+  }
+}
+
+type Outcome =
+  | { readonly ok: true; readonly status: number }
+  | {
+      readonly ok: false;
+      readonly status: number | null;
+      readonly detail: string;
+      readonly retryAfterSeconds?: number;
+    };
+
+async function attemptDelivery(
+  send: WebhookFetch,
+  endpoint: WebhookEndpoint,
+  signing: Parameters<typeof webhookHeaders>[0],
+): Promise<Outcome> {
+  let response: Response;
+  try {
+    response = await send(endpoint.url, {
+      method: 'POST',
+      headers: {
+        ...endpoint.headers,
+        'content-type': WEBHOOK_CONTENT_TYPE,
+        // LAST, so an endpoint row's own headers can never overwrite the signature it is proved by.
+        ...webhookHeaders(signing),
+      },
+      body: signing.body,
+      // Never followed: a 3xx would re-POST a body signed for one host to whatever the receiver
+      // named, and the signature would travel with it.
+      redirect: 'manual',
+    });
+  } catch (error) {
+    // `renderThrowable`, never `String(error)` or `${error}`: this string lands in a `cause` and on
+    // a durable ledger row, and a null-prototype throwable makes both of those a `TypeError`.
+    return { ok: false, status: null, detail: renderThrowable(error) };
+  }
+  if (response.ok) return { ok: true, status: response.status };
+  const stated = retryAfterSeconds(response);
+  return {
+    ok: false,
+    status: response.status,
+    detail: `status ${response.status}`,
+    ...(stated === undefined ? {} : { retryAfterSeconds: stated }),
+  };
+}
+
+/** Which of the three failure codes this outcome is. The split is "can the same request land?". */
+function deliveryError(
+  name: string,
+  endpoint: WebhookEndpoint,
+  outcome: Extract<Outcome, { ok: false }>,
+): Error {
+  const base = { webhook: name, endpointId: endpoint.id, url: endpoint.url };
+  if (outcome.status === null) {
+    return new WebhookDeliveryFailedError({ ...base, status: null, detail: outcome.detail });
+  }
+  if (!isRetryableStatus(outcome.status)) {
+    return new WebhookDeliveryRejectedError({ ...base, status: outcome.status });
+  }
+  if (outcome.retryAfterSeconds !== undefined) {
+    return new WebhookDeliveryThrottledError({
+      ...base,
+      status: outcome.status,
+      retryAfterSeconds: outcome.retryAfterSeconds,
+    });
+  }
+  return new WebhookDeliveryFailedError({
+    ...base,
+    status: outcome.status,
+    detail: outcome.detail,
+  });
+}