@tokenoftrust/storefront-runner 1.3.4-rc.3 → 1.4.0

package/apps/storefront/src/lib/webhooks/deliveryStore.ts

@@ -0,0 +1,668 @@
+/**
+ * Webhook event/delivery storage — PrivateApps epic, D4 (Chunk A: Storage
+ * only). Owns the `webhook_events` / `webhook_deliveries` /
+ * `webhook_delivery_attempts` / `webhook_install_health` rows described in
+ * `../d1/schema.ts`. Signing (Chunk B), the retry/backoff dispatcher
+ * (Chunk C), event emission/targeting (Chunk D), and the `/webhooks/*`
+ * routes (Chunk E) all build on this store; none of that policy lives here.
+ *
+ * Pure over the `Queryable` seam (see `../d1/catalog.ts`) — the same shape
+ * `AppRegistryService` uses, so this runs unchanged against real D1 in the
+ * Worker and an in-memory `node:sqlite` in tests.
+ *
+ * Two ids matter and are easy to conflate:
+ *   - `eventId` (`webhook_events.id`) — the semantic event; ONE per business
+ *     occurrence (e.g. one order.created). Never sent on the wire directly.
+ *   - `deliveryId` (`webhook_deliveries.delivery_id`) — the CloudEvents
+ *     envelope `id` for one (event × target install) delivery. This is what
+ *     goes on the wire and is what receivers dedupe on; it stays stable
+ *     across that delivery's own retries AND across a manual replay (Chunk
+ *     E's `requeueForReplay` resets state/attempts on the SAME row rather
+ *     than minting a new one — a receiver that already acked a delivered
+ *     delivery must still dedupe a replayed resend of it).
+ *
+ * Responsibility split for the two mutating paths a dispatcher (Chunk C)
+ * drives per attempt:
+ *   - `recordAttempt` is dumb bookkeeping of ONE HTTP attempt's outcome: it
+ *     appends to `webhook_delivery_attempts`, bumps the delivery's
+ *     `attempts`/`last_*` columns, and folds the outcome into the install's
+ *     health rollup (`lastSuccessAt`/`failureStreak`/`last_*`/recent trace
+ *     ids). It never changes `state`.
+ *   - `transitionState` is the retry-POLICY decision (how many attempts,
+ *     what backoff, when to give up) — deliberately left to the caller, not
+ *     computed here. It only moves `state` (and `next_attempt_at`), and
+ *     bumps `dlqCount` on the one transition that means "gave up":
+ *     dead-lettering.
+ *
+ * The `Queryable` seam has no multi-statement transaction primitive, so
+ * "transactionally-in-spirit" here means: do the writes in a fixed order
+ * (attempt row, then delivery row, then health rollup) and always re-read
+ * the delivery via `getOrThrowDelivery` before returning it, the same
+ * read-after-write discipline `AppRegistryService` uses.
+ */
+import type { Queryable } from "../d1/catalog.js";
+
+/** Mirrors the `Delivery.topic` / CloudEvents channel enum in the D1/D2 contract. */
+export type WebhookTopic =
+  | "app.installed"
+  | "app.uninstalled"
+  | "catalog.product.updated"
+  | "inventory.changed"
+  | "order.created"
+  | "order.fulfilled"
+  | "customer.marketing_consent.updated"
+  | "attribution.finalized";
+
+/** Mirrors `Delivery.state` in `docs/private-apps/contract/openapi.yaml` EXACTLY. */
+export type DeliveryState = "pending" | "retrying" | "delivered" | "dead-lettered";
+
+export interface WebhookEvent {
+  id: string;
+  tenantId: string;
+  topic: WebhookTopic;
+  dataschemaVersion: string;
+  /** The CloudEvents `data` payload — event-specific shape, opaque to the store. */
+  payload: unknown;
+  createdAt: string;
+}
+
+export interface WebhookDelivery {
+  deliveryId: string;
+  eventId: string;
+  installId: string;
+  tenantId: string;
+  topic: WebhookTopic;
+  state: DeliveryState;
+  attempts: number;
+  lastStatus: number | null;
+  lastErrorClass: string | null;
+  lastTraceId: string | null;
+  nextAttemptAt: string | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+/** The openapi `InstallHealth` shape — no field is required there, so a fresh install reads as healthy/zeroed. */
+export interface InstallHealth {
+  status: "healthy" | "degraded" | "unhealthy";
+  lastSuccessAt?: string;
+  failureStreak: number;
+  dlqCount: number;
+  /** p95 delivery latency (ms) across the install's recent attempts — see {@link WebhookDeliveryStore.getInstallHealth}. Absent when no attempt has a recorded duration. */
+  p95DeliveryMs?: number;
+  lastStatus?: number;
+  lastErrorClass?: string;
+  recentTraceIds: string[];
+}
+
+export interface CreateEventInput {
+  /** Optional caller-supplied event id (e.g. for a caller-owned idempotency key). Defaults to a fresh UUID. */
+  eventId?: string;
+  tenantId: string;
+  topic: WebhookTopic;
+  dataschemaVersion: string;
+  payload: unknown;
+}
+
+export interface DeliveryTarget {
+  installId: string;
+  /** Optional caller-supplied delivery id. Defaults to a fresh UUID (the wire CloudEvents `id`). */
+  deliveryId?: string;
+}
+
+export interface CreateEventResult {
+  event: WebhookEvent;
+  deliveries: WebhookDelivery[];
+}
+
+export interface ListDeliveriesOptions {
+  state?: DeliveryState;
+  /** Cap the result, newest first. Used internally by {@link WebhookDeliveryStore.getInstallHealth} to bound its p95 sample; unset means unbounded (the historical behavior). */
+  limit?: number;
+}
+
+export interface RecordAttemptInput {
+  deliveryId: string;
+  attemptNo: number;
+  startedAt: string;
+  durationMs?: number | null;
+  httpStatus?: number | null;
+  errorClass?: string | null;
+  traceId?: string | null;
+}
+
+export interface TransitionStateInput {
+  deliveryId: string;
+  toState: DeliveryState;
+  /** Required (and only meaningful) when `toState` is `"retrying"`. */
+  nextAttemptAt?: string | null;
+}
+
+export interface ScanDueRetriesInput {
+  /** Deliveries with `next_attempt_at <= now` are due. */
+  now: string;
+  limit?: number;
+}
+
+export type WebhookStoreErrorCode = "not_found" | "invalid_transition";
+
+/** Thrown by every rule violation below — callers branch on `.code`. */
+export class WebhookStoreError extends Error {
+  constructor(
+    message: string,
+    readonly code: WebhookStoreErrorCode,
+  ) {
+    super(message);
+    this.name = "WebhookStoreError";
+  }
+}
+
+const MAX_RECENT_TRACE_IDS = 10;
+
+// How many of an install's most recent deliveries feed the p95 sample in
+// `getInstallHealth` — a bound, not a precise percentile guarantee; recent
+// behavior is what the health view cares about, not all-time history.
+const P95_SAMPLE_SIZE = 50;
+
+// A delivery only ever moves forward via `transitionState`; `delivered`/
+// `dead-lettered` are terminal THERE. A manual replay (Chunk E's
+// `requeueForReplay`) is the one deliberate exception: it resets a delivery
+// out of a terminal state back to `pending` on the SAME row, bypassing this
+// table entirely — see that method's own doc comment for why.
+const ALLOWED_TRANSITIONS: Record<DeliveryState, readonly DeliveryState[]> = {
+  pending: ["retrying", "delivered", "dead-lettered"],
+  retrying: ["retrying", "delivered", "dead-lettered"],
+  delivered: [],
+  "dead-lettered": [],
+};
+
+// Failure-streak thresholds for the `InstallHealth.status` rollup. A DLQ hit
+// always reads as unhealthy, regardless of streak.
+const DEGRADED_FAILURE_STREAK = 1;
+const UNHEALTHY_FAILURE_STREAK = 5;
+
+function isSuccessAttempt(httpStatus: number | null | undefined): boolean {
+  return httpStatus != null && httpStatus >= 200 && httpStatus < 300;
+}
+
+/** Prepend a trace id (if present) and cap the list, newest first. */
+function pushTraceId(existing: string[], traceId: string | null | undefined): string[] {
+  if (!traceId) return existing;
+  return [traceId, ...existing].slice(0, MAX_RECENT_TRACE_IDS);
+}
+
+/** Raw `webhook_events` row shape (snake_case columns). */
+interface EventRow {
+  id: string;
+  tenant_id: string;
+  topic: string;
+  dataschema_version: string;
+  payload: string;
+  created_at: string;
+}
+
+/** Raw `webhook_deliveries` row shape (snake_case columns). */
+interface DeliveryRow {
+  delivery_id: string;
+  event_id: string;
+  install_id: string;
+  tenant_id: string;
+  topic: string;
+  state: string;
+  attempts: number;
+  last_status: number | null;
+  last_error_class: string | null;
+  last_trace_id: string | null;
+  next_attempt_at: string | null;
+  created_at: string;
+  updated_at: string;
+}
+
+/** Raw `webhook_install_health` row shape (snake_case columns). */
+interface HealthRow {
+  install_id: string;
+  last_success_at: string | null;
+  failure_streak: number;
+  dlq_count: number;
+  last_status: number | null;
+  last_error_class: string | null;
+  recent_trace_ids: string;
+  updated_at: string;
+}
+
+function toDomain(row: DeliveryRow): WebhookDelivery {
+  return {
+    deliveryId: row.delivery_id,
+    eventId: row.event_id,
+    installId: row.install_id,
+    tenantId: row.tenant_id,
+    topic: row.topic as WebhookTopic,
+    state: row.state as DeliveryState,
+    attempts: row.attempts,
+    lastStatus: row.last_status,
+    lastErrorClass: row.last_error_class,
+    lastTraceId: row.last_trace_id,
+    nextAttemptAt: row.next_attempt_at,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+  };
+}
+
+function toEventDomain(row: EventRow): WebhookEvent {
+  return {
+    id: row.id,
+    tenantId: row.tenant_id,
+    topic: row.topic as WebhookTopic,
+    dataschemaVersion: row.dataschema_version,
+    payload: JSON.parse(row.payload) as unknown,
+    createdAt: row.created_at,
+  };
+}
+
+function healthStatus(row: HealthRow | null): InstallHealth["status"] {
+  if (!row) return "healthy";
+  if (row.dlq_count > 0 || row.failure_streak >= UNHEALTHY_FAILURE_STREAK) return "unhealthy";
+  if (row.failure_streak >= DEGRADED_FAILURE_STREAK) return "degraded";
+  return "healthy";
+}
+
+function toInstallHealth(row: HealthRow | null): InstallHealth {
+  return {
+    status: healthStatus(row),
+    lastSuccessAt: row?.last_success_at ?? undefined,
+    failureStreak: row?.failure_streak ?? 0,
+    dlqCount: row?.dlq_count ?? 0,
+    lastStatus: row?.last_status ?? undefined,
+    lastErrorClass: row?.last_error_class ?? undefined,
+    recentTraceIds: row ? (JSON.parse(row.recent_trace_ids) as string[]) : [],
+  };
+}
+
+export class WebhookDeliveryStore {
+  constructor(
+    private readonly db: Queryable,
+    private readonly now: () => string = () => new Date().toISOString(),
+  ) {}
+
+  /**
+   * Write one immutable event, then fan it out to one `pending` delivery per
+   * target install. Targeting (which installs get the event) is the caller's
+   * job (Chunk D) — this just persists whatever target list it's handed.
+   */
+  async createEvent(input: CreateEventInput, targets: readonly DeliveryTarget[]): Promise<CreateEventResult> {
+    const eventId = input.eventId ?? crypto.randomUUID();
+    const now = this.now();
+
+    await this.db.run(
+      `INSERT INTO webhook_events (id, tenant_id, topic, dataschema_version, payload, created_at)
+       VALUES (?,?,?,?,?,?)`,
+      eventId,
+      input.tenantId,
+      input.topic,
+      input.dataschemaVersion,
+      JSON.stringify(input.payload),
+      now,
+    );
+
+    const deliveries: WebhookDelivery[] = [];
+    for (const target of targets) {
+      const deliveryId = target.deliveryId ?? crypto.randomUUID();
+      await this.db.run(
+        `INSERT INTO webhook_deliveries
+           (delivery_id, event_id, install_id, tenant_id, topic, state, attempts, created_at, updated_at)
+         VALUES (?,?,?,?,?,'pending',0,?,?)`,
+        deliveryId,
+        eventId,
+        target.installId,
+        input.tenantId,
+        input.topic,
+        now,
+        now,
+      );
+      deliveries.push(await this.getOrThrowDelivery(deliveryId));
+    }
+
+    return {
+      event: {
+        id: eventId,
+        tenantId: input.tenantId,
+        topic: input.topic,
+        dataschemaVersion: input.dataschemaVersion,
+        payload: input.payload,
+        createdAt: now,
+      },
+      deliveries,
+    };
+  }
+
+  /** Read one delivery by id, or `null` if it doesn't exist. */
+  async getDelivery(deliveryId: string): Promise<WebhookDelivery | null> {
+    const row = await this.db.first<DeliveryRow>(
+      `SELECT * FROM webhook_deliveries WHERE delivery_id = ?`,
+      deliveryId,
+    );
+    return row ? toDomain(row) : null;
+  }
+
+  /**
+   * Read one event by id, or `null` if it doesn't exist. Added for the
+   * dispatcher/delivery engine (Chunk C): `attemptDelivery` resolves a
+   * delivery's `eventId` via `getDelivery`, then needs this to fetch the
+   * `payload`/`dataschemaVersion` that only `webhook_events` (not
+   * `webhook_deliveries`) carries, in order to build the CloudEvents
+   * envelope's `data`.
+   */
+  async getEvent(eventId: string): Promise<WebhookEvent | null> {
+    const row = await this.db.first<EventRow>(`SELECT * FROM webhook_events WHERE id = ?`, eventId);
+    return row ? toEventDomain(row) : null;
+  }
+
+  /** All deliveries for one install (an install only ever sees its own — enforced by the caller), newest first. Optionally capped via `opts.limit`. */
+  async listDeliveriesByInstall(
+    installId: string,
+    opts: ListDeliveriesOptions = {},
+  ): Promise<WebhookDelivery[]> {
+    const params: unknown[] = [installId];
+    let sql = `SELECT * FROM webhook_deliveries WHERE install_id = ?`;
+    if (opts.state) {
+      sql += ` AND state = ?`;
+      params.push(opts.state);
+    }
+    sql += ` ORDER BY created_at DESC`;
+    if (opts.limit != null) {
+      sql += ` LIMIT ?`;
+      params.push(opts.limit);
+    }
+    const rows = await this.db.all<DeliveryRow>(sql, ...params);
+    return rows.map(toDomain);
+  }
+
+  /**
+   * All deliveries for a tenant, across every install (the ops/support admin
+   * view — an install only ever sees its OWN deliveries via
+   * {@link listDeliveriesByInstall}; this is the tenant-wide superset), newest
+   * first. Added for D4 Chunk E's admin route; Chunk A/B/C never call this.
+   */
+  async listDeliveriesByTenant(
+    tenantId: string,
+    opts: ListDeliveriesOptions = {},
+  ): Promise<WebhookDelivery[]> {
+    const rows = opts.state
+      ? await this.db.all<DeliveryRow>(
+          `SELECT * FROM webhook_deliveries WHERE tenant_id = ? AND state = ? ORDER BY created_at DESC`,
+          tenantId,
+          opts.state,
+        )
+      : await this.db.all<DeliveryRow>(
+          `SELECT * FROM webhook_deliveries WHERE tenant_id = ? ORDER BY created_at DESC`,
+          tenantId,
+        );
+    return rows.map(toDomain);
+  }
+
+  /**
+   * Record the outcome of one HTTP delivery attempt: append it to
+   * `webhook_delivery_attempts`, bump the delivery's `attempts`/`last_*`
+   * columns, and fold the outcome into the install's health rollup. Does
+   * NOT change `state` — see {@link transitionState}.
+   */
+  async recordAttempt(input: RecordAttemptInput): Promise<WebhookDelivery> {
+    const row = await this.mustFindDeliveryRow(input.deliveryId);
+    const now = this.now();
+
+    await this.db.run(
+      `INSERT INTO webhook_delivery_attempts
+         (id, delivery_id, attempt_no, started_at, duration_ms, http_status, error_class, trace_id)
+       VALUES (?,?,?,?,?,?,?,?)`,
+      crypto.randomUUID(),
+      input.deliveryId,
+      input.attemptNo,
+      input.startedAt,
+      input.durationMs ?? null,
+      input.httpStatus ?? null,
+      input.errorClass ?? null,
+      input.traceId ?? null,
+    );
+
+    await this.db.run(
+      `UPDATE webhook_deliveries
+          SET attempts = ?, last_status = ?, last_error_class = ?, last_trace_id = ?, updated_at = ?
+        WHERE delivery_id = ?`,
+      row.attempts + 1,
+      input.httpStatus ?? null,
+      input.errorClass ?? null,
+      input.traceId ?? null,
+      now,
+      input.deliveryId,
+    );
+
+    await this.foldAttemptIntoHealth(row.install_id, input, now);
+
+    return this.getOrThrowDelivery(input.deliveryId);
+  }
+
+  /**
+   * Move a delivery's `state` forward per `ALLOWED_TRANSITIONS`. The
+   * caller (Chunk C's dispatcher) decides WHEN to retry vs. give up; this
+   * only records that decision. Dead-lettering bumps the install's
+   * `dlqCount` — the one rollup field {@link recordAttempt} doesn't own.
+   */
+  async transitionState(input: TransitionStateInput): Promise<WebhookDelivery> {
+    const row = await this.mustFindDeliveryRow(input.deliveryId);
+    const from = row.state as DeliveryState;
+    if (!ALLOWED_TRANSITIONS[from].includes(input.toState)) {
+      throw new WebhookStoreError(
+        `Cannot transition delivery from "${from}" to "${input.toState}".`,
+        "invalid_transition",
+      );
+    }
+    if (input.toState === "retrying" && !input.nextAttemptAt) {
+      throw new WebhookStoreError(`A transition to "retrying" requires nextAttemptAt.`, "invalid_transition");
+    }
+
+    const now = this.now();
+    const nextAttemptAt = input.toState === "retrying" ? (input.nextAttemptAt as string) : null;
+    await this.db.run(
+      `UPDATE webhook_deliveries SET state = ?, next_attempt_at = ?, updated_at = ? WHERE delivery_id = ?`,
+      input.toState,
+      nextAttemptAt,
+      now,
+      input.deliveryId,
+    );
+
+    if (input.toState === "dead-lettered") {
+      await this.bumpDlqCount(row.install_id, now);
+    }
+
+    return this.getOrThrowDelivery(input.deliveryId);
+  }
+
+  /**
+   * D4 Chunk E's replay support: force a delivery back to `pending` with a
+   * fresh attempt budget — from ANY current state, INCLUDING the terminal
+   * `delivered`/`dead-lettered` states `transitionState`/`ALLOWED_TRANSITIONS`
+   * refuse to leave. This is deliberately a separate write path, not a call
+   * through `transitionState`: a replay is an operator/app decision to retry
+   * something the normal retry policy already gave up on (or already
+   * finished), not a policy-driven transition.
+   *
+   * The delivery's `deliveryId` is NEVER reassigned — see the header note on
+   * why it must stay stable across a replay (a receiver that already acked a
+   * `delivered` delivery needs the resend to carry the same wire id to
+   * dedupe it). `attempts` resets to 0 (a fresh {@link MAX_DELIVERY_ATTEMPTS}
+   * budget, tracked in `dispatcher.ts`) rather than continuing the old count
+   * — a replay is a new delivery cycle, not "attempt 7" of the old one.
+   * `webhook_delivery_attempts` — the append-only audit trail — is NEVER
+   * touched here; only this row's own counters move.
+   *
+   * A delivery leaving `dead-lettered` is no longer parked in the DLQ, so the
+   * install health rollup's `dlqCount` — read here as a CURRENT gauge of
+   * still-dead-lettered deliveries, not a lifetime counter — is decremented
+   * (floored at 0) to match.
+   */
+  async requeueForReplay(deliveryId: string): Promise<WebhookDelivery> {
+    const row = await this.mustFindDeliveryRow(deliveryId);
+    const now = this.now();
+
+    await this.db.run(
+      `UPDATE webhook_deliveries
+          SET state = 'pending', attempts = 0, next_attempt_at = ?, updated_at = ?
+        WHERE delivery_id = ?`,
+      now,
+      now,
+      deliveryId,
+    );
+
+    if (row.state === "dead-lettered") {
+      await this.decrementDlqCount(row.install_id, now);
+    }
+
+    return this.getOrThrowDelivery(deliveryId);
+  }
+
+  /**
+   * The `InstallHealth` rollup for one install — a never-attempted install
+   * reads as healthy/zeroed. `p95DeliveryMs` is computed here at READ time
+   * (not tracked as a running rollup column): no migration, no extra write
+   * on the `recordAttempt` hot path, just this one bounded read.
+   */
+  async getInstallHealth(installId: string): Promise<InstallHealth> {
+    const row = await this.db.first<HealthRow>(
+      `SELECT * FROM webhook_install_health WHERE install_id = ?`,
+      installId,
+    );
+    const health = toInstallHealth(row);
+    const p95DeliveryMs = await this.computeP95DeliveryMs(installId);
+    return p95DeliveryMs != null ? { ...health, p95DeliveryMs } : health;
+  }
+
+  /**
+   * p95 across `webhook_delivery_attempts.duration_ms` for the install's
+   * {@link P95_SAMPLE_SIZE} most recent deliveries — one bounded `IN (...)`
+   * read over that fixed set of delivery ids, sorted in JS (the sample is
+   * small enough that a DB-side percentile isn't worth the SQL). Nearest-rank
+   * method: `ceil(n * 0.95)`th smallest value, 1-indexed. `undefined` when no
+   * attempt in the sample has a recorded duration.
+   */
+  private async computeP95DeliveryMs(installId: string): Promise<number | undefined> {
+    const recent = await this.listDeliveriesByInstall(installId, { limit: P95_SAMPLE_SIZE });
+    if (recent.length === 0) return undefined;
+
+    const placeholders = recent.map(() => "?").join(",");
+    const rows = await this.db.all<{ duration_ms: number | null }>(
+      `SELECT duration_ms FROM webhook_delivery_attempts WHERE delivery_id IN (${placeholders})`,
+      ...recent.map((d) => d.deliveryId),
+    );
+    const durations = rows
+      .map((r) => r.duration_ms)
+      .filter((d): d is number => d != null)
+      .sort((a, b) => a - b);
+    if (durations.length === 0) return undefined;
+
+    const index = Math.min(durations.length - 1, Math.ceil(durations.length * 0.95) - 1);
+    return durations[index];
+  }
+
+  /** Deliveries in `retrying` whose `next_attempt_at` is due, earliest first. Uses `idx_webhook_deliveries_due`. */
+  async scanDueRetries(params: ScanDueRetriesInput): Promise<WebhookDelivery[]> {
+    const limit = params.limit ?? 100;
+    const rows = await this.db.all<DeliveryRow>(
+      `SELECT * FROM webhook_deliveries
+        WHERE state = 'retrying' AND next_attempt_at <= ?
+        ORDER BY next_attempt_at ASC
+        LIMIT ?`,
+      params.now,
+      limit,
+    );
+    return rows.map(toDomain);
+  }
+
+  private async foldAttemptIntoHealth(installId: string, attempt: RecordAttemptInput, now: string): Promise<void> {
+    const existing = await this.findHealthRow(installId);
+    const success = isSuccessAttempt(attempt.httpStatus);
+    const recentTraceIds = pushTraceId(
+      existing ? (JSON.parse(existing.recent_trace_ids) as string[]) : [],
+      attempt.traceId,
+    );
+
+    await this.writeHealthRow(installId, now, {
+      lastSuccessAt: success ? now : existing?.last_success_at ?? null,
+      failureStreak: success ? 0 : (existing?.failure_streak ?? 0) + 1,
+      dlqCount: existing?.dlq_count ?? 0,
+      lastStatus: attempt.httpStatus ?? existing?.last_status ?? null,
+      lastErrorClass: success ? null : attempt.errorClass ?? existing?.last_error_class ?? null,
+      recentTraceIds,
+    });
+  }
+
+  private async bumpDlqCount(installId: string, now: string): Promise<void> {
+    const existing = await this.findHealthRow(installId);
+    await this.writeHealthRow(installId, now, {
+      lastSuccessAt: existing?.last_success_at ?? null,
+      failureStreak: existing?.failure_streak ?? 0,
+      dlqCount: (existing?.dlq_count ?? 0) + 1,
+      lastStatus: existing?.last_status ?? null,
+      lastErrorClass: existing?.last_error_class ?? null,
+      recentTraceIds: existing ? (JSON.parse(existing.recent_trace_ids) as string[]) : [],
+    });
+  }
+
+  /** {@link requeueForReplay}'s counterpart to `bumpDlqCount` — floors at 0; a no-op if the install has no health row yet. */
+  private async decrementDlqCount(installId: string, now: string): Promise<void> {
+    const existing = await this.findHealthRow(installId);
+    if (!existing) return;
+    await this.writeHealthRow(installId, now, {
+      lastSuccessAt: existing.last_success_at,
+      failureStreak: existing.failure_streak,
+      dlqCount: Math.max(0, existing.dlq_count - 1),
+      lastStatus: existing.last_status,
+      lastErrorClass: existing.last_error_class,
+      recentTraceIds: JSON.parse(existing.recent_trace_ids) as string[],
+    });
+  }
+
+  private async findHealthRow(installId: string): Promise<HealthRow | null> {
+    return this.db.first<HealthRow>(`SELECT * FROM webhook_install_health WHERE install_id = ?`, installId);
+  }
+
+  private async writeHealthRow(
+    installId: string,
+    now: string,
+    patch: {
+      lastSuccessAt: string | null;
+      failureStreak: number;
+      dlqCount: number;
+      lastStatus: number | null;
+      lastErrorClass: string | null;
+      recentTraceIds: string[];
+    },
+  ): Promise<void> {
+    await this.db.run(
+      `INSERT OR REPLACE INTO webhook_install_health
+         (install_id, last_success_at, failure_streak, dlq_count, last_status, last_error_class, recent_trace_ids, updated_at)
+       VALUES (?,?,?,?,?,?,?,?)`,
+      installId,
+      patch.lastSuccessAt,
+      patch.failureStreak,
+      patch.dlqCount,
+      patch.lastStatus,
+      patch.lastErrorClass,
+      JSON.stringify(patch.recentTraceIds),
+      now,
+    );
+  }
+
+  /** Fetch a row or throw the generic fail-closed `not_found` — no enumeration detail. */
+  private async mustFindDeliveryRow(deliveryId: string): Promise<DeliveryRow> {
+    const row = await this.db.first<DeliveryRow>(
+      `SELECT * FROM webhook_deliveries WHERE delivery_id = ?`,
+      deliveryId,
+    );
+    if (!row) throw new WebhookStoreError(`Webhook delivery not found.`, "not_found");
+    return row;
+  }
+
+  private async getOrThrowDelivery(deliveryId: string): Promise<WebhookDelivery> {
+    return toDomain(await this.mustFindDeliveryRow(deliveryId));
+  }
+}