@k2b/cloud 0.43.0 → 0.45.0

package/src/services/app-events/admin.ts

@@ -0,0 +1,250 @@
+import type { TopicDeadLetter, TopicInventoryEntry } from "@k2b/sync";
+import type { SQL } from "bun";
+import { resolveCapabilityManifestPresentation } from "../../_internal/capabilities";
+import type { AppEventEnvelope } from "../../contracts/app-events";
+import type { CapabilityEventCatalogEntry } from "../../contracts/capabilities";
+import type { CapabilityRegistryEntry } from "../../contracts/registry";
+import { type AuditActor, audit } from "../audit";
+import { toPgTextArray } from "../postgres";
+import type { StoredDeclaration } from "./declarations";
+
+type Dependencies = {
+  registry: () => Promise<CapabilityRegistryEntry[]>;
+  deadLetters: (after?: string) => Promise<TopicDeadLetter<AppEventEnvelope>[]>;
+  streams: () => AsyncIterable<Pick<TopicInventoryEntry, "id" | "firstSequence">>;
+};
+export function createEventAdministration(db: SQL, dependencies: Dependencies) {
+  const catalog = async (locale = "en"): Promise<{ events: CapabilityEventCatalogEntry[] }> => {
+    const [entries, declarations, storedSubscriptions] = await Promise.all([
+      dependencies.registry(),
+      db<StoredDeclaration[]>`SELECT * FROM events.app_declarations WHERE retired_at IS NULL ORDER BY type, version`,
+      db<
+        {
+          appId: string;
+          subscriptionId: string;
+          event_type: string;
+          version: number;
+          state: "active" | "blocked" | "cancelled";
+          reason: string | null;
+          firstSeenAt: Date;
+        }[]
+      >`
+        SELECT app_id AS "appId", id AS "subscriptionId", event_type, version, state, reason, first_seen_at AS "firstSeenAt"
+        FROM events.app_subscriptions ORDER BY id
+      `,
+    ]);
+    return {
+      events: declarations.map((declaration) => {
+        const entry = entries.find(({ appId }) => appId === declaration.app_id);
+        const presentation = entry
+          ? resolveCapabilityManifestPresentation(entry.manifest, entry.presentation, locale).events?.find(
+              (event) => `${entry.appId}.${event.localId}` === declaration.type && event.version === declaration.version,
+            )
+          : undefined;
+        return {
+          type: declaration.type,
+          version: declaration.version,
+          title: presentation?.title ?? declaration.title,
+          description: presentation?.description ?? declaration.description,
+          subject: `${declaration.app_id}.${declaration.subject}`,
+          place: declaration.place ? `${declaration.app_id}.${declaration.place}` : null,
+          automation: declaration.automation,
+          dataSchema: declaration.data_schema,
+          subscribers: storedSubscriptions
+            .filter((subscription) => subscription.event_type === declaration.type && subscription.version === declaration.version)
+            .map(({ event_type: _eventType, version: _version, firstSeenAt, ...subscription }) => ({
+              ...subscription,
+              firstSeenAt: firstSeenAt.toISOString(),
+            })),
+        };
+      }),
+    };
+  };
+  const subscriptions = async (app?: string) => {
+    return {
+      items: await db`
+        SELECT subscription.id, subscription.app_id AS "appId", subscription.event_type AS "eventType", subscription.version,
+          subscription.state, subscription.reason, subscription.first_seen_at AS "firstSeenAt",
+          (SELECT jsonb_object_agg(state, count) FROM (
+            SELECT state, count(*)::int AS count FROM events.app_deliveries
+            WHERE subscription_id = subscription.id GROUP BY state
+          ) counts) AS counts
+        FROM events.app_subscriptions subscription
+        WHERE (${app ?? null}::text IS NULL OR subscription.app_id = ${app ?? null}) ORDER BY subscription.id
+      `,
+    };
+  };
+  const deliveries = async (filter: { app?: string; state?: string; eventType?: string; offset?: number; limit?: number } = {}) => {
+    const limit = Math.max(1, Math.min(100, filter.limit ?? 50));
+    const offset = Math.max(0, filter.offset ?? 0);
+    const rows = await db`
+      SELECT delivery.id, delivery.event_id AS "eventId", delivery.subscription_id AS "subscriptionId",
+        subscription.app_id AS "appId", event.type AS "eventType", delivery.state, delivery.code, delivery.attempts,
+        delivery.last_error AS "lastError", delivery.last_attempt_at AS "lastAttemptAt",
+        delivery.next_attempt_at AS "nextAttemptAt", delivery.created_at AS "createdAt",
+        (SELECT earlier.id FROM events.app_deliveries earlier
+          WHERE earlier.subscription_id = delivery.subscription_id AND earlier.subject_key = delivery.subject_key
+            AND earlier.seq < delivery.seq AND earlier.state IN ('pending', 'retrying', 'failed')
+            AND delivery.state IN ('pending', 'retrying') ORDER BY earlier.seq LIMIT 1) AS "heldBy",
+        count(*) OVER()::int AS total
+      FROM events.app_deliveries delivery
+      JOIN events.app_subscriptions subscription ON subscription.id = delivery.subscription_id
+      JOIN events.app_events event ON event.id = delivery.event_id
+      WHERE (${filter.app ?? null}::text IS NULL OR subscription.app_id = ${filter.app ?? null})
+        AND (${filter.state ?? null}::text IS NULL OR delivery.state = ${filter.state ?? null})
+        AND (${filter.eventType ?? null}::text IS NULL OR event.type = ${filter.eventType ?? null})
+      ORDER BY delivery.seq DESC LIMIT ${limit} OFFSET ${offset}
+    `;
+    return {
+      items: rows.map((row: Record<string, unknown>) => {
+        const { total, ...item } = row;
+        return item;
+      }),
+      pagination: { offset, limit, total: Number(rows[0]?.total ?? 0) },
+    };
+  };
+  const redeliver = async (id: string, actor: AuditActor) =>
+    db.begin(async (tx) => {
+      const changed = await tx`
+        UPDATE events.app_deliveries delivery SET state = 'pending', code = NULL, last_error = NULL,
+          attempts = 0, unreachable_count = 0, retained_from = now(), next_attempt_at = now(), claimed_until = NULL, claim_token = NULL, updated_at = now()
+        WHERE delivery.id = ${id}::uuid AND delivery.state IN ('failed', 'cancelled', 'done')
+          AND EXISTS(SELECT FROM events.app_subscriptions subscription
+            WHERE subscription.id = delivery.subscription_id AND subscription.state = 'active')
+          AND EXISTS(SELECT FROM events.app_events event WHERE event.id = delivery.event_id AND event.state = 'accepted')
+        RETURNING event_id AS "eventId", id
+      `;
+      if (!changed.length) return null;
+      await audit.record(
+        {
+          action: "app_events.delivery.redeliver",
+          outcome: "allowed",
+          actor,
+          target: { type: "app_delivery", id },
+          metadata: { eventId: changed[0]?.eventId },
+        },
+        tx,
+      );
+      return changed[0];
+    });
+
+  const status = async () => {
+    const [stored] = await db`
+      SELECT state, code, last_sequence AS "lastSequence", gap_from AS "gapFrom", gap_to AS "gapTo", updated_at AS "updatedAt"
+      FROM events.app_broker_status WHERE id = true
+    `;
+    const [rejected] = await db`SELECT count(*)::int AS count FROM events.app_events WHERE state = 'rejected'`;
+    const rejections = await db`
+      SELECT type, source, code, received_at AS time FROM events.app_events
+      WHERE state = 'rejected' ORDER BY seq DESC LIMIT 10
+    `;
+    const deadLetters = await dependencies.deadLetters();
+    return {
+      ...stored,
+      rejectedCount: Number(rejected?.count ?? 0),
+      rejections,
+      rejectionLimit: 10,
+      deadLetters: deadLetters.map(({ messageId, eventId, consumer, attempts, failedAt, reason }) => ({
+        messageId,
+        eventId,
+        consumer,
+        attempts,
+        failedAt,
+        reason,
+      })),
+      deadLetterLimit: 100,
+    };
+  };
+  const checkGap = async () => {
+    const [stored] = await db<{ last_sequence: string }[]>`
+      SELECT last_sequence FROM events.app_broker_status WHERE id = true
+    `;
+    let position = Number(stored?.last_sequence ?? 0);
+    const seen = new Set<number>();
+    const deadline = performance.now() + 30_000;
+    let after: string | undefined;
+    while (true) {
+      if (performance.now() >= deadline) return;
+      const page = await dependencies.deadLetters(after);
+      for (const deadLetter of page) {
+        if (deadLetter.consumer === "core-app-events-v1" && deadLetter.event) seen.add(deadLetter.event.sequence);
+      }
+      if (page.length < 100) break;
+      const next = page.at(-1)?.messageId;
+      if (!next || next === after) return;
+      after = next;
+    }
+    if (seen.size) {
+      await db`
+        UPDATE events.app_broker_status SET unseen_sequences = unseen_sequences - (
+          SELECT range_agg(int8range(sequence, sequence + 1, '[)'))
+          FROM unnest(${toPgTextArray([...seen].map(String))}::bigint[]) AS sequence
+        ) WHERE id = true AND NOT isempty(unseen_sequences)
+      `;
+    }
+    while (seen.has(position + 1)) position++;
+    await db`
+      UPDATE events.app_broker_status SET last_sequence = ${position}, updated_at = now()
+      WHERE id = true AND last_sequence < ${position}
+    `;
+    for await (const stream of dependencies.streams()) {
+      if (stream.id !== "cloud:app-events") continue;
+      await db`
+        UPDATE events.app_broker_status SET state = 'gap_detected', code = 'reconcile_required',
+          gap_from = COALESCE(gap_from, LEAST(lower(unseen_sequences), last_sequence + 1)), gap_to = ${stream.firstSequence - 1}, updated_at = now()
+        WHERE id = true AND LEAST(lower(unseen_sequences), last_sequence + 1) < ${stream.firstSequence}
+          AND (state <> 'gap_detected' OR gap_to IS DISTINCT FROM ${stream.firstSequence - 1}::bigint)
+      `;
+      break;
+    }
+  };
+  const acknowledgeGap = async (actor: AuditActor) =>
+    db.begin(async (tx) => {
+      const [previous] = await tx`
+        SELECT state, gap_from, gap_to FROM events.app_broker_status WHERE id = true FOR UPDATE
+      `;
+      if (previous?.state !== "gap_detected") return null;
+      const [result] = await tx`
+        UPDATE events.app_broker_status SET state = 'ready', code = NULL,
+          last_sequence = GREATEST(last_sequence, gap_to),
+          unseen_sequences = unseen_sequences - int8multirange(int8range(NULL, gap_to + 1, '[)')), gap_from = NULL, gap_to = NULL, updated_at = now()
+        WHERE id = true RETURNING state, last_sequence AS "lastSequence"
+      `;
+      await audit.record(
+        {
+          action: "app_events.gap.acknowledge",
+          outcome: "allowed",
+          actor,
+          target: { type: "app_event_broker", id: "core-app-events-v1" },
+          metadata: { gapFrom: previous.gap_from, gapTo: previous.gap_to },
+        },
+        tx,
+      );
+      return result;
+    });
+  const retain = async () =>
+    db.begin(async (tx) => {
+      await tx`
+        UPDATE events.app_deliveries SET state = 'cancelled', code = 'retention_expired',
+          claimed_until = NULL, claim_token = NULL, updated_at = now()
+        WHERE id IN (
+          SELECT id FROM events.app_deliveries
+          WHERE state IN ('pending', 'retrying', 'failed') AND retained_from < now() - interval '30 days' LIMIT 1000
+        )
+      `;
+      await tx`
+        DELETE FROM events.app_deliveries WHERE id IN (
+          SELECT id FROM events.app_deliveries
+          WHERE state IN ('done', 'cancelled') AND updated_at < now() - interval '7 days' LIMIT 1000
+        )
+      `;
+      await tx`
+        DELETE FROM events.app_events WHERE id IN (
+          SELECT event.id FROM events.app_events event
+          WHERE received_at < now() - interval '30 days'
+            AND NOT EXISTS(SELECT FROM events.app_deliveries delivery WHERE delivery.event_id = event.id) LIMIT 1000
+        )
+      `;
+    });
+  return { catalog, subscriptions, deliveries, redeliver, status, checkGap, acknowledgeGap, retain };
+}

package/src/services/app-events/declarations.ts

@@ -0,0 +1,187 @@
+import type { SQL } from "bun";
+import { capabilityHash } from "../../_internal/capability-schemas";
+import { appEventSubscriptionId } from "../../contracts/app-events";
+import type { CapabilityRegistryEntry } from "../../contracts/registry";
+import { audit } from "../audit";
+import { toPgTextArray } from "../postgres";
+
+export type StoredDeclaration = {
+  app_id: string;
+  type: string;
+  version: number;
+  subject: string;
+  place: string | null;
+  automation: boolean;
+  title: string;
+  description: string;
+  data_schema: Record<string, unknown>;
+  schema_hash: string;
+  retired_at: Date | null;
+};
+
+type Transition = { id: string; previous_state: string; state: string; reason: string | null };
+async function auditTransitions(tx: SQL, transitions: Transition[]) {
+  for (const transition of transitions) {
+    await audit.record(
+      {
+        action: "app_events.subscription.state_changed",
+        outcome: "allowed",
+        target: { type: "app_subscription", id: transition.id },
+        metadata: { previousState: transition.previous_state, state: transition.state, reason: transition.reason },
+      },
+      tx,
+    );
+  }
+}
+
+export function createDeclarationSync(db: SQL, registry: () => Promise<CapabilityRegistryEntry[]>) {
+  let previousSnapshot: string | undefined;
+  let lastCheckedAt = -Infinity;
+  let running: Promise<void> | undefined;
+
+  async function sync(entries: CapabilityRegistryEntry[]) {
+    await db.begin(async (tx) => {
+      for (const entry of entries.toSorted((left, right) => left.appId.localeCompare(right.appId))) {
+        const hash = capabilityHash({ events: entry.manifest.events ?? [], subscribes: entry.manifest.subscribes ?? [] });
+        const [known] = await tx`SELECT hash FROM events.app_manifests WHERE app_id = ${entry.appId}`;
+        if (known?.hash === hash) continue;
+        // Serialize only replicas observing a changed manifest for this application.
+        await tx`SELECT pg_advisory_xact_lock(hashtext(${`cloud.app-events:${entry.appId}`}))`;
+        const [stored] = await tx`SELECT hash FROM events.app_manifests WHERE app_id = ${entry.appId}`;
+        if (stored?.hash === hash) continue;
+        const declared = (entry.manifest.events ?? []).map((event) => `${entry.appId}.${event.localId}@${event.version}`);
+        await tx`
+          UPDATE events.app_declarations SET retired_at = now()
+          WHERE app_id = ${entry.appId} AND retired_at IS NULL
+            AND NOT(type || '@' || version::text = ANY(${toPgTextArray(declared)}::text[]))
+        `;
+        for (const event of entry.manifest.events ?? []) {
+          const type = `${entry.appId}.${event.localId}`;
+          const schemaHash = event.schemaHash;
+          await tx`
+            INSERT INTO events.app_declarations (
+              app_id, type, version, subject, place, automation, title, description, data_schema, schema_hash
+            ) VALUES (
+              ${entry.appId}, ${type}, ${event.version}, ${event.subject}, ${event.place ?? null},
+              ${event.automation !== false}, ${event.title}, ${event.description},
+              ${JSON.stringify(event.dataSchema)}::text::jsonb, ${schemaHash}
+            ) ON CONFLICT(app_id, type, version) DO UPDATE SET
+              subject = EXCLUDED.subject, place = EXCLUDED.place, automation = EXCLUDED.automation,
+              title = EXCLUDED.title, description = EXCLUDED.description, data_schema = EXCLUDED.data_schema,
+              schema_hash = EXCLUDED.schema_hash, retired_at = NULL
+            WHERE (app_declarations.subject, app_declarations.place, app_declarations.automation,
+              app_declarations.title, app_declarations.description, app_declarations.schema_hash)
+              IS DISTINCT FROM (EXCLUDED.subject, EXCLUDED.place, EXCLUDED.automation,
+                EXCLUDED.title, EXCLUDED.description, EXCLUDED.schema_hash)
+              OR app_declarations.retired_at IS NOT NULL
+          `;
+        }
+        const ids: string[] = [];
+        for (const subscription of entry.manifest.subscribes ?? []) {
+          const id = appEventSubscriptionId(entry.appId, subscription.type, subscription.version);
+          ids.push(id);
+          const inserted = await tx`
+            INSERT INTO events.app_subscriptions(id, app_id, event_type, version, state, reason)
+            VALUES (${id}, ${entry.appId}, ${subscription.type}, ${subscription.version}, 'blocked', 'event_not_declared')
+            ON CONFLICT DO NOTHING RETURNING id
+          `;
+          if (inserted.length) {
+            await audit.record(
+              {
+                action: "app_events.subscription.first_seen",
+                outcome: "allowed",
+                target: { type: "app_subscription", id },
+                metadata: { appId: entry.appId, eventType: subscription.type, version: subscription.version },
+              },
+              tx,
+            );
+          } else {
+            const restored = await tx<Transition[]>`
+              WITH previous AS MATERIALIZED (
+                SELECT id, state AS previous_state FROM events.app_subscriptions WHERE id = ${id} AND state = 'cancelled'
+                FOR UPDATE
+              ) UPDATE events.app_subscriptions subscription SET
+                state = 'blocked', reason = 'event_not_declared', updated_at = now()
+              FROM previous WHERE subscription.id = previous.id
+              RETURNING subscription.id, previous.previous_state, subscription.state, subscription.reason
+            `;
+            await auditTransitions(tx, restored);
+          }
+        }
+        const removed = await tx<Transition[]>`
+          WITH previous AS MATERIALIZED (
+            SELECT id, state AS previous_state FROM events.app_subscriptions
+            WHERE app_id = ${entry.appId} AND NOT(id = ANY(${toPgTextArray(ids)}::text[])) AND state <> 'cancelled'
+            ORDER BY id FOR UPDATE
+          ) UPDATE events.app_subscriptions subscription SET
+            state = 'cancelled', reason = 'subscription_removed', updated_at = now()
+          FROM previous WHERE subscription.id = previous.id
+          RETURNING subscription.id, previous.previous_state, subscription.state, subscription.reason
+        `;
+        await auditTransitions(tx, removed);
+        if (removed.length) {
+          await tx`
+            UPDATE events.app_deliveries SET state = 'cancelled', code = 'subscription_removed',
+              claimed_until = NULL, claim_token = NULL, updated_at = now()
+            WHERE subscription_id = ANY(${toPgTextArray(removed.map(({ id }) => id))}::text[])
+              AND state IN ('pending', 'retrying', 'failed')
+          `;
+        }
+        await tx`
+          INSERT INTO events.app_manifests(app_id, hash) VALUES (${entry.appId}, ${hash})
+          ON CONFLICT(app_id) DO UPDATE SET hash = EXCLUDED.hash, updated_at = now()
+          WHERE app_manifests.hash IS DISTINCT FROM EXCLUDED.hash
+        `;
+      }
+      // A newly observed publisher can unblock an offline subscriber too.
+      const transitions = await tx<Transition[]>`
+        WITH desired AS MATERIALIZED (
+          SELECT subscription.id, subscription.state AS previous_state,
+            CASE WHEN declaration.type IS NULL THEN 'blocked'
+              WHEN declaration.place IS NOT NULL THEN 'blocked' ELSE 'active' END AS state,
+            CASE WHEN declaration.type IS NULL THEN 'event_not_declared'
+              WHEN declaration.place IS NOT NULL THEN 'resource_events_unsupported' ELSE NULL END AS reason
+          FROM events.app_subscriptions subscription
+          LEFT JOIN events.app_declarations declaration ON declaration.type = subscription.event_type
+            AND declaration.version = subscription.version
+            AND declaration.app_id = split_part(subscription.event_type, '.', 1) AND declaration.retired_at IS NULL
+          WHERE subscription.state <> 'cancelled'
+            AND (subscription.state, subscription.reason) IS DISTINCT FROM (
+              CASE WHEN declaration.type IS NULL OR declaration.place IS NOT NULL THEN 'blocked' ELSE 'active' END,
+              CASE WHEN declaration.type IS NULL THEN 'event_not_declared'
+                WHEN declaration.place IS NOT NULL THEN 'resource_events_unsupported' ELSE NULL END
+            )
+          ORDER BY subscription.id FOR UPDATE OF subscription
+        ) UPDATE events.app_subscriptions subscription SET state = desired.state, reason = desired.reason, updated_at = now()
+        FROM desired WHERE subscription.id = desired.id
+          AND (subscription.state, subscription.reason) IS DISTINCT FROM (desired.state, desired.reason)
+        RETURNING subscription.id, desired.previous_state, subscription.state, subscription.reason
+      `;
+      await auditTransitions(tx, transitions);
+    });
+  }
+
+  async function reconcileSubscriptions(snapshot?: CapabilityRegistryEntry[]) {
+    if (running) await running;
+    const entries = snapshot ?? (await registry());
+    const snapshotHash = capabilityHash(
+      entries
+        .map(({ appId, manifest }) => ({
+          appId,
+          events: manifest.events ?? [],
+          subscribes: manifest.subscribes ?? [],
+        }))
+        .sort((left, right) => left.appId.localeCompare(right.appId)),
+    );
+    if (snapshotHash === previousSnapshot && performance.now() - lastCheckedAt < 30_000) return;
+    running = sync(entries);
+    try {
+      await running;
+      previousSnapshot = snapshotHash;
+      lastCheckedAt = performance.now();
+    } finally {
+      running = undefined;
+    }
+  }
+  return { reconcileSubscriptions };
+}

package/src/services/app-events/deliver.ts

@@ -0,0 +1,218 @@
+import type { SQL } from "bun";
+import type { getCapability } from "../../_internal/registry";
+import type { AppEventEnvelope } from "../../contracts/app-events";
+import { audit } from "../audit";
+import { signAppEventInvocation } from "../identity/invocation-token";
+import { toPgTextArray } from "../postgres";
+
+export const APP_EVENT_DELIVERY_BACKOFF_MS = [1_000, 5_000, 30_000, 120_000, 600_000, 1_800_000, 3_600_000] as const;
+export const APP_EVENT_DELIVERY_ATTEMPTS = APP_EVENT_DELIVERY_BACKOFF_MS.length + 1;
+const BATCH_SIZE = 32;
+const CONCURRENCY = 4;
+// The subscriber has 30 seconds to answer before Core ends the request.
+const DELIVERY_TIMEOUT_MS = 45_000;
+type Delivery = {
+  id: string;
+  event_id: string;
+  subscription_id: string;
+  attempts: number;
+  unreachable_count: number;
+  claim_token: string;
+  envelope: AppEventEnvelope;
+  app_id: string;
+};
+type Dependencies = {
+  fetch?: typeof fetch;
+  sign?: typeof signAppEventInvocation;
+};
+const message = (error: unknown) => (error instanceof Error ? error.message : String(error)).slice(0, 1000);
+async function auditFailure(tx: SQL, row: Delivery, code: string | null) {
+  await audit.record(
+    {
+      action: "app_events.delivery.failed",
+      outcome: "failed",
+      target: { type: "app_delivery", id: row.id },
+      error: { code: code ?? "handler_failed" },
+      metadata: { eventId: row.event_id, subscriptionId: row.subscription_id, attempts: row.attempts },
+    },
+    tx,
+  );
+}
+
+export function createEventDelivery(
+  db: SQL,
+  findApp: typeof getCapability,
+  reconcileSubscriptions: () => Promise<void>,
+  dependencies: Dependencies,
+) {
+  async function postpone(row: Delivery) {
+    const delay = Math.min(30_000 * 2 ** Math.min(row.unreachable_count, 7), 3_600_000);
+    await db`
+      UPDATE events.app_deliveries SET state = 'retrying', code = 'subscriber_unreachable', attempts = attempts - 1,
+        unreachable_count = unreachable_count + 1, claimed_until = NULL, claim_token = NULL,
+        next_attempt_at = now() + ${delay} * interval '1 millisecond', updated_at = now()
+      WHERE id = ${row.id}::uuid AND claim_token = ${row.claim_token}::uuid AND state IN ('pending', 'retrying')
+    `;
+  }
+  const claim = async () =>
+    db.begin(async (tx) => {
+      const exhausted = await tx<Delivery[]>`
+        UPDATE events.app_deliveries SET state = 'failed', code = 'attempts_exhausted',
+          claimed_until = NULL, claim_token = NULL, updated_at = now()
+        WHERE id IN (
+          SELECT id FROM events.app_deliveries WHERE state IN ('pending', 'retrying')
+            AND attempts >= ${APP_EVENT_DELIVERY_ATTEMPTS} AND (claimed_until IS NULL OR claimed_until <= now())
+          ORDER BY seq LIMIT ${BATCH_SIZE} FOR UPDATE SKIP LOCKED
+        ) RETURNING *
+      `;
+      for (const delivery of exhausted) await auditFailure(tx, delivery, "attempts_exhausted");
+      const rows = await tx<Delivery[]>`
+        WITH candidates AS MATERIALIZED (
+          SELECT delivery.id FROM events.app_deliveries delivery
+          JOIN events.app_subscriptions subscription ON subscription.id = delivery.subscription_id
+          WHERE delivery.state IN ('pending', 'retrying') AND delivery.next_attempt_at <= now()
+            AND (delivery.claimed_until IS NULL OR delivery.claimed_until <= now())
+            AND delivery.attempts < ${APP_EVENT_DELIVERY_ATTEMPTS} AND subscription.state = 'active'
+            AND NOT EXISTS (
+              SELECT FROM events.app_deliveries earlier
+              WHERE earlier.subscription_id = delivery.subscription_id AND earlier.subject_key = delivery.subject_key
+                AND earlier.seq < delivery.seq AND earlier.state IN ('pending', 'retrying', 'failed')
+            )
+          ORDER BY delivery.seq LIMIT ${CONCURRENCY} FOR UPDATE OF delivery SKIP LOCKED
+        ) UPDATE events.app_deliveries delivery SET claimed_until = now() + interval '60 seconds',
+          claim_token = gen_random_uuid(), attempts = delivery.attempts + 1, last_attempt_at = now(), updated_at = now()
+        FROM candidates WHERE delivery.id = candidates.id RETURNING delivery.*
+      `;
+      if (!rows.length) return [];
+      return tx<Delivery[]>`
+        SELECT delivery.*, subscription.app_id, event.envelope
+        FROM events.app_deliveries delivery
+        JOIN events.app_subscriptions subscription ON subscription.id = delivery.subscription_id
+        JOIN events.app_events event ON event.id = delivery.event_id
+        WHERE delivery.id = ANY(${toPgTextArray(rows.map((row) => row.id))}::uuid[]) ORDER BY delivery.seq
+      `;
+    });
+
+  const finish = async (
+    row: Delivery,
+    state: "done" | "cancelled" | "retrying" | "failed",
+    code: string | null,
+    error: string | null = null,
+  ) =>
+    db.begin(async (tx) => {
+      const delay = APP_EVENT_DELIVERY_BACKOFF_MS[Math.min(row.attempts - 1, APP_EVENT_DELIVERY_BACKOFF_MS.length - 1)] ?? 0;
+      const changed = await tx`
+        UPDATE events.app_deliveries SET state = ${state}, code = ${code}, last_error = ${error},
+          claimed_until = NULL, claim_token = NULL, unreachable_count = 0,
+          next_attempt_at = now() + ${delay} * interval '1 millisecond', updated_at = now()
+        WHERE id = ${row.id}::uuid AND claim_token = ${row.claim_token}::uuid AND state IN ('pending', 'retrying')
+        RETURNING id
+      `;
+      if (state === "done" && changed.length) {
+        await tx`
+          UPDATE events.app_deliveries delivery SET next_attempt_at = now(), unreachable_count = 0, updated_at = now()
+          FROM events.app_subscriptions subscription
+          WHERE subscription.id = delivery.subscription_id AND subscription.app_id = ${row.app_id}
+            AND delivery.state = 'retrying' AND delivery.code = 'subscriber_unreachable'
+            AND delivery.claim_token IS NULL AND delivery.next_attempt_at > now()
+        `;
+      }
+      if (state === "failed" && changed.length) await auditFailure(tx, row, code);
+    });
+
+  const deliver = async (row: Delivery) => {
+    try {
+      const [subscription] = await db`
+        SELECT state, reason FROM events.app_subscriptions WHERE id = ${row.subscription_id}
+      `;
+      if (subscription?.state === "cancelled") {
+        await finish(row, "cancelled", "subscription_removed");
+        return;
+      }
+      if (subscription?.state !== "active") {
+        // A declaration transition can block a subscription after this delivery was claimed.
+        await db`
+          UPDATE events.app_deliveries SET state = 'retrying', code = ${subscription?.reason ?? "subscription_blocked"},
+            attempts = attempts - 1, claimed_until = NULL, claim_token = NULL, next_attempt_at = now(), updated_at = now()
+          WHERE id = ${row.id}::uuid AND claim_token = ${row.claim_token}::uuid AND state IN ('pending', 'retrying')
+        `;
+        return;
+      }
+      const app = await findApp(row.app_id);
+      if (!app) {
+        await postpone(row);
+        return;
+      }
+      if (row.envelope.place) {
+        await finish(row, "failed", "resource_events_unsupported");
+        return;
+      }
+      const payload = { envelope: row.envelope, subscriptionId: row.subscription_id, attempt: row.attempts };
+      const token = await (dependencies.sign ?? signAppEventInvocation)({
+        appId: row.app_id,
+        subscriptionId: row.subscription_id,
+        eventId: row.event_id,
+        payload,
+      });
+      const url = new URL("/api/_internal/events/v1/deliver", app.endpoint);
+      let response: Response;
+      try {
+        response = await (dependencies.fetch ?? fetch)(url, {
+          method: "POST",
+          headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
+          body: JSON.stringify(payload),
+          redirect: "error",
+          signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS),
+        });
+      } catch (error) {
+        const code = error instanceof Error && "code" in error ? error.code : undefined;
+        // Only errors that prove no request reached the subscriber refund an attempt.
+        if (["ConnectionRefused", "ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "EHOSTUNREACH", "ENETUNREACH"].includes(String(code))) {
+          await postpone(row);
+        } else {
+          await finish(
+            row,
+            row.attempts >= APP_EVENT_DELIVERY_ATTEMPTS ? "failed" : "retrying",
+            error instanceof Error && error.name === "TimeoutError" ? "handler_timeout" : "connection_lost",
+            message(error),
+          );
+        }
+        return;
+      }
+      if (response.ok) {
+        await finish(row, "done", null);
+        return;
+      }
+      if (response.status === 422) {
+        await finish(row, "failed", "schema_mismatch");
+        return;
+      }
+      if (response.status === 410) {
+        await finish(row, row.attempts >= APP_EVENT_DELIVERY_ATTEMPTS ? "failed" : "retrying", "subscription_unavailable");
+        return;
+      }
+      throw new Error(`Subscriber returned HTTP ${response.status}`);
+    } catch (error) {
+      await finish(
+        row,
+        row.attempts >= APP_EVENT_DELIVERY_ATTEMPTS ? "failed" : "retrying",
+        row.attempts >= APP_EVENT_DELIVERY_ATTEMPTS ? "attempts_exhausted" : "handler_failed",
+        message(error),
+      );
+    }
+  };
+
+  const work = async () => {
+    await reconcileSubscriptions();
+    let total = 0;
+    for (let batch = 0; batch < BATCH_SIZE / CONCURRENCY; batch++) {
+      const rows = await claim();
+      await Promise.all(rows.map(deliver));
+      total += rows.length;
+      if (!rows.length) break;
+    }
+    return total;
+  };
+
+  return { claim, deliver, work };
+}