@antasphere/chassis-server 1.1.0 → 1.2.0

package/dist/webhooks/service.js

@@ -0,0 +1,861 @@
+import { randomUUID } from 'node:crypto';
+import { and, desc, eq, sql } from 'drizzle-orm';
+import { Counter } from 'prom-client';
+import { user as userTable, webhookDeliveries, webhookEndpoints, workspaceMembers, workspaces } from '@antasphere/chassis-db';
+import { createSecretBox } from '../secret-box.js';
+import { createdAtText, decodeKeysetCursor, encodeKeysetCursor, keysetBeforeValue } from '../pagination.js';
+import { WEBHOOK_PING_EVENT } from './catalog.js';
+import { endpointWants } from './match.js';
+import { buildWebhookDisabledEmail } from './mail.js';
+import { checkWebhookUrl, safePost, WebhookUrlRefusedError } from './safe-fetch.js';
+import { generateWebhookSecret, webhookHeaders } from './signing.js';
+import { sendInTx } from './tx-send.js';
+/**
+ * Outbound webhooks (PRDCT-3482): the chassis's machinery, tool-neutral.
+ *
+ * EMIT: a domain write calls `emit(tx, …)` INSIDE its own transaction; one
+ * pg-boss job (the event's envelope) is written by the same connection, so
+ * the event is durable with the write and never exists for a rolled-back
+ * one. A per-replica cache of "does any enabled endpoint of this workspace
+ * want this type" (`listenerTtlMs`) makes an emit nobody listens to cost
+ * one map lookup: a view with no endpoint costs no job.
+ *
+ * FAN-OUT (`webhook-events`): the event is matched against the workspace's
+ * enabled endpoints (type, then filters); one `webhook_deliveries` row and
+ * one delivery job per match, in one transaction.
+ *
+ * DELIVERY (`webhook-deliveries`): the envelope is signed (Standard
+ * Webhooks, two signatures during a rotation overlap) and POSTed through the
+ * guarded fetch. A 2xx is `delivered`; anything else fails the job, which
+ * pg-boss retries on the queue's budget (ten retries from thirty seconds,
+ * about eight and a half hours), then moves to `webhook-deliveries-held`:
+ * the row is marked `held`, logged, counted, and queued again an hour later
+ * with the whole budget once more, round after round, until the receiver
+ * takes it or the endpoint is disabled. A streak of
+ * `disableAfterFailures` failures spanning `disableAfterMs` switches the
+ * endpoint off and mails its creator once.
+ *
+ * The secret is stored sealed (`secret-box.ts`, info `chassis:webhook-endpoint`)
+ * and shown once, at creation and at rotation.
+ */
+export const WEBHOOK_EVENTS_QUEUE = 'webhook-events';
+export const WEBHOOK_DELIVERIES_QUEUE = 'webhook-deliveries';
+export const WEBHOOK_DELIVERIES_HELD_QUEUE = 'webhook-deliveries-held';
+export const WEBHOOK_PURGE_QUEUE = 'webhook-purge';
+/** The HKDF info of the endpoints' secret box: never the idempotency one. */
+export const WEBHOOK_SECRET_BOX_INFO = 'chassis:webhook-endpoint';
+export const DEFAULT_WEBHOOK_DIALS = {
+    retryLimit: 10,
+    retryDelaySeconds: 30,
+    heldDelaySeconds: 3600,
+    disableAfterFailures: 20,
+    disableAfterMs: 2 * 24 * 3600 * 1000,
+    listenerTtlMs: 30_000,
+    previousSecretOverlapMs: 24 * 3600 * 1000,
+    timeoutMs: 10_000
+};
+export class WebhookRefusedError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = 'WebhookRefusedError';
+    }
+}
+/**
+ * Thrown by an attempt that did not run because the endpoint already holds
+ * its share of this replica's pollers (`MAX_IN_FLIGHT_PER_ENDPOINT`): the
+ * worker puts the job back in the queue a moment later, its retry budget
+ * untouched, so one dead receiver can never take every slot (verifier
+ * round 2, MJ-R2-1).
+ */
+export class WebhookDeliveryDeferred extends Error {
+    deliveryId;
+    constructor(deliveryId) {
+        super('deferred');
+        this.deliveryId = deliveryId;
+        this.name = 'WebhookDeliveryDeferred';
+    }
+}
+/** How many attempts one endpoint may have in flight on one replica at once. */
+export const MAX_IN_FLIGHT_PER_ENDPOINT = 2;
+/** Thrown by a retryable attempt that failed, so pg-boss retries the job. */
+export class WebhookDeliveryFailed extends Error {
+    deliveryId;
+    constructor(deliveryId, message) {
+        super(message);
+        this.deliveryId = deliveryId;
+        this.name = 'WebhookDeliveryFailed';
+    }
+}
+const MAX_ERROR_TEXT = 300;
+export class WebhookService {
+    deps;
+    dials;
+    promMetrics;
+    box;
+    listeners = new Map();
+    deliveries;
+    held;
+    disabled;
+    now;
+    /** Filled by boot once pg-boss exists (`attach`); the jobs are declared before it. */
+    boss = null;
+    /** Attempts in flight per endpoint on this replica (the per-endpoint bound). */
+    inFlight = new Map();
+    constructor(deps) {
+        this.deps = deps;
+        this.dials = { ...DEFAULT_WEBHOOK_DIALS, ...deps.dials };
+        this.box = createSecretBox(deps.authSecret, WEBHOOK_SECRET_BOX_INFO);
+        this.now = deps.now ?? (() => new Date());
+        this.deliveries = new Counter({
+            name: 'webhook_delivery_attempts_total',
+            help: 'Webhook delivery attempts by outcome: delivered, failed (will retry), refused (the endpoint off or the address refused)',
+            labelNames: ['outcome'],
+            registers: []
+        });
+        this.held = new Counter({
+            name: 'webhook_deliveries_held_total',
+            help: 'Webhook deliveries that ran out of retries and were held for an hourly re-drive',
+            registers: []
+        });
+        this.disabled = new Counter({
+            name: 'webhook_endpoints_disabled_total',
+            help: 'Webhook endpoints switched off after a failure streak',
+            registers: []
+        });
+        this.promMetrics = [this.deliveries, this.held, this.disabled];
+    }
+    attach(boss) {
+        this.boss = boss;
+    }
+    get catalog() {
+        return this.deps.catalog;
+    }
+    // ── Emit ─────────────────────────────────────────────────────────────────
+    async emit(conn, event) {
+        const declared = this.deps.catalog.events.get(event.type);
+        if (!declared) {
+            // A type the catalog does not know is a programming error, not a runtime condition.
+            throw new Error(`webhooks: "${event.type}" is not a declared event type`);
+        }
+        if (!(await this.wants(event.workspaceId, event.type)))
+            return null;
+        const parsed = declared.data.safeParse(event.data);
+        if (!parsed.success) {
+            this.deps.logger.error({ type: event.type, issues: parsed.error.issues.slice(0, 3) }, 'webhooks: an emit carried data outside its declared schema — skipped');
+            return null;
+        }
+        const subjects = {};
+        for (const key of declared.subjects) {
+            const value = event.subjects?.[key];
+            if (value === undefined) {
+                this.deps.logger.error({ type: event.type, subject: key }, 'webhooks: an emit left out a declared subject — skipped');
+                return null;
+            }
+            const declaredSubject = this.deps.catalog.subjects[key];
+            if (declaredSubject?.list === true) {
+                subjects[key] = typeof value === 'string' ? [value] : [...value];
+            }
+            else if (typeof value === 'string') {
+                subjects[key] = value;
+            }
+            else {
+                this.deps.logger.error({ type: event.type, subject: key }, 'webhooks: an emit gave a list for a subject declared as one id — skipped');
+                return null;
+            }
+        }
+        const boss = this.requireBoss();
+        const job = {
+            id: randomUUID(),
+            type: event.type,
+            occurredAt: this.now().toISOString(),
+            workspaceId: event.workspaceId,
+            subjects,
+            data: parsed.data
+        };
+        // The fan-out's own retries are short: it only reads and inserts locally.
+        const sent = await sendInTx(boss, conn, WEBHOOK_EVENTS_QUEUE, job, {
+            retryLimit: 5,
+            retryDelay: 5,
+            retryBackoff: true
+        });
+        if (sent === null)
+            throw new Error(`webhooks: the queue ${WEBHOOK_EVENTS_QUEUE} does not exist`);
+        return job.id;
+    }
+    /** Does any enabled endpoint of the workspace subscribe to this type? Cached per replica. */
+    async wants(workspaceId, type) {
+        const cached = this.listeners.get(workspaceId);
+        const patterns = cached && this.now().getTime() - cached.at < this.dials.listenerTtlMs
+            ? cached.patterns
+            : await this.loadListeners(workspaceId);
+        return patterns.some((p) => endpointWants({ enabled: true, events: p, filters: {} }, type, {}));
+    }
+    async loadListeners(workspaceId) {
+        const rows = await this.deps.db
+            .select({ events: webhookEndpoints.events })
+            .from(webhookEndpoints)
+            .where(and(eq(webhookEndpoints.workspaceId, workspaceId), eq(webhookEndpoints.enabled, true)));
+        const patterns = rows.map((r) => r.events);
+        this.listeners.set(workspaceId, { at: this.now().getTime(), patterns });
+        return patterns;
+    }
+    /** Forget the cached answer of a workspace (every write of its endpoints calls it). */
+    invalidateListeners(workspaceId) {
+        this.listeners.delete(workspaceId);
+    }
+    /**
+     * The chassis's own events, bridged from the in-process bus: fired after
+     * the write committed, so never for a rolled-back one, and at most once
+     * per replica that saw it. The events emitted inside a transaction the
+     * chassis owns (projects) do not pass here.
+     */
+    bindBus(events) {
+        const emit = async (event) => {
+            try {
+                await this.emit(this.deps.db, event);
+            }
+            catch (err) {
+                this.deps.logger.error({ err, type: event.type }, 'webhooks: a bus-bridged emit failed');
+            }
+        };
+        events.on('member.joined', ({ workspaceId, userId, role }) => emit({ workspaceId, type: 'member.joined', data: { userId, role } }));
+        events.on('member.removed', ({ workspaceId, userId }) => emit({ workspaceId, type: 'member.removed', data: { userId } }));
+        events.on('workspace.created', ({ workspaceId, ownerUserId }) => emit({ workspaceId, type: 'workspace.created', data: { ownerUserId } }));
+        events.on('apikey.created', ({ workspaceId, apiKeyId }) => emit({ workspaceId, type: 'apikey.created', data: { apiKeyId } }));
+        events.on('apikey.revoked', ({ workspaceId, apiKeyId }) => emit({ workspaceId, type: 'apikey.revoked', data: { apiKeyId } }));
+    }
+    // ── Fan-out ───────────────────────────────────────────────────────────────
+    /** One delivery row and one delivery job per endpoint that wants the event; answers how many. */
+    async fanOut(job) {
+        const endpoints = await this.deps.db
+            .select()
+            .from(webhookEndpoints)
+            .where(and(eq(webhookEndpoints.workspaceId, job.workspaceId), eq(webhookEndpoints.enabled, true)));
+        const matching = endpoints.filter((e) => endpointWants({ enabled: e.enabled, events: e.events, filters: e.filters }, job.type, job.subjects));
+        if (matching.length === 0)
+            return 0;
+        const payload = this.envelopeOf(job);
+        await this.deps.db.transaction(async (tx) => {
+            for (const endpoint of matching) {
+                const [row] = await tx
+                    .insert(webhookDeliveries)
+                    .values({
+                    endpointId: endpoint.id,
+                    workspaceId: job.workspaceId,
+                    eventId: job.id,
+                    eventType: job.type,
+                    payload,
+                    status: 'pending'
+                })
+                    .returning({ id: webhookDeliveries.id });
+                await this.queueDelivery(tx, row.id);
+            }
+        });
+        return matching.length;
+    }
+    /** The envelope as the receiver reads it (without `attempt`, stamped at each send). */
+    envelopeOf(job) {
+        const subjects = {};
+        for (const [key, value] of Object.entries(job.subjects)) {
+            const field = this.deps.catalog.subjects[key]?.field ?? key;
+            subjects[field] = value;
+        }
+        return {
+            id: job.id,
+            type: job.type,
+            occurredAt: job.occurredAt,
+            workspaceId: job.workspaceId,
+            subjects,
+            data: job.data
+        };
+    }
+    async queueDelivery(conn, deliveryId, startAfterSeconds = 0) {
+        const sent = await sendInTx(this.requireBoss(), conn, WEBHOOK_DELIVERIES_QUEUE, { deliveryId }, {
+            retryLimit: this.dials.retryLimit,
+            retryDelay: this.dials.retryDelaySeconds,
+            retryBackoff: true,
+            ...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {})
+        });
+        if (sent === null)
+            throw new Error(`webhooks: the queue ${WEBHOOK_DELIVERIES_QUEUE} does not exist`);
+    }
+    // ── Delivery ──────────────────────────────────────────────────────────────
+    /**
+     * One attempt of a delivery. `retryable` (the queue's worker): a failure
+     * throws `WebhookDeliveryFailed` so pg-boss retries. Not retryable (the
+     * test ping, run inline): a failure marks the row `failed` and answers it.
+     * Answers the row as it is after the attempt.
+     */
+    async attempt(deliveryId, opts) {
+        const [delivery] = await this.deps.db
+            .select()
+            .from(webhookDeliveries)
+            .where(eq(webhookDeliveries.id, deliveryId))
+            .limit(1);
+        if (!delivery)
+            return null; // the endpoint (and its rows) are gone
+        if (delivery.status === 'delivered')
+            return delivery;
+        const [endpoint] = await this.deps.db
+            .select()
+            .from(webhookEndpoints)
+            .where(eq(webhookEndpoints.id, delivery.endpointId))
+            .limit(1);
+        const now = this.now();
+        if (!endpoint || !endpoint.enabled) {
+            this.deliveries.inc({ outcome: 'refused' });
+            return this.finish(delivery, {
+                status: 'failed',
+                lastError: 'endpoint_disabled',
+                lastStatusCode: null,
+                now
+            });
+        }
+        const attempt = delivery.attempt + 1;
+        const body = JSON.stringify({ ...delivery.payload, attempt });
+        // The secret is opened INSIDE the attempt's own handling: a row whose
+        // secret cannot be opened (a rotated auth secret, a damaged row) ends the
+        // delivery as failed `secret_unreadable`, visible on the page, never a
+        // throw that loops through the retries with nothing recorded (verifier
+        // round 1, MN3).
+        let headers;
+        try {
+            const secrets = [this.box.open(endpoint.secret)];
+            if (endpoint.previousSecret &&
+                endpoint.previousSecretExpiresAt &&
+                endpoint.previousSecretExpiresAt > now) {
+                secrets.push(this.box.open(endpoint.previousSecret));
+            }
+            headers = {
+                ...webhookHeaders(secrets, delivery.eventId, Math.floor(now.getTime() / 1000), body),
+                'user-agent': `${this.deps.identity.slug}-webhooks/${this.deps.version}`
+            };
+        }
+        catch (e) {
+            this.deliveries.inc({ outcome: 'refused' });
+            this.deps.logger.error({ err: e, endpointId: endpoint.id, deliveryId: delivery.id }, 'webhooks: the endpoint’s secret cannot be opened — the delivery is failed and the endpoint switched off');
+            // An endpoint whose secret never opens would fail every event for ever
+            // (verifier round 2, N-R2-2): it is switched off here, with its reason.
+            await this.deps.db
+                .update(webhookEndpoints)
+                .set({ enabled: false, disabledAt: now, disabledReason: 'secret_unreadable', updatedAt: now })
+                .where(and(eq(webhookEndpoints.id, endpoint.id), eq(webhookEndpoints.enabled, true)));
+            this.invalidateListeners(endpoint.workspaceId);
+            return this.finish(delivery, {
+                status: 'failed',
+                attempt,
+                lastError: 'secret_unreadable',
+                lastStatusCode: null,
+                now
+            });
+        }
+        if (opts.retryable && (this.inFlight.get(endpoint.id) ?? 0) >= MAX_IN_FLIGHT_PER_ENDPOINT) {
+            throw new WebhookDeliveryDeferred(delivery.id);
+        }
+        this.inFlight.set(endpoint.id, (this.inFlight.get(endpoint.id) ?? 0) + 1);
+        let statusCode = null;
+        let error = null;
+        try {
+            const res = await safePost(endpoint.url, headers, body, this.deps.policy, {
+                timeoutMs: this.dials.timeoutMs,
+                ...this.deps.fetchOptions
+            });
+            statusCode = res.status;
+            if (res.status < 200 || res.status >= 300)
+                error = `HTTP ${res.status}`;
+        }
+        catch (e) {
+            error =
+                e instanceof WebhookUrlRefusedError
+                    ? `url_refused:${e.code}`
+                    : e.name === 'TimeoutError' || e.name === 'AbortError'
+                        ? 'timeout'
+                        : String(e.message ?? e).slice(0, MAX_ERROR_TEXT);
+        }
+        finally {
+            const left = (this.inFlight.get(endpoint.id) ?? 1) - 1;
+            if (left <= 0)
+                this.inFlight.delete(endpoint.id);
+            else
+                this.inFlight.set(endpoint.id, left);
+        }
+        if (error === null) {
+            this.deliveries.inc({ outcome: 'delivered' });
+            await this.deps.db
+                .update(webhookEndpoints)
+                .set({
+                consecutiveFailures: 0,
+                firstFailureAt: null,
+                lastDeliveryAt: now,
+                lastDeliveryStatus: 'delivered',
+                updatedAt: now
+            })
+                .where(eq(webhookEndpoints.id, endpoint.id));
+            return this.finish(delivery, {
+                status: 'delivered',
+                attempt,
+                lastStatusCode: statusCode,
+                lastError: null,
+                now,
+                deliveredAt: now
+            });
+        }
+        this.deliveries.inc({ outcome: 'failed' });
+        // The streak is counted IN the statement (never read then written):
+        // replicas failing the same endpoint at once each add their one, and the
+        // switch-off below is a second conditional statement only one of them
+        // wins (verifier round 1, MN5).
+        const [counted] = await this.deps.db
+            .update(webhookEndpoints)
+            .set({
+            consecutiveFailures: sql `${webhookEndpoints.consecutiveFailures} + 1`,
+            firstFailureAt: sql `COALESCE(${webhookEndpoints.firstFailureAt}, ${now})`,
+            lastDeliveryAt: now,
+            lastDeliveryStatus: 'failed',
+            updatedAt: now
+        })
+            .where(eq(webhookEndpoints.id, endpoint.id))
+            .returning({
+            streak: webhookEndpoints.consecutiveFailures,
+            firstFailureAt: webhookEndpoints.firstFailureAt
+        });
+        const streak = counted?.streak ?? endpoint.consecutiveFailures + 1;
+        const firstFailureAt = counted?.firstFailureAt ?? now;
+        let disable = streak >= this.dials.disableAfterFailures &&
+            now.getTime() - firstFailureAt.getTime() >= this.dials.disableAfterMs;
+        if (disable) {
+            const [won] = await this.deps.db
+                .update(webhookEndpoints)
+                .set({ enabled: false, disabledAt: now, disabledReason: 'delivery_failures', updatedAt: now })
+                .where(and(eq(webhookEndpoints.id, endpoint.id), eq(webhookEndpoints.enabled, true)))
+                .returning({ id: webhookEndpoints.id });
+            disable = won !== undefined;
+        }
+        if (disable) {
+            this.invalidateListeners(endpoint.workspaceId);
+            this.disabled.inc();
+            this.deps.logger.error({
+                endpointId: endpoint.id,
+                workspaceId: endpoint.workspaceId,
+                streak,
+                since: firstFailureAt.toISOString()
+            }, 'webhooks: an endpoint was switched off after a failure streak (webhook_endpoints_disabled_total)');
+            await this.mailDisabled(endpoint, streak, firstFailureAt).catch((err) => this.deps.logger.error({ err, endpointId: endpoint.id }, 'webhooks: the auto-disable mail failed'));
+        }
+        const nextAttemptAt = opts.retryable && attempt <= this.dials.retryLimit
+            ? new Date(now.getTime() + this.dials.retryDelaySeconds * 2 ** (attempt - 1) * 1000)
+            : null;
+        const row = await this.finish(delivery, {
+            status: opts.retryable ? 'pending' : 'failed',
+            attempt,
+            lastStatusCode: statusCode,
+            lastError: error,
+            nextAttemptAt,
+            now
+        });
+        if (opts.retryable)
+            throw new WebhookDeliveryFailed(delivery.id, error);
+        return row;
+    }
+    async finish(delivery, patch) {
+        const [row] = await this.deps.db
+            .update(webhookDeliveries)
+            .set({
+            status: patch.status,
+            attempt: patch.attempt ?? delivery.attempt,
+            lastStatusCode: patch.lastStatusCode,
+            lastError: patch.lastError,
+            nextAttemptAt: patch.nextAttemptAt ?? null,
+            deliveredAt: patch.deliveredAt ?? null,
+            updatedAt: patch.now
+        })
+            .where(eq(webhookDeliveries.id, delivery.id))
+            .returning();
+        return row;
+    }
+    /** Put a delivery back in the queue after `inSeconds` (a deferred attempt). */
+    async requeue(deliveryId, inSeconds) {
+        await this.queueDelivery(this.deps.db, deliveryId, inSeconds);
+    }
+    /**
+     * The held queue's work: mark the row held and queue it again after the
+     * hold, in ONE transaction, so a re-send that fails leaves the row pending
+     * and the held job's own retry finds it again (the chassis suite's round 1:
+     * a row marked held outside the transaction was never re-driven once its
+     * send had failed). A row already delivered, failed or gone is left alone.
+     */
+    async hold(deliveryId) {
+        const now = this.now();
+        const row = await this.deps.db.transaction(async (tx) => {
+            const [updated] = await tx
+                .update(webhookDeliveries)
+                .set({
+                status: 'held',
+                nextAttemptAt: new Date(now.getTime() + this.dials.heldDelaySeconds * 1000),
+                updatedAt: now
+            })
+                .where(and(eq(webhookDeliveries.id, deliveryId), eq(webhookDeliveries.status, 'pending')))
+                .returning({ id: webhookDeliveries.id, endpointId: webhookDeliveries.endpointId });
+            if (!updated)
+                return null;
+            await this.queueDelivery(tx, deliveryId, this.dials.heldDelaySeconds);
+            return updated;
+        });
+        if (!row)
+            return;
+        // Counted and logged once the hold committed: a rolled-back hold is not one.
+        this.held.inc();
+        this.deps.logger.error({ deliveryId, endpointId: row.endpointId, retryAfterSeconds: this.dials.heldDelaySeconds }, 'webhooks: a delivery exhausted its retries — held, re-driven after the hold (webhook_deliveries_held_total)');
+    }
+    async mailDisabled(endpoint, streak, since) {
+        if (!endpoint.createdBy || !this.deps.email.delivers)
+            return;
+        // Only a person still an ACTIVE member of the workspace is told: a
+        // creator who left takes no workspace name and no link with them
+        // (verifier round 1, MN4).
+        const [creator] = await this.deps.db
+            .select({ email: userTable.email, name: userTable.name })
+            .from(userTable)
+            .innerJoin(workspaceMembers, and(eq(workspaceMembers.userId, userTable.id), eq(workspaceMembers.workspaceId, endpoint.workspaceId), eq(workspaceMembers.isActive, true)))
+            .where(eq(userTable.id, endpoint.createdBy))
+            .limit(1);
+        if (!creator)
+            return;
+        const [workspace] = await this.deps.db
+            .select({ name: workspaces.name })
+            .from(workspaces)
+            .where(eq(workspaces.id, endpoint.workspaceId))
+            .limit(1);
+        const mail = buildWebhookDisabledEmail({
+            brand: this.deps.brand,
+            recipientName: creator.name,
+            url: endpoint.url,
+            description: endpoint.description,
+            workspaceName: workspace?.name ?? 'your workspace',
+            consecutiveFailures: streak,
+            firstFailureAt: since,
+            manageUrl: `${this.deps.publicBaseUrl}/webhooks?endpoint=${endpoint.id}`
+        });
+        await this.deps.email.send({ to: creator.email, ...mail });
+    }
+    /** The nightly purge: deliveries older than `retentionDays`, in bounded batches. */
+    async purge(retentionDays) {
+        if (retentionDays <= 0)
+            return 0;
+        const BATCH = 5000;
+        let total = 0;
+        for (let i = 0; i < 10_000; i++) {
+            const res = await this.deps.db.execute(sql `
+        DELETE FROM webhook_deliveries
+        WHERE id IN (
+          SELECT id FROM webhook_deliveries
+          WHERE created_at < now() - make_interval(days => ${retentionDays})
+          ORDER BY created_at
+          LIMIT ${BATCH}
+        )
+      `);
+            const deleted = res.rowCount ?? 0;
+            total += deleted;
+            if (deleted < BATCH)
+                break;
+        }
+        return total;
+    }
+    // ── Endpoints ─────────────────────────────────────────────────────────────
+    /** The URL rule at registration (the address itself is judged at every delivery). */
+    checkUrl(url) {
+        try {
+            checkWebhookUrl(url, this.deps.policy);
+        }
+        catch (e) {
+            if (e instanceof WebhookUrlRefusedError)
+                throw new WebhookRefusedError('url_refused', e.message);
+            throw e;
+        }
+    }
+    /**
+     * The subscription is checked against the catalog at registration
+     * (verifier round 1, MN1): an exact type must be declared, a family must
+     * have at least one declared type, `*` is always right; every filter key
+     * must be a declared subject. A typo would otherwise subscribe to nothing,
+     * silently.
+     */
+    checkSubscription(events, filters) {
+        const known = this.deps.catalog.events;
+        const families = new Set([...known.keys()].map((t) => t.slice(0, t.indexOf('.'))));
+        for (const pattern of events ?? []) {
+            if (pattern === '*')
+                continue;
+            if (pattern.endsWith('.*')) {
+                if (!families.has(pattern.slice(0, -2))) {
+                    throw new WebhookRefusedError('unknown_event', `"${pattern}" matches no event family of this instance`);
+                }
+            }
+            else if (!known.has(pattern)) {
+                throw new WebhookRefusedError('unknown_event', `"${pattern}" is not an event type of this instance`);
+            }
+        }
+        for (const key of Object.keys(filters ?? {})) {
+            if (!(key in this.deps.catalog.subjects)) {
+                throw new WebhookRefusedError('unknown_subject', `"${key}" is not a subject of this instance`);
+            }
+        }
+    }
+    async create(workspaceId, createdBy, input) {
+        this.checkUrl(input.url);
+        this.checkSubscription(input.events, input.filters);
+        const secret = generateWebhookSecret();
+        const [row] = await this.deps.db
+            .insert(webhookEndpoints)
+            .values({
+            workspaceId,
+            createdBy,
+            url: input.url,
+            description: input.description ?? '',
+            events: [...input.events],
+            filters: normalizeFilters(input.filters),
+            enabled: input.enabled ?? true,
+            secret: this.box.seal(secret)
+        })
+            .returning();
+        this.invalidateListeners(workspaceId);
+        return { endpoint: endpointToWire(row), secret };
+    }
+    async get(workspaceId, id) {
+        const row = await this.row(workspaceId, id);
+        return row ? endpointToWire(row) : null;
+    }
+    async list(workspaceId, cursor, limit) {
+        const after = decodeKeysetCursor(cursor);
+        const rows = await this.deps.db
+            .select({ row: webhookEndpoints, createdAtText: createdAtText(webhookEndpoints.createdAt) })
+            .from(webhookEndpoints)
+            .where(and(eq(webhookEndpoints.workspaceId, workspaceId), ...(after
+            ? [
+                keysetBeforeValue({
+                    id: webhookEndpoints.id,
+                    createdAt: webhookEndpoints.createdAt,
+                    cursor: after
+                })
+            ]
+            : [])))
+            .orderBy(desc(webhookEndpoints.createdAt), desc(webhookEndpoints.id))
+            .limit(limit + 1);
+        const page = rows.slice(0, limit);
+        const last = rows.length > limit ? page[page.length - 1] : undefined;
+        return {
+            webhooks: page.map((r) => endpointToWire(r.row)),
+            nextCursor: last ? encodeKeysetCursor({ id: last.row.id, createdAtText: last.createdAtText }) : null
+        };
+    }
+    /** Endpoints of the workspace, for the plan limit's count hook. */
+    async count(workspaceId) {
+        const [row] = await this.deps.db
+            .select({ n: sql `count(*)::int` })
+            .from(webhookEndpoints)
+            .where(eq(webhookEndpoints.workspaceId, workspaceId));
+        return Number(row?.n ?? 0);
+    }
+    async update(workspaceId, id, patch) {
+        if (patch.url !== undefined)
+            this.checkUrl(patch.url);
+        this.checkSubscription(patch.events, patch.filters);
+        const now = this.now();
+        const reEnabled = patch.enabled === true;
+        const [row] = await this.deps.db
+            .update(webhookEndpoints)
+            .set({
+            ...(patch.url !== undefined ? { url: patch.url } : {}),
+            ...(patch.description !== undefined ? { description: patch.description } : {}),
+            ...(patch.events !== undefined ? { events: [...patch.events] } : {}),
+            ...(patch.filters !== undefined ? { filters: normalizeFilters(patch.filters) } : {}),
+            ...(patch.enabled !== undefined ? { enabled: patch.enabled } : {}),
+            // Switching an endpoint back on clears the auto-disable and its streak.
+            ...(reEnabled
+                ? { disabledAt: null, disabledReason: null, consecutiveFailures: 0, firstFailureAt: null }
+                : {}),
+            updatedAt: now
+        })
+            .where(and(eq(webhookEndpoints.id, id), eq(webhookEndpoints.workspaceId, workspaceId)))
+            .returning();
+        this.invalidateListeners(workspaceId);
+        return row ? endpointToWire(row) : null;
+    }
+    async remove(workspaceId, id) {
+        const [row] = await this.deps.db
+            .delete(webhookEndpoints)
+            .where(and(eq(webhookEndpoints.id, id), eq(webhookEndpoints.workspaceId, workspaceId)))
+            .returning();
+        this.invalidateListeners(workspaceId);
+        return row ? endpointToWire(row) : null;
+    }
+    async rotateSecret(workspaceId, id) {
+        const current = await this.row(workspaceId, id);
+        if (!current)
+            return null;
+        const now = this.now();
+        const secret = generateWebhookSecret();
+        const previousSecretExpiresAt = new Date(now.getTime() + this.dials.previousSecretOverlapMs);
+        const [row] = await this.deps.db
+            .update(webhookEndpoints)
+            .set({
+            secret: this.box.seal(secret),
+            previousSecret: current.secret,
+            previousSecretExpiresAt,
+            updatedAt: now
+        })
+            .where(eq(webhookEndpoints.id, id))
+            .returning();
+        return {
+            endpoint: endpointToWire(row),
+            secret,
+            previousSecretExpiresAt: previousSecretExpiresAt.toISOString()
+        };
+    }
+    /** A signed `webhook.ping`, attempted inline, once; the row says how it went. */
+    async test(workspaceId, id, sentBy) {
+        const endpoint = await this.row(workspaceId, id);
+        if (!endpoint)
+            return null;
+        const job = {
+            id: randomUUID(),
+            type: WEBHOOK_PING_EVENT,
+            occurredAt: this.now().toISOString(),
+            workspaceId,
+            subjects: {},
+            data: { endpointId: id, sentBy }
+        };
+        const [row] = await this.deps.db
+            .insert(webhookDeliveries)
+            .values({
+            endpointId: id,
+            workspaceId,
+            eventId: job.id,
+            eventType: job.type,
+            payload: this.envelopeOf(job),
+            status: 'pending'
+        })
+            .returning({ id: webhookDeliveries.id });
+        const result = await this.attempt(row.id, { retryable: false });
+        return result ? deliveryToWire(result) : null;
+    }
+    async listDeliveries(workspaceId, id, query) {
+        const endpoint = await this.row(workspaceId, id);
+        if (!endpoint)
+            return null;
+        const after = decodeKeysetCursor(query.cursor);
+        const rows = await this.deps.db
+            .select({ row: webhookDeliveries, createdAtText: createdAtText(webhookDeliveries.createdAt) })
+            .from(webhookDeliveries)
+            .where(and(eq(webhookDeliveries.endpointId, id), ...(query.status ? [eq(webhookDeliveries.status, query.status)] : []), ...(after
+            ? [
+                keysetBeforeValue({
+                    id: webhookDeliveries.id,
+                    createdAt: webhookDeliveries.createdAt,
+                    cursor: after
+                })
+            ]
+            : [])))
+            .orderBy(desc(webhookDeliveries.createdAt), desc(webhookDeliveries.id))
+            .limit(query.limit + 1);
+        const page = rows.slice(0, query.limit);
+        const last = rows.length > query.limit ? page[page.length - 1] : undefined;
+        return {
+            deliveries: page.map((r) => deliveryToWire(r.row)),
+            nextCursor: last ? encodeKeysetCursor({ id: last.row.id, createdAtText: last.createdAtText }) : null
+        };
+    }
+    async getDelivery(workspaceId, id, deliveryId) {
+        const row = await this.deliveryRow(workspaceId, id, deliveryId);
+        return row ? deliveryToWire(row) : null;
+    }
+    /** The same event queued again for the endpoint, as a new row (the event id unchanged: the receiver's dedupe key). */
+    async redeliver(workspaceId, id, deliveryId) {
+        const source = await this.deliveryRow(workspaceId, id, deliveryId);
+        if (!source)
+            return null;
+        const row = await this.deps.db.transaction(async (tx) => {
+            const [inserted] = await tx
+                .insert(webhookDeliveries)
+                .values({
+                endpointId: source.endpointId,
+                workspaceId: source.workspaceId,
+                eventId: source.eventId,
+                eventType: source.eventType,
+                payload: source.payload,
+                status: 'pending'
+            })
+                .returning();
+            await this.queueDelivery(tx, inserted.id);
+            return inserted;
+        });
+        return deliveryToWire(row);
+    }
+    async row(workspaceId, id) {
+        const [row] = await this.deps.db
+            .select()
+            .from(webhookEndpoints)
+            .where(and(eq(webhookEndpoints.id, id), eq(webhookEndpoints.workspaceId, workspaceId)))
+            .limit(1);
+        return row;
+    }
+    async deliveryRow(workspaceId, id, deliveryId) {
+        const [row] = await this.deps.db
+            .select()
+            .from(webhookDeliveries)
+            .where(and(eq(webhookDeliveries.id, deliveryId), eq(webhookDeliveries.endpointId, id), eq(webhookDeliveries.workspaceId, workspaceId)))
+            .limit(1);
+        return row;
+    }
+    requireBoss() {
+        if (!this.boss)
+            throw new Error('webhooks: the job runtime is not attached yet (boot order)');
+        return this.boss;
+    }
+}
+/** Filters as stored: every key kept, its ids deduplicated; an empty object when none. */
+function normalizeFilters(filters) {
+    const out = {};
+    for (const [key, ids] of Object.entries(filters ?? {}))
+        out[key] = [...new Set(ids)];
+    return out;
+}
+export function endpointToWire(row) {
+    return {
+        id: row.id,
+        url: row.url,
+        description: row.description,
+        events: row.events,
+        filters: row.filters,
+        enabled: row.enabled,
+        createdBy: row.createdBy,
+        createdAt: row.createdAt.toISOString(),
+        updatedAt: row.updatedAt.toISOString(),
+        disabledAt: row.disabledAt?.toISOString() ?? null,
+        disabledReason: row.disabledReason,
+        consecutiveFailures: row.consecutiveFailures,
+        lastDeliveryAt: row.lastDeliveryAt?.toISOString() ?? null,
+        lastDeliveryStatus: row.lastDeliveryStatus,
+        previousSecretExpiresAt: row.previousSecretExpiresAt?.toISOString() ?? null
+    };
+}
+export function deliveryToWire(row) {
+    return {
+        id: row.id,
+        endpointId: row.endpointId,
+        eventId: row.eventId,
+        eventType: row.eventType,
+        status: row.status,
+        attempt: row.attempt,
+        lastStatusCode: row.lastStatusCode,
+        lastError: row.lastError,
+        nextAttemptAt: row.nextAttemptAt?.toISOString() ?? null,
+        deliveredAt: row.deliveredAt?.toISOString() ?? null,
+        createdAt: row.createdAt.toISOString(),
+        updatedAt: row.updatedAt.toISOString(),
+        payload: row.payload
+    };
+}
+//# sourceMappingURL=service.js.map