@objectstack/plugin-webhooks 5.1.0 → 6.0.0

package/src/auto-enqueuer.ts

@@ -0,0 +1,335 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+import type { IDataEngine, IRealtimeService, RealtimeEventPayload } from '@objectstack/spec/contracts';
+import type { IWebhookOutbox } from './outbox.js';
+
+/**
+ * Optional logger interface (subset of console / kernel logger).
+ */
+interface OptionalLogger {
+    info?(msg: string, meta?: unknown): void;
+    warn?(msg: string, meta?: unknown): void;
+    debug?(msg: string, meta?: unknown): void;
+    error?(msg: string, err?: unknown, meta?: unknown): void;
+}
+
+/**
+ * Per-row subscription cached in memory. Mirrors a subset of the
+ * `sys_webhook` object — only what the auto-enqueuer needs to match an
+ * event and build an `EnqueueInput`.
+ */
+interface CachedSubscription {
+    id: string;
+    name: string;
+    objectName: string | undefined; // empty = matches all objects (manual-only is filtered out earlier)
+    triggers: Set<'create' | 'update' | 'delete' | 'undelete'>;
+    url: string;
+    method?: string;
+    headers?: Record<string, string>;
+    secret?: string;
+    timeoutMs?: number;
+}
+
+export interface AutoEnqueuerOptions {
+    /**
+     * Object name holding webhook subscriptions. Defaults to `sys_webhook`,
+     * the platform-objects schema authored in apps.
+     */
+    subscriptionsObject?: string;
+
+    /**
+     * Periodic full-cache refresh interval (ms). Belt-and-braces in case
+     * the subscription-change event is missed. Default 60s.
+     */
+    refreshIntervalMs?: number;
+
+    logger?: OptionalLogger;
+}
+
+/**
+ * Bridge between `IRealtimeService` (`data.record.*` events emitted by
+ * the engine) and `IWebhookOutbox` (durable delivery rows the dispatcher
+ * picks up).
+ *
+ * ## Why a separate class
+ * Keeps `WebhookOutboxPlugin` lean: the plugin wires services, this
+ * class owns the runtime fan-out logic + subscription cache.
+ *
+ * ## Hot path
+ * Every `engine.insert/update/delete` fires a `data.record.*` event.
+ * The handler:
+ *   1. Looks up matching subscriptions in an in-memory `Map<object, sub[]>`
+ *      — O(1) per event, no DB hit on the write path.
+ *   2. Calls `outbox.enqueue()` fire-and-forget for each match. The
+ *      enqueue itself is a single INSERT, which runs *after* the user's
+ *      request has already returned.
+ *
+ * Net cost on the write path: one synchronous Map lookup (~microseconds).
+ *
+ * ## Cache freshness
+ * The cache is rebuilt:
+ *   1. Once on `start()`.
+ *   2. On every `data.record.{created,updated,deleted}` event whose
+ *      object is `sys_webhook` (self-healing — when a user toggles a
+ *      webhook, the handler refreshes the cache before returning).
+ *   3. Periodically (default 60s) as belt-and-braces.
+ *
+ * For multi-node clusters this is *eventually consistent* — node B may
+ * not see node A's edit for up to one cycle. That's acceptable for
+ * webhook configuration changes (humans don't expect millisecond
+ * propagation) and matches Hasura's behaviour.
+ *
+ * ## Determinism
+ * `eventId` is computed from `${object}:${recordId}:${type}:${timestamp}`
+ * so the outbox dedup index catches duplicates that could arise from
+ * upstream replay or buggy producers — and is stable across nodes.
+ */
+export class AutoEnqueuer {
+    private readonly subscriptions = new Map<string, CachedSubscription[]>();
+    private readonly subscriptionsObject: string;
+    private readonly refreshIntervalMs: number;
+    private readonly logger: OptionalLogger;
+    private subId: string | undefined;
+    private subIdSelfHeal: string | undefined;
+    private refreshTimer: ReturnType<typeof setInterval> | undefined;
+    private running = false;
+    private refreshing: Promise<void> | undefined;
+
+    constructor(
+        private readonly engine: IDataEngine,
+        private readonly realtime: IRealtimeService,
+        private readonly outbox: IWebhookOutbox,
+        opts: AutoEnqueuerOptions = {},
+    ) {
+        this.subscriptionsObject = opts.subscriptionsObject ?? 'sys_webhook';
+        this.refreshIntervalMs = opts.refreshIntervalMs ?? 60_000;
+        this.logger = opts.logger ?? {};
+    }
+
+    /**
+     * Load the subscription cache and start listening for events.
+     */
+    async start(): Promise<void> {
+        if (this.running) return;
+        this.running = true;
+
+        await this.refresh();
+
+        // Main subscription: every data event → match → enqueue.
+        this.subId = await this.realtime.subscribe(
+            'webhook-auto-enqueuer',
+            (event) => this.handleEvent(event),
+        );
+
+        // Self-healing: any change to sys_webhook refreshes the cache.
+        this.subIdSelfHeal = await this.realtime.subscribe(
+            'webhook-auto-enqueuer-self-heal',
+            (event) => this.handleSelfHealEvent(event),
+            { object: this.subscriptionsObject },
+        );
+
+        if (this.refreshIntervalMs > 0) {
+            this.refreshTimer = setInterval(() => {
+                this.refresh().catch((err) =>
+                    this.logger.warn?.('[webhook-auto-enqueuer] periodic refresh failed', err),
+                );
+            }, this.refreshIntervalMs);
+            // Don't keep the process alive solely for this timer.
+            this.refreshTimer.unref?.();
+        }
+    }
+
+    async stop(): Promise<void> {
+        if (!this.running) return;
+        this.running = false;
+        if (this.subId) await this.realtime.unsubscribe(this.subId);
+        if (this.subIdSelfHeal) await this.realtime.unsubscribe(this.subIdSelfHeal);
+        if (this.refreshTimer) clearInterval(this.refreshTimer);
+        this.subId = undefined;
+        this.subIdSelfHeal = undefined;
+        this.refreshTimer = undefined;
+    }
+
+    /**
+     * Force-refresh the subscription cache from storage. Concurrent
+     * callers share a single in-flight refresh.
+     */
+    async refresh(): Promise<void> {
+        if (this.refreshing) return this.refreshing;
+        this.refreshing = this.doRefresh().finally(() => {
+            this.refreshing = undefined;
+        });
+        return this.refreshing;
+    }
+
+    private async doRefresh(): Promise<void> {
+        let rows: any[];
+        try {
+            rows = await this.engine.find(this.subscriptionsObject, {
+                where: { active: true },
+            });
+        } catch (err) {
+            this.logger.warn?.(
+                `[webhook-auto-enqueuer] failed to load ${this.subscriptionsObject}`,
+                err,
+            );
+            return;
+        }
+
+        const next = new Map<string, CachedSubscription[]>();
+        for (const row of rows) {
+            const sub = this.parseRow(row);
+            if (!sub) continue;
+            // Empty objectName == "any object" → indexed under '*'.
+            const key = sub.objectName ?? '*';
+            const arr = next.get(key) ?? [];
+            arr.push(sub);
+            next.set(key, arr);
+        }
+
+        this.subscriptions.clear();
+        for (const [k, v] of next) this.subscriptions.set(k, v);
+
+        this.logger.debug?.('[webhook-auto-enqueuer] cache refreshed', {
+            objects: this.subscriptions.size,
+            rows: rows.length,
+        });
+    }
+
+    private parseRow(row: any): CachedSubscription | null {
+        if (!row?.id || !row?.url) return null;
+        const triggersField = (row.triggers ?? '') as string;
+        const triggers = new Set(
+            triggersField
+                .split(',')
+                .map((s: string) => s.trim().toLowerCase())
+                .filter(Boolean) as Array<'create' | 'update' | 'delete' | 'undelete'>,
+        );
+        if (triggers.size === 0) {
+            // Manual-only webhook (no triggers) — skip auto-enqueue.
+            return null;
+        }
+
+        // The "definition_json" field carries advanced config (headers,
+        // secret, timeout); attempt a best-effort parse. Fall back to
+        // top-level fields where present.
+        let defn: Record<string, any> = {};
+        if (typeof row.definition_json === 'string' && row.definition_json.length > 0) {
+            try {
+                defn = JSON.parse(row.definition_json) ?? {};
+            } catch {
+                defn = {};
+            }
+        }
+
+        return {
+            id: row.id as string,
+            name: (row.name as string) ?? row.id,
+            objectName: row.object_name ? String(row.object_name) : undefined,
+            triggers,
+            url: String(row.url),
+            method: row.method ?? defn.method ?? 'POST',
+            headers: defn.headers,
+            secret: defn.secret,
+            timeoutMs: defn.timeoutMs,
+        };
+    }
+
+    /**
+     * Handler for the firehose subscription.
+     *
+     * NOTE: we intentionally `void` the inner enqueue() so the realtime
+     * publisher (and therefore the user's request) is never blocked on
+     * webhook persistence.
+     */
+    private handleEvent(event: RealtimeEventPayload): void {
+        if (!event.type?.startsWith('data.record.')) return;
+        if (!event.object) return;
+        if (event.object === this.subscriptionsObject) return; // self-heal handles its own
+
+        const action = event.type.slice('data.record.'.length) as
+            | 'created' | 'updated' | 'deleted' | 'undeleted' | string;
+        const trigger = mapActionToTrigger(action);
+        if (!trigger) return;
+
+        const subs = [
+            ...(this.subscriptions.get(event.object) ?? []),
+            ...(this.subscriptions.get('*') ?? []),
+        ];
+        if (subs.length === 0) return;
+
+        const payload = event.payload ?? {};
+        const recordId =
+            (payload as any).recordId ??
+            (payload as any).id ??
+            (payload as any).after?.id ??
+            (payload as any).before?.id ??
+            'unknown';
+
+        // Deterministic eventId — same input on any node → same id.
+        // Includes timestamp so two distinct updates to the same record
+        // don't accidentally dedup.
+        const eventId = `${event.object}:${recordId}:${action}:${event.timestamp}`;
+
+        for (const sub of subs) {
+            if (!sub.triggers.has(trigger)) continue;
+
+            // Fire-and-forget — never await on the hot path.
+            void this.outbox
+                .enqueue({
+                    webhookId: sub.id,
+                    eventId,
+                    eventType: event.type,
+                    url: sub.url,
+                    method: sub.method,
+                    headers: sub.headers,
+                    secret: sub.secret,
+                    timeoutMs: sub.timeoutMs,
+                    payload: {
+                        object: event.object,
+                        recordId,
+                        action,
+                        timestamp: event.timestamp,
+                        ...payload,
+                    },
+                })
+                .catch((err) =>
+                    this.logger.warn?.('[webhook-auto-enqueuer] enqueue failed', {
+                        webhook: sub.name,
+                        eventId,
+                        err: (err as Error)?.message ?? err,
+                    }),
+                );
+        }
+    }
+
+    private handleSelfHealEvent(event: RealtimeEventPayload): void {
+        if (event.object !== this.subscriptionsObject) return;
+        if (!event.type?.startsWith('data.record.')) return;
+        this.refresh().catch((err) =>
+            this.logger.warn?.('[webhook-auto-enqueuer] self-heal refresh failed', err),
+        );
+    }
+
+    /** Test / admin accessor. */
+    snapshot(): ReadonlyMap<string, ReadonlyArray<CachedSubscription>> {
+        return this.subscriptions;
+    }
+}
+
+function mapActionToTrigger(
+    action: string,
+): 'create' | 'update' | 'delete' | 'undelete' | null {
+    switch (action) {
+        case 'created':
+            return 'create';
+        case 'updated':
+            return 'update';
+        case 'deleted':
+            return 'delete';
+        case 'undeleted':
+            return 'undelete';
+        default:
+            return null;
+    }
+}

package/src/dispatcher.test.ts

@@ -0,0 +1,324 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * Cross-node webhook dispatcher contract test.
+ *
+ * Builds two `WebhookDispatcher` instances that share one in-memory outbox
+ * AND one cluster `ILock`/`IPubSub` (simulating two nodes sharing one
+ * Redis/Postgres). Asserts:
+ *
+ *   1. Every enqueued delivery is POSTed *exactly once* (no double-fire).
+ *   2. Work is distributed across both nodes (no starvation).
+ *   3. 5xx responses are retried per the Stripe-style schedule.
+ *   4. 4xx (permanent) responses go straight to `dead`.
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+    ComposedClusterService,
+    MemoryCounter,
+    MemoryKV,
+    MemoryLock,
+    MemoryPubSub,
+} from '@objectstack/service-cluster';
+import type { IClusterService } from '@objectstack/spec/contracts';
+import { WebhookDispatcher } from './dispatcher.js';
+import type { FetchImpl } from './http-sender.js';
+import { MemoryWebhookOutbox } from './memory-outbox.js';
+import { hashPartition } from './partition.js';
+
+interface SharedCluster {
+    nodeA: IClusterService;
+    nodeB: IClusterService;
+}
+
+function makeSharedCluster(): SharedCluster {
+    // ONE lock + pubsub shared by both "nodes" — this is what makes the test
+    // a realistic cross-node simulation.
+    const lock = new MemoryLock();
+    const pubsub = new MemoryPubSub();
+    const kv = new MemoryKV();
+    const counter = new MemoryCounter();
+    return {
+        nodeA: new ComposedClusterService('node-A', 'memory', pubsub, lock, kv, counter),
+        nodeB: new ComposedClusterService('node-B', 'memory', pubsub, lock, kv, counter),
+    };
+}
+
+function makeFetchImpl(opts: {
+    status?: number;
+    log?: { url: string; deliveryId: string }[];
+}): FetchImpl {
+    const status = opts.status ?? 200;
+    return async (url, init) => {
+        opts.log?.push({
+            url,
+            deliveryId: init.headers['X-Objectstack-Delivery'] ?? '',
+        });
+        return {
+            ok: status >= 200 && status < 300,
+            status,
+            async text() {
+                return '';
+            },
+        };
+    };
+}
+
+async function flushTicks(...dispatchers: WebhookDispatcher[]): Promise<void> {
+    // Run several rounds with the nodes ticking *concurrently* so they
+    // genuinely contend for the cluster lock — sequential ticks would let
+    // whichever node ran first drain every partition.
+    for (let round = 0; round < 6; round++) {
+        await Promise.all(dispatchers.map((d) => d.tick()));
+    }
+}
+
+describe('WebhookDispatcher cross-node', () => {
+    it('exactly-once: 50 deliveries across 2 nodes → 50 POSTs total', async () => {
+        const cluster = makeSharedCluster();
+        const outbox = new MemoryWebhookOutbox();
+        const log: { url: string; deliveryId: string }[] = [];
+        const fetchImpl = makeFetchImpl({ status: 200, log });
+
+        const partitionCount = 4;
+        const a = new WebhookDispatcher({
+            nodeId: 'node-A',
+            cluster: cluster.nodeA,
+            outbox,
+            fetchImpl,
+            partitionCount,
+            intervalMs: 1_000_000, // disable timer; we drive with tick()
+        });
+        const b = new WebhookDispatcher({
+            nodeId: 'node-B',
+            cluster: cluster.nodeB,
+            outbox,
+            fetchImpl,
+            partitionCount,
+            intervalMs: 1_000_000,
+        });
+
+        for (let i = 0; i < 50; i++) {
+            await outbox.enqueue({
+                webhookId: `wh-${i % 10}`, // 10 webhooks → spread across 4 partitions
+                eventId: `evt-${i}`,
+                eventType: 'data.record.created',
+                url: `https://example.test/${i}`,
+                payload: { i },
+            });
+        }
+
+        await flushTicks(a, b);
+
+        expect(log).toHaveLength(50);
+        const uniqueIds = new Set(log.map((l) => l.deliveryId));
+        expect(uniqueIds.size).toBe(50);
+
+        const success = await outbox.list({ status: 'success' });
+        expect(success).toHaveLength(50);
+    });
+
+    it('partition affinity: dispatcher only claims rows for partitions it locked', async () => {
+        const cluster = makeSharedCluster();
+        const outbox = new MemoryWebhookOutbox();
+        const partitionCount = 8;
+
+        // For each attempt, record (nodeId, partitionForWebhook).
+        const observed: { nodeId: string; partition: number }[] = [];
+
+        const make = (nodeId: string, c: IClusterService) =>
+            new WebhookDispatcher({
+                nodeId,
+                cluster: c,
+                outbox,
+                fetchImpl: makeFetchImpl({ status: 200 }),
+                partitionCount,
+                intervalMs: 1_000_000,
+                onAttempt: (delivery) => {
+                    observed.push({
+                        nodeId,
+                        partition: hashPartition(delivery.webhookId, partitionCount),
+                    });
+                },
+            });
+
+        const a = make('node-A', cluster.nodeA);
+        const b = make('node-B', cluster.nodeB);
+
+        for (let i = 0; i < 30; i++) {
+            await outbox.enqueue({
+                webhookId: `wh-${i % 5}`,
+                eventId: `evt-${i}`,
+                eventType: 't',
+                url: 'https://example.test/x',
+                payload: { i },
+            });
+        }
+        await flushTicks(a, b);
+
+        expect(observed).toHaveLength(30);
+        // Each row's partition came from hash(webhookId, 8) — only 5 distinct
+        // webhook ids → at most 5 distinct partitions.
+        const partitionsTouched = new Set(observed.map((o) => o.partition));
+        expect(partitionsTouched.size).toBeLessThanOrEqual(5);
+    });
+
+    it('load distribution: both nodes process some rows', async () => {
+        const cluster = makeSharedCluster();
+        const outbox = new MemoryWebhookOutbox();
+        const partitionCount = 8;
+        const counts: Record<string, number> = { 'node-A': 0, 'node-B': 0 };
+
+        const make = (nodeId: string, c: IClusterService) =>
+            new WebhookDispatcher({
+                nodeId,
+                cluster: c,
+                outbox,
+                fetchImpl: makeFetchImpl({ status: 200 }),
+                partitionCount,
+                intervalMs: 1_000_000,
+                onAttempt: () => {
+                    counts[nodeId] += 1;
+                },
+            });
+
+        const a = make('node-A', cluster.nodeA);
+        const b = make('node-B', cluster.nodeB);
+
+        // Lots of distinct webhookIds → spreads work across many partitions.
+        for (let i = 0; i < 200; i++) {
+            await outbox.enqueue({
+                webhookId: `wh-${i}`,
+                eventId: `evt-${i}`,
+                eventType: 't',
+                url: 'https://example.test/x',
+                payload: { i },
+            });
+        }
+        await flushTicks(a, b);
+
+        // Each node should have processed at least one row — proving the
+        // rotation/offset isn't pinning all work to node A.
+        expect(counts['node-A']).toBeGreaterThan(0);
+        expect(counts['node-B']).toBeGreaterThan(0);
+        expect(counts['node-A'] + counts['node-B']).toBe(200);
+    });
+
+    it('5xx is retried: row stays pending with future nextRetryAt', async () => {
+        const cluster = makeSharedCluster();
+        const outbox = new MemoryWebhookOutbox();
+        const a = new WebhookDispatcher({
+            nodeId: 'node-A',
+            cluster: cluster.nodeA,
+            outbox,
+            fetchImpl: makeFetchImpl({ status: 503 }),
+            partitionCount: 1,
+            intervalMs: 1_000_000,
+            rng: () => 0.5,
+        });
+
+        await outbox.enqueue({
+            webhookId: 'wh-x',
+            eventId: 'evt-x',
+            eventType: 't',
+            url: 'https://example.test/fail',
+            payload: {},
+        });
+
+        await a.tick();
+        const rows = await outbox.list();
+        expect(rows[0].status).toBe('pending');
+        expect(rows[0].attempts).toBe(1);
+        expect(rows[0].nextRetryAt).toBeGreaterThan(Date.now());
+    });
+
+    it('4xx is permanent: row moves to dead', async () => {
+        const cluster = makeSharedCluster();
+        const outbox = new MemoryWebhookOutbox();
+        const a = new WebhookDispatcher({
+            nodeId: 'node-A',
+            cluster: cluster.nodeA,
+            outbox,
+            fetchImpl: makeFetchImpl({ status: 404 }),
+            partitionCount: 1,
+            intervalMs: 1_000_000,
+        });
+
+        await outbox.enqueue({
+            webhookId: 'wh-x',
+            eventId: 'evt-x',
+            eventType: 't',
+            url: 'https://example.test/missing',
+            payload: {},
+        });
+
+        await a.tick();
+        const dead = await outbox.list({ status: 'dead' });
+        expect(dead).toHaveLength(1);
+        expect(dead[0].responseCode).toBe(404);
+    });
+
+    it('dedup: identical (eventId, webhookId) enqueues collapse to one row', async () => {
+        const outbox = new MemoryWebhookOutbox();
+        const id1 = await outbox.enqueue({
+            webhookId: 'wh-1',
+            eventId: 'evt-dup',
+            eventType: 't',
+            url: 'https://example.test/',
+            payload: {},
+        });
+        const id2 = await outbox.enqueue({
+            webhookId: 'wh-1',
+            eventId: 'evt-dup',
+            eventType: 't',
+            url: 'https://example.test/',
+            payload: {},
+        });
+        expect(id1).toBe(id2);
+        const rows = await outbox.list();
+        expect(rows).toHaveLength(1);
+    });
+
+    it('lock prevents same partition being claimed twice in a tick', async () => {
+        const cluster = makeSharedCluster();
+        const outbox = new MemoryWebhookOutbox();
+        const log: { url: string; deliveryId: string }[] = [];
+        const fetchImpl = makeFetchImpl({ status: 200, log });
+
+        // Single partition → both nodes contend for the same lock.
+        const a = new WebhookDispatcher({
+            nodeId: 'node-A',
+            cluster: cluster.nodeA,
+            outbox,
+            fetchImpl,
+            partitionCount: 1,
+            intervalMs: 1_000_000,
+        });
+        const b = new WebhookDispatcher({
+            nodeId: 'node-B',
+            cluster: cluster.nodeB,
+            outbox,
+            fetchImpl,
+            partitionCount: 1,
+            intervalMs: 1_000_000,
+        });
+
+        for (let i = 0; i < 5; i++) {
+            await outbox.enqueue({
+                webhookId: 'wh-1',
+                eventId: `evt-${i}`,
+                eventType: 't',
+                url: 'https://example.test/',
+                payload: { i },
+            });
+        }
+
+        // Fire both ticks "simultaneously" — only one should claim the partition.
+        await Promise.all([a.tick(), b.tick()]);
+
+        expect(log).toHaveLength(5);
+        const uniqueIds = new Set(log.map((l) => l.deliveryId));
+        expect(uniqueIds.size).toBe(5);
+    });
+});