@objectstack/plugin-webhooks 5.0.0 → 5.2.0

package/src/dispatcher.ts

@@ -0,0 +1,218 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+import type { IClusterService, LockHandle } from '@objectstack/spec/contracts';
+import type { FetchImpl } from './http-sender.js';
+import { classifyAttempt, sendOnce } from './http-sender.js';
+import type { IWebhookOutbox, WebhookDelivery } from './outbox.js';
+
+/**
+ * Minimal logger surface — kernel's `Logger` is compatible (extra params
+ * accepted). Keeping it permissive avoids a hard dependency on the spec
+ * Logger interface here.
+ */
+export interface DispatcherLogger {
+    warn: (msg: string, meta?: any) => void;
+    info?: (msg: string, meta?: any) => void;
+}
+
+export interface DispatcherOptions {
+    /** Stable id identifying this dispatcher node. */
+    nodeId: string;
+    /** Cluster service providing `lock` (and optional metrics). */
+    cluster: IClusterService;
+    /** Outbox backend. */
+    outbox: IWebhookOutbox;
+    /**
+     * How many partitions to split work across. Each tick the dispatcher
+     * attempts to acquire each partition's lock independently — the node
+     * that wins owns that partition for the duration of the batch.
+     *
+     * Default: 8 (matches webhook-delivery.mdx §4 example).
+     */
+    partitionCount?: number;
+    /** Max rows to claim from each partition per tick. Default 32. */
+    batchSize?: number;
+    /** Tick interval in ms. Default 250. */
+    intervalMs?: number;
+    /** Per-partition lock TTL. Default = 5 × intervalMs. */
+    lockTtlMs?: number;
+    /** Visibility timeout for claimed rows. Default = 2 × lockTtlMs. */
+    claimTtlMs?: number;
+    /** Override `globalThis.fetch` (tests). */
+    fetchImpl?: FetchImpl;
+    /** Hook fired after every attempt — observability hook. */
+    onAttempt?: (delivery: WebhookDelivery, success: boolean) => void;
+    /** RNG override for the retry-jitter schedule (tests). */
+    rng?: () => number;
+    /** Logger callback (optional). */
+    logger?: DispatcherLogger;
+}
+
+/**
+ * Cross-node webhook dispatcher.
+ *
+ * **Design** — each tick the dispatcher iterates over `partitionCount`
+ * logical partitions. For each, it tries to acquire a cluster-scoped lock
+ * (`webhook.dispatcher.partition.{i}`) with a short TTL. If it wins the
+ * lock, it claims up to `batchSize` ready rows whose `hash(webhookId) mod
+ * partitionCount === i`, POSTs them, and acks. The lock is released
+ * immediately after the batch so other nodes can fairly rotate through.
+ *
+ * **Why per-partition locks rather than one global lock?**
+ *
+ *   1. Throughput — N nodes can process N partitions concurrently.
+ *   2. Partition affinity — rows for the same webhook always sort into the
+ *      same partition, preserving in-order delivery per webhook.
+ *   3. Failure isolation — a stuck node only blocks its partition until the
+ *      TTL elapses; other partitions keep moving.
+ *
+ * **At-least-once, not exactly-once.** Receivers MUST be idempotent on the
+ * `X-Objectstack-Delivery` (== row id) header. If the HTTP call succeeds
+ * but the ack write fails, the row reverts to pending after the claim TTL
+ * and will be re-posted.
+ */
+export class WebhookDispatcher {
+    private readonly opts: Required<
+        Omit<DispatcherOptions, 'onAttempt' | 'fetchImpl' | 'rng' | 'logger'>
+    > & Pick<DispatcherOptions, 'onAttempt' | 'fetchImpl' | 'rng' | 'logger'>;
+    private timer: ReturnType<typeof setInterval> | undefined;
+    private running = false;
+    private inflightTick: Promise<void> | undefined;
+
+    constructor(options: DispatcherOptions) {
+        const intervalMs = options.intervalMs ?? 250;
+        const lockTtlMs = options.lockTtlMs ?? intervalMs * 5;
+        this.opts = {
+            nodeId: options.nodeId,
+            cluster: options.cluster,
+            outbox: options.outbox,
+            partitionCount: options.partitionCount ?? 8,
+            batchSize: options.batchSize ?? 32,
+            intervalMs,
+            lockTtlMs,
+            claimTtlMs: options.claimTtlMs ?? lockTtlMs * 2,
+            onAttempt: options.onAttempt,
+            fetchImpl: options.fetchImpl,
+            rng: options.rng,
+            logger: options.logger,
+        };
+    }
+
+    /** Begin the periodic loop. Safe to call once; subsequent calls are no-ops. */
+    start(): void {
+        if (this.running) return;
+        this.running = true;
+        // Fire one tick immediately so single-row tests don't wait the interval.
+        this.scheduleTick();
+        this.timer = setInterval(() => this.scheduleTick(), this.opts.intervalMs);
+    }
+
+    /** Stop the loop and wait for the in-flight tick to drain. */
+    async stop(): Promise<void> {
+        if (!this.running) return;
+        this.running = false;
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = undefined;
+        }
+        if (this.inflightTick) {
+            try {
+                await this.inflightTick;
+            } catch {
+                /* swallow — already logged */
+            }
+        }
+    }
+
+    /**
+     * Run one full tick (all partitions, single attempt each). Exposed for
+     * deterministic tests that want to step the dispatcher manually.
+     */
+    async tick(): Promise<void> {
+        await this.runTick();
+    }
+
+    private scheduleTick(): void {
+        if (this.inflightTick) return; // skip if previous tick still running
+        this.inflightTick = this.runTick()
+            .catch((err) => {
+                this.opts.logger?.warn?.('webhook-dispatcher: tick failed', {
+                    nodeId: this.opts.nodeId,
+                    error: (err as Error)?.message ?? String(err),
+                });
+            })
+            .finally(() => {
+                this.inflightTick = undefined;
+            });
+    }
+
+    private async runTick(): Promise<void> {
+        const partitionCount = this.opts.partitionCount;
+        // Walk partitions in a rotated order per node so contention spreads.
+        const offset = stableNodeOffset(this.opts.nodeId, partitionCount);
+        for (let step = 0; step < partitionCount; step++) {
+            const i = (offset + step) % partitionCount;
+            await this.runPartition(i);
+        }
+    }
+
+    private async runPartition(index: number): Promise<void> {
+        const key = `webhook.dispatcher.partition.${index}`;
+        const handle: LockHandle | null = await this.opts.cluster.lock.acquire(key, {
+            ttlMs: this.opts.lockTtlMs,
+            // waitMs=0 → fail-fast; we'll try this partition again next tick.
+            waitMs: 0,
+        });
+        if (!handle) return;
+
+        try {
+            const claimed = await this.opts.outbox.claim({
+                nodeId: this.opts.nodeId,
+                limit: this.opts.batchSize,
+                partition: { index, count: this.opts.partitionCount },
+                claimTtlMs: this.opts.claimTtlMs,
+            });
+            if (claimed.length === 0) return;
+            // Renew before potentially long HTTP work — and bound batch time.
+            await handle.renew(this.opts.lockTtlMs);
+            for (const row of claimed) {
+                if (!handle.isHeld()) break; // lost the lock — abandon remaining rows
+                await this.processRow(row);
+            }
+        } finally {
+            await handle.release();
+        }
+    }
+
+    private async processRow(row: WebhookDelivery): Promise<void> {
+        const fetchImpl = (this.opts.fetchImpl ?? (globalThis.fetch as unknown as FetchImpl)) as FetchImpl | undefined;
+        if (!fetchImpl) {
+            this.opts.logger?.warn?.('webhook-dispatcher: no fetch impl available', {
+                rowId: row.id,
+            });
+            await this.opts.outbox.ack(row.id, {
+                success: false,
+                error: 'no fetch implementation',
+                durationMs: 0,
+                dead: true,
+            });
+            return;
+        }
+        const outcome = await sendOnce(row, fetchImpl);
+        const result = classifyAttempt(outcome, row.attempts, Date.now(), this.opts.rng);
+        await this.opts.outbox.ack(row.id, result);
+        this.opts.onAttempt?.(row, result.success);
+    }
+}
+
+/**
+ * Spread starting partition per node so a 2-node cluster with 8 partitions
+ * doesn't have both nodes serialise on partition 0 every tick.
+ */
+function stableNodeOffset(nodeId: string, partitionCount: number): number {
+    let h = 0;
+    for (let i = 0; i < nodeId.length; i++) {
+        h = (h * 31 + nodeId.charCodeAt(i)) | 0;
+    }
+    return Math.abs(h) % partitionCount;
+}

package/src/http-sender.ts

@@ -0,0 +1,187 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { createHmac, randomUUID } from 'node:crypto';
+import type { WebhookDelivery, AckResult } from './outbox.js';
+
+/**
+ * Default per-request timeout. Receivers SHOULD respond within ~30s; we
+ * cap aggressively to free dispatcher slots.
+ */
+export const DEFAULT_TIMEOUT_MS = 15_000;
+
+/** Truncate response bodies to keep storage cost predictable. */
+const RESPONSE_BODY_CAP = 16 * 1024;
+
+export type FetchImpl = (
+    input: string,
+    init: {
+        method: string;
+        headers: Record<string, string>;
+        body: string;
+        signal: AbortSignal;
+    },
+) => Promise<{
+    ok: boolean;
+    status: number;
+    text(): Promise<string>;
+}>;
+
+/** Single HTTP attempt classified to an `AckResult` shape (without nextRetryAt). */
+export type AttemptOutcome =
+    | { success: true; httpStatus: number; responseBody?: string; durationMs: number }
+    | {
+          success: false;
+          retriable: boolean;
+          httpStatus?: number;
+          responseBody?: string;
+          error?: string;
+          durationMs: number;
+      };
+
+/**
+ * Send one HTTP attempt for the delivery. Pure (no DB writes) so the
+ * dispatcher owns retry-schedule + ack logic.
+ *
+ *   - 2xx                       → success
+ *   - 4xx (except 408/429)      → permanent failure (retriable = false → goes to `dead`)
+ *   - 408, 429, 5xx, transport  → retriable
+ */
+export async function sendOnce(
+    delivery: WebhookDelivery,
+    fetchImpl: FetchImpl,
+): Promise<AttemptOutcome> {
+    const body =
+        typeof delivery.payload === 'string'
+            ? delivery.payload
+            : JSON.stringify(delivery.payload);
+
+    const headers: Record<string, string> = {
+        'Content-Type': 'application/json',
+        'User-Agent': 'ObjectStack-Webhooks/1.0',
+        'X-Objectstack-Event': delivery.eventType,
+        'X-Objectstack-Delivery': delivery.id,
+        'X-Objectstack-Attempt': String(delivery.attempts + 1),
+        ...(delivery.headers ?? {}),
+    };
+    if (delivery.secret) {
+        const sig = createHmac('sha256', delivery.secret).update(body).digest('hex');
+        headers['X-Objectstack-Signature'] = `sha256=${sig}`;
+    }
+
+    const timeoutMs = delivery.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    const start = Date.now();
+    try {
+        const res = await fetchImpl(delivery.url, {
+            method: delivery.method ?? 'POST',
+            headers,
+            body,
+            signal: controller.signal,
+        });
+        clearTimeout(timer);
+        const responseText = await safeReadBody(res);
+        const durationMs = Date.now() - start;
+        if (res.ok) {
+            return { success: true, httpStatus: res.status, responseBody: responseText, durationMs };
+        }
+        const retriable = res.status === 408 || res.status === 429 || res.status >= 500;
+        return {
+            success: false,
+            retriable,
+            httpStatus: res.status,
+            responseBody: responseText,
+            error: `HTTP ${res.status}`,
+            durationMs,
+        };
+    } catch (err: unknown) {
+        clearTimeout(timer);
+        const durationMs = Date.now() - start;
+        const e = err as { name?: string; message?: string };
+        const error = e?.name === 'AbortError' ? `timeout after ${timeoutMs}ms` : e?.message ?? String(err);
+        return { success: false, retriable: true, error, durationMs };
+    }
+}
+
+async function safeReadBody(res: { text(): Promise<string> }): Promise<string | undefined> {
+    try {
+        const text = await res.text();
+        return text.length > RESPONSE_BODY_CAP ? text.slice(0, RESPONSE_BODY_CAP) : text;
+    } catch {
+        return undefined;
+    }
+}
+
+/**
+ * Stripe-style retry schedule. Returns the next `nextRetryAt` ms (relative
+ * to `now`) given how many attempts have already happened, or `null` if
+ * the row should be moved to `dead`.
+ *
+ *   attempt 1 fails -> retry in ~1s
+ *   attempt 2 fails -> ~10s
+ *   attempt 3 fails -> ~1m
+ *   attempt 4 fails -> ~10m
+ *   attempt 5 fails -> ~1h
+ *   attempt 6 fails -> ~6h
+ *   attempt 7 fails -> ~24h
+ *   attempt 8+ fails -> dead
+ *
+ * Each delay is multiplied by jitter ∈ [0.8, 1.2].
+ */
+export function nextRetryDelayMs(
+    attemptsSoFar: number,
+    rng: () => number = Math.random,
+): number | null {
+    const SCHEDULE = [1_000, 10_000, 60_000, 600_000, 3_600_000, 21_600_000, 86_400_000];
+    if (attemptsSoFar < 1 || attemptsSoFar > SCHEDULE.length) return null;
+    const base = SCHEDULE[attemptsSoFar - 1];
+    const jitter = 0.8 + rng() * 0.4;
+    return Math.floor(base * jitter);
+}
+
+/**
+ * Compose an `AckResult` from an `AttemptOutcome`, applying the retry
+ * schedule on retriable failures.
+ */
+export function classifyAttempt(
+    outcome: AttemptOutcome,
+    attemptsSoFar: number,
+    now: number = Date.now(),
+    rng?: () => number,
+): AckResult {
+    if (outcome.success) return outcome;
+    if (!outcome.retriable) {
+        return {
+            success: false,
+            httpStatus: outcome.httpStatus,
+            responseBody: outcome.responseBody,
+            error: outcome.error,
+            durationMs: outcome.durationMs,
+            dead: true,
+        };
+    }
+    const delay = nextRetryDelayMs(attemptsSoFar + 1, rng);
+    if (delay === null) {
+        return {
+            success: false,
+            httpStatus: outcome.httpStatus,
+            responseBody: outcome.responseBody,
+            error: outcome.error,
+            durationMs: outcome.durationMs,
+            dead: true,
+        };
+    }
+    return {
+        success: false,
+        httpStatus: outcome.httpStatus,
+        responseBody: outcome.responseBody,
+        error: outcome.error,
+        durationMs: outcome.durationMs,
+        nextRetryAt: now + delay,
+    };
+}
+
+/** Generate a fresh delivery id (UUID v4). Exposed for tests. */
+export function newDeliveryId(): string {
+    return randomUUID();
+}

package/src/index.ts

@@ -1,19 +1,55 @@
-// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
 
 /**
  * @objectstack/plugin-webhooks
  *
- * Outbound webhook delivery for `data.record.created`,
- * `data.record.updated`, and `data.record.deleted` events. Subscribes to
- * the realtime service, fans events out to one or more configured HTTP
- * sinks, signs each request with HMAC-SHA256, and retries transient
- * failures with exponential backoff.
+ * Persistent, cluster-aware webhook outbox + dispatcher.
+ *
+ * Implements stages 3–5 of the pipeline in
+ * `content/docs/concepts/webhook-delivery.mdx` (Persist · Dispatch ·
+ * Retry). Stages 1–2 (Event capture · Match) integrate via the
+ * `webhook.outbox.enqueue()` service consumers call after persistence.
+ *
+ * The first real cross-node consumer of `cluster.lock`.
+ *
+ * ## Subpath exports
+ * - `@objectstack/plugin-webhooks/sql` — `SqlWebhookOutbox` (production
+ *   storage; durable rows via ObjectQL / any driver)
+ * - `@objectstack/plugin-webhooks/schema` — `SysWebhookDelivery` object
+ *   schema to register in `defineStack({ objects: [...] })`
+ *
+ * The main entry intentionally ships only the `MemoryWebhookOutbox` so
+ * downstream bundles don't pay for the SQL impl unless they import it.
  */
 
-export { WebhooksPlugin } from './webhooks-plugin.js';
+export {
+    WebhookOutboxPlugin,
+    type WebhookOutboxPluginOptions,
+} from './webhook-outbox-plugin.js';
+
+export { WebhookDispatcher, type DispatcherOptions } from './dispatcher.js';
+export { MemoryWebhookOutbox } from './memory-outbox.js';
+export { AutoEnqueuer, type AutoEnqueuerOptions } from './auto-enqueuer.js';
+export {
+    DeliveryRetentionSweeper,
+    type DeliveryRetentionOptions,
+} from './retention.js';
+export { hashPartition } from './partition.js';
+export {
+    sendOnce,
+    classifyAttempt,
+    nextRetryDelayMs,
+    DEFAULT_TIMEOUT_MS,
+    type AttemptOutcome,
+    type FetchImpl,
+} from './http-sender.js';
 export type {
-  WebhooksPluginOptions,
-  WebhookSink,
-  WebhookDeliveryRecord,
-  WebhookDeliveryStatus,
-} from './webhooks-plugin.js';
+    AckFailure,
+    AckResult,
+    AckSuccess,
+    ClaimOptions,
+    DeliveryStatus,
+    EnqueueInput,
+    IWebhookOutbox,
+    WebhookDelivery,
+} from './outbox.js';

package/src/memory-outbox.ts

@@ -0,0 +1,127 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { randomUUID } from 'node:crypto';
+import type {
+    AckResult,
+    ClaimOptions,
+    EnqueueInput,
+    DeliveryStatus,
+    IWebhookOutbox,
+    WebhookDelivery,
+} from './outbox.js';
+import { hashPartition } from './partition.js';
+
+/**
+ * In-memory `IWebhookOutbox` for tests and single-process development.
+ *
+ * Implements the atomic-claim semantics by running its claim/ack logic
+ * synchronously (single-threaded JS event loop) inside one `Map`. Two
+ * `MemoryWebhookOutbox` instances do NOT share state — for the cross-node
+ * test the *same* instance is passed to both dispatchers (simulating one
+ * shared database).
+ *
+ * A production SQL-backed implementation will live in a sibling file and
+ * use `SELECT ... FOR UPDATE SKIP LOCKED`.
+ */
+export class MemoryWebhookOutbox implements IWebhookOutbox {
+    private readonly rows = new Map<string, WebhookDelivery>();
+    /** Dedup index keyed by `${eventId}::${webhookId}` -> row id. */
+    private readonly dedup = new Map<string, string>();
+
+    async enqueue(input: EnqueueInput): Promise<string> {
+        const dedupKey = `${input.eventId}::${input.webhookId}`;
+        const existing = this.dedup.get(dedupKey);
+        if (existing) return existing;
+
+        const id = randomUUID();
+        const now = Date.now();
+        const row: WebhookDelivery = {
+            id,
+            webhookId: input.webhookId,
+            eventId: input.eventId,
+            eventType: input.eventType,
+            url: input.url,
+            method: input.method ?? 'POST',
+            headers: input.headers,
+            secret: input.secret,
+            timeoutMs: input.timeoutMs,
+            payload: input.payload,
+            status: 'pending',
+            attempts: 0,
+            createdAt: now,
+            updatedAt: now,
+        };
+        this.rows.set(id, row);
+        this.dedup.set(dedupKey, id);
+        return id;
+    }
+
+    async claim(opts: ClaimOptions): Promise<WebhookDelivery[]> {
+        const now = opts.now ?? Date.now();
+        const claimed: WebhookDelivery[] = [];
+
+        // First pass: reap expired in_flight rows (visibility timeout).
+        for (const row of this.rows.values()) {
+            if (
+                row.status === 'in_flight' &&
+                row.claimedAt !== undefined &&
+                now - row.claimedAt > opts.claimTtlMs
+            ) {
+                row.status = 'pending';
+                row.claimedBy = undefined;
+                row.claimedAt = undefined;
+                row.updatedAt = now;
+            }
+        }
+
+        for (const row of this.rows.values()) {
+            if (claimed.length >= opts.limit) break;
+            if (row.status !== 'pending') continue;
+            if (row.nextRetryAt !== undefined && row.nextRetryAt > now) continue;
+            if (opts.partition) {
+                const p = hashPartition(row.webhookId, opts.partition.count);
+                if (p !== opts.partition.index) continue;
+            }
+            row.status = 'in_flight';
+            row.claimedBy = opts.nodeId;
+            row.claimedAt = now;
+            row.updatedAt = now;
+            claimed.push({ ...row });
+        }
+        return claimed;
+    }
+
+    async ack(id: string, result: AckResult): Promise<void> {
+        const row = this.rows.get(id);
+        if (!row) return;
+        const now = Date.now();
+        row.attempts += 1;
+        row.lastAttemptedAt = now;
+        row.updatedAt = now;
+        row.claimedBy = undefined;
+        row.claimedAt = undefined;
+        row.responseCode = result.httpStatus;
+        row.responseBody = result.responseBody;
+
+        let status: DeliveryStatus;
+        if (result.success) {
+            status = 'success';
+            row.nextRetryAt = undefined;
+            row.error = undefined;
+        } else if (result.dead) {
+            status = 'dead';
+            row.error = result.error;
+            row.nextRetryAt = undefined;
+        } else {
+            status = 'pending';
+            row.error = result.error;
+            row.nextRetryAt = result.nextRetryAt;
+        }
+        row.status = status;
+    }
+
+    async list(filter?: { status?: DeliveryStatus }): Promise<WebhookDelivery[]> {
+        const all = Array.from(this.rows.values()).map((r) => ({ ...r }));
+        return filter?.status ? all.filter((r) => r.status === filter.status) : all;
+    }
+}