@objectstack/plugin-webhooks 5.1.0 → 5.2.0

package/dist/index.d.ts

@@ -1,104 +1,455 @@
 import { Plugin, PluginContext } from '@objectstack/core';
+import { IDataEngine, IRealtimeService, IClusterService } from '@objectstack/spec/contracts';
+import { I as IWebhookOutbox, a as AckResult, W as WebhookDelivery, E as EnqueueInput, C as ClaimOptions, D as DeliveryStatus } from './outbox-bPQmKYPN.js';
+export { A as AckFailure, b as AckSuccess } from './outbox-bPQmKYPN.js';
 
 /**
- * A single webhook delivery target.
+ * Optional logger interface (subset of console / kernel logger).
  */
-interface WebhookSink {
-    /** Unique sink id used for log correlation. */
+interface OptionalLogger$1 {
+    info?(msg: string, meta?: unknown): void;
+    warn?(msg: string, meta?: unknown): void;
+    debug?(msg: string, meta?: unknown): void;
+    error?(msg: string, err?: unknown, meta?: unknown): void;
+}
+/**
+ * Per-row subscription cached in memory. Mirrors a subset of the
+ * `sys_webhook` object — only what the auto-enqueuer needs to match an
+ * event and build an `EnqueueInput`.
+ */
+interface CachedSubscription {
     id: string;
-    /** Target HTTPS URL. */
+    name: string;
+    objectName: string | undefined;
+    triggers: Set<'create' | 'update' | 'delete' | 'undelete'>;
     url: string;
-    /** Optional HMAC-SHA256 secret. When set, `X-Objectstack-Signature: sha256=…` is added. */
+    method?: string;
+    headers?: Record<string, string>;
     secret?: string;
+    timeoutMs?: number;
+}
+interface AutoEnqueuerOptions {
     /**
-     * Restrict to specific object names (logical names, e.g. `lead`, `account`).
-     * Omit / empty → all objects.
+     * Object name holding webhook subscriptions. Defaults to `sys_webhook`,
+     * the platform-objects schema authored in apps.
      */
-    objects?: string[];
+    subscriptionsObject?: string;
     /**
-     * Restrict to specific event types. Omit / empty → all `data.record.*` events.
+     * Periodic full-cache refresh interval (ms). Belt-and-braces in case
+     * the subscription-change event is missed. Default 60s.
      */
-    eventTypes?: string[];
-    /** Extra headers to send (Authorization, Tenant, etc.). */
-    headers?: Record<string, string>;
-    /** Per-request timeout in milliseconds. Default 5000. */
-    timeoutMs?: number;
-    /** Retry attempts on transient failure. Default 3. Set 0 to disable retries. */
-    retries?: number;
+    refreshIntervalMs?: number;
+    logger?: OptionalLogger$1;
 }
 /**
- * Delivery attempt outcome surfaced to in-process listeners / tests.
+ * Bridge between `IRealtimeService` (`data.record.*` events emitted by
+ * the engine) and `IWebhookOutbox` (durable delivery rows the dispatcher
+ * picks up).
+ *
+ * ## Why a separate class
+ * Keeps `WebhookOutboxPlugin` lean: the plugin wires services, this
+ * class owns the runtime fan-out logic + subscription cache.
+ *
+ * ## Hot path
+ * Every `engine.insert/update/delete` fires a `data.record.*` event.
+ * The handler:
+ *   1. Looks up matching subscriptions in an in-memory `Map<object, sub[]>`
+ *      — O(1) per event, no DB hit on the write path.
+ *   2. Calls `outbox.enqueue()` fire-and-forget for each match. The
+ *      enqueue itself is a single INSERT, which runs *after* the user's
+ *      request has already returned.
+ *
+ * Net cost on the write path: one synchronous Map lookup (~microseconds).
+ *
+ * ## Cache freshness
+ * The cache is rebuilt:
+ *   1. Once on `start()`.
+ *   2. On every `data.record.{created,updated,deleted}` event whose
+ *      object is `sys_webhook` (self-healing — when a user toggles a
+ *      webhook, the handler refreshes the cache before returning).
+ *   3. Periodically (default 60s) as belt-and-braces.
+ *
+ * For multi-node clusters this is *eventually consistent* — node B may
+ * not see node A's edit for up to one cycle. That's acceptable for
+ * webhook configuration changes (humans don't expect millisecond
+ * propagation) and matches Hasura's behaviour.
+ *
+ * ## Determinism
+ * `eventId` is computed from `${object}:${recordId}:${type}:${timestamp}`
+ * so the outbox dedup index catches duplicates that could arise from
+ * upstream replay or buggy producers — and is stable across nodes.
  */
-type WebhookDeliveryStatus = 'ok' | 'retrying' | 'failed';
-interface WebhookDeliveryRecord {
-    sinkId: string;
-    url: string;
-    eventType: string;
-    object?: string;
-    status: WebhookDeliveryStatus;
+declare class AutoEnqueuer {
+    private readonly engine;
+    private readonly realtime;
+    private readonly outbox;
+    private readonly subscriptions;
+    private readonly subscriptionsObject;
+    private readonly refreshIntervalMs;
+    private readonly logger;
+    private subId;
+    private subIdSelfHeal;
+    private refreshTimer;
+    private running;
+    private refreshing;
+    constructor(engine: IDataEngine, realtime: IRealtimeService, outbox: IWebhookOutbox, opts?: AutoEnqueuerOptions);
+    /**
+     * Load the subscription cache and start listening for events.
+     */
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    /**
+     * Force-refresh the subscription cache from storage. Concurrent
+     * callers share a single in-flight refresh.
+     */
+    refresh(): Promise<void>;
+    private doRefresh;
+    private parseRow;
+    /**
+     * Handler for the firehose subscription.
+     *
+     * NOTE: we intentionally `void` the inner enqueue() so the realtime
+     * publisher (and therefore the user's request) is never blocked on
+     * webhook persistence.
+     */
+    private handleEvent;
+    private handleSelfHealEvent;
+    /** Test / admin accessor. */
+    snapshot(): ReadonlyMap<string, ReadonlyArray<CachedSubscription>>;
+}
+
+/**
+ * Default per-request timeout. Receivers SHOULD respond within ~30s; we
+ * cap aggressively to free dispatcher slots.
+ */
+declare const DEFAULT_TIMEOUT_MS = 15000;
+type FetchImpl = (input: string, init: {
+    method: string;
+    headers: Record<string, string>;
+    body: string;
+    signal: AbortSignal;
+}) => Promise<{
+    ok: boolean;
+    status: number;
+    text(): Promise<string>;
+}>;
+/** Single HTTP attempt classified to an `AckResult` shape (without nextRetryAt). */
+type AttemptOutcome = {
+    success: true;
+    httpStatus: number;
+    responseBody?: string;
+    durationMs: number;
+} | {
+    success: false;
+    retriable: boolean;
     httpStatus?: number;
-    attempt: number;
+    responseBody?: string;
     error?: string;
+    durationMs: number;
+};
+/**
+ * Send one HTTP attempt for the delivery. Pure (no DB writes) so the
+ * dispatcher owns retry-schedule + ack logic.
+ *
+ *   - 2xx                       → success
+ *   - 4xx (except 408/429)      → permanent failure (retriable = false → goes to `dead`)
+ *   - 408, 429, 5xx, transport  → retriable
+ */
+declare function sendOnce(delivery: WebhookDelivery, fetchImpl: FetchImpl): Promise<AttemptOutcome>;
+/**
+ * Stripe-style retry schedule. Returns the next `nextRetryAt` ms (relative
+ * to `now`) given how many attempts have already happened, or `null` if
+ * the row should be moved to `dead`.
+ *
+ *   attempt 1 fails -> retry in ~1s
+ *   attempt 2 fails -> ~10s
+ *   attempt 3 fails -> ~1m
+ *   attempt 4 fails -> ~10m
+ *   attempt 5 fails -> ~1h
+ *   attempt 6 fails -> ~6h
+ *   attempt 7 fails -> ~24h
+ *   attempt 8+ fails -> dead
+ *
+ * Each delay is multiplied by jitter ∈ [0.8, 1.2].
+ */
+declare function nextRetryDelayMs(attemptsSoFar: number, rng?: () => number): number | null;
+/**
+ * Compose an `AckResult` from an `AttemptOutcome`, applying the retry
+ * schedule on retriable failures.
+ */
+declare function classifyAttempt(outcome: AttemptOutcome, attemptsSoFar: number, now?: number, rng?: () => number): AckResult;
+
+/**
+ * Minimal logger surface — kernel's `Logger` is compatible (extra params
+ * accepted). Keeping it permissive avoids a hard dependency on the spec
+ * Logger interface here.
+ */
+interface DispatcherLogger {
+    warn: (msg: string, meta?: any) => void;
+    info?: (msg: string, meta?: any) => void;
+}
+interface DispatcherOptions {
+    /** Stable id identifying this dispatcher node. */
+    nodeId: string;
+    /** Cluster service providing `lock` (and optional metrics). */
+    cluster: IClusterService;
+    /** Outbox backend. */
+    outbox: IWebhookOutbox;
+    /**
+     * How many partitions to split work across. Each tick the dispatcher
+     * attempts to acquire each partition's lock independently — the node
+     * that wins owns that partition for the duration of the batch.
+     *
+     * Default: 8 (matches webhook-delivery.mdx §4 example).
+     */
+    partitionCount?: number;
+    /** Max rows to claim from each partition per tick. Default 32. */
+    batchSize?: number;
+    /** Tick interval in ms. Default 250. */
+    intervalMs?: number;
+    /** Per-partition lock TTL. Default = 5 × intervalMs. */
+    lockTtlMs?: number;
+    /** Visibility timeout for claimed rows. Default = 2 × lockTtlMs. */
+    claimTtlMs?: number;
+    /** Override `globalThis.fetch` (tests). */
+    fetchImpl?: FetchImpl;
+    /** Hook fired after every attempt — observability hook. */
+    onAttempt?: (delivery: WebhookDelivery, success: boolean) => void;
+    /** RNG override for the retry-jitter schedule (tests). */
+    rng?: () => number;
+    /** Logger callback (optional). */
+    logger?: DispatcherLogger;
+}
+/**
+ * Cross-node webhook dispatcher.
+ *
+ * **Design** — each tick the dispatcher iterates over `partitionCount`
+ * logical partitions. For each, it tries to acquire a cluster-scoped lock
+ * (`webhook.dispatcher.partition.{i}`) with a short TTL. If it wins the
+ * lock, it claims up to `batchSize` ready rows whose `hash(webhookId) mod
+ * partitionCount === i`, POSTs them, and acks. The lock is released
+ * immediately after the batch so other nodes can fairly rotate through.
+ *
+ * **Why per-partition locks rather than one global lock?**
+ *
+ *   1. Throughput — N nodes can process N partitions concurrently.
+ *   2. Partition affinity — rows for the same webhook always sort into the
+ *      same partition, preserving in-order delivery per webhook.
+ *   3. Failure isolation — a stuck node only blocks its partition until the
+ *      TTL elapses; other partitions keep moving.
+ *
+ * **At-least-once, not exactly-once.** Receivers MUST be idempotent on the
+ * `X-Objectstack-Delivery` (== row id) header. If the HTTP call succeeds
+ * but the ack write fails, the row reverts to pending after the claim TTL
+ * and will be re-posted.
+ */
+declare class WebhookDispatcher {
+    private readonly opts;
+    private timer;
+    private running;
+    private inflightTick;
+    constructor(options: DispatcherOptions);
+    /** Begin the periodic loop. Safe to call once; subsequent calls are no-ops. */
+    start(): void;
+    /** Stop the loop and wait for the in-flight tick to drain. */
+    stop(): Promise<void>;
+    /**
+     * Run one full tick (all partitions, single attempt each). Exposed for
+     * deterministic tests that want to step the dispatcher manually.
+     */
+    tick(): Promise<void>;
+    private scheduleTick;
+    private runTick;
+    private runPartition;
+    private processRow;
+}
+
+interface OptionalLogger {
+    info?(msg: string, meta?: unknown): void;
+    warn?(msg: string, meta?: unknown): void;
+    debug?(msg: string, meta?: unknown): void;
+}
+interface DeliveryRetentionOptions {
+    /**
+     * Object name backing the outbox. Defaults to `sys_webhook_delivery`.
+     */
+    objectName?: string;
+    /**
+     * How long to keep `success` rows. Default 7 days. Set to `0` to
+     * disable the success sweep (keep forever — not recommended in
+     * production).
+     */
+    successTtlMs?: number;
+    /**
+     * How long to keep `dead` rows. Default 30 days. Set to `0` to
+     * keep forever.
+     */
+    deadTtlMs?: number;
+    /**
+     * How often to run the sweep. Default 1h.
+     */
+    sweepIntervalMs?: number;
+    logger?: OptionalLogger;
 }
 /**
- * Plugin configuration.
+ * Periodically prunes `sys_webhook_delivery` rows so the table doesn't
+ * grow unbounded.
+ *
+ * Without this every successful POST would leave a permanent row. At
+ * even moderate scale (10 events/s × 3 webhooks = 30 rows/s = ~2.6M
+ * rows/day) the table becomes a problem within a week.
  *
- * Sinks may be supplied programmatically OR via env vars when none are
- * passed (suitable for 12-factor / Docker deployments):
+ * Retention defaults mirror Stripe/GitHub:
+ *   - `success`: 7 days
+ *   - `dead`:   30 days (kept longer for audit & manual re-delivery)
+ *   - `pending`/`in_flight`/`failed`: never auto-pruned (they're
+ *     either live work or signal something needs human attention)
  *
- *   OBJECTSTACK_WEBHOOK_URL       — single URL, or comma-separated URLs.
- *   OBJECTSTACK_WEBHOOK_SECRET    — HMAC secret applied to all env-sourced URLs.
- *   OBJECTSTACK_WEBHOOK_OBJECTS   — comma-separated object whitelist.
- *   OBJECTSTACK_WEBHOOK_EVENTS    — comma-separated event-type whitelist
- *                                   (e.g. `data.record.created`).
+ * Runs on whichever node holds the sweeper interval — it doesn't need
+ * a cluster lock because DELETE WHERE created_at < threshold is
+ * idempotent; multiple nodes running concurrently is wasteful but
+ * safe.
  */
-interface WebhooksPluginOptions {
-    /** Explicit sink list (takes precedence over env vars). */
-    sinks?: WebhookSink[];
-    /** Override fetch (mainly for tests). Defaults to globalThis.fetch. */
-    fetchImpl?: typeof fetch;
-    /** Hook invoked with each delivery outcome (mainly for tests / metrics). */
-    onDelivery?: (record: WebhookDeliveryRecord) => void;
+declare class DeliveryRetentionSweeper {
+    private readonly engine;
+    private readonly objectName;
+    private readonly successTtlMs;
+    private readonly deadTtlMs;
+    private readonly sweepIntervalMs;
+    private readonly logger;
+    private timer;
+    private running;
+    constructor(engine: IDataEngine, opts?: DeliveryRetentionOptions);
+    start(): void;
+    stop(): void;
+    /** Run one sweep immediately. Returns the number of rows deleted. */
+    sweep(now?: number): Promise<{
+        success: number;
+        dead: number;
+    }>;
+}
+
+interface WebhookOutboxPluginOptions extends Partial<Omit<DispatcherOptions, 'cluster' | 'outbox' | 'nodeId'>> {
+    /**
+     * Override the outbox backend. If omitted a fresh `MemoryWebhookOutbox`
+     * is used — fine for local development, **not for production**: each
+     * node will see only its own rows.
+     *
+     * Pass a factory if you need the kernel-resolved `IDataEngine`:
+     *
+     * ```ts
+     * outbox: (ctx) => new SqlWebhookOutbox(
+     *   ctx.getService('objectql'), { partitionCount: 8 },
+     * ),
+     * ```
+     */
+    outbox?: IWebhookOutbox | ((ctx: PluginContext) => IWebhookOutbox);
+    /**
+     * Stable node id. If omitted, uses `process.env.OBJECTSTACK_NODE_ID`
+     * or a random UUID generated at plugin init.
+     */
+    nodeId?: string;
+    /**
+     * If `false`, the plugin registers the outbox/dispatcher services but
+     * does NOT auto-start the loop — useful for tests that want to step
+     * the dispatcher manually via `dispatcher.tick()`.
+     *
+     * Default: true.
+     */
+    autoStart?: boolean;
+    /**
+     * Auto-enqueue config. When enabled (default `true` if the realtime
+     * + data engine services are available), the plugin subscribes to
+     * `data.record.*` events emitted by the engine and automatically
+     * enqueues a delivery row for every matching `sys_webhook` row.
+     *
+     * Set `false` to disable and only use the imperative
+     * `outbox.enqueue()` API.
+     */
+    autoEnqueue?: boolean | AutoEnqueuerOptions;
+    /**
+     * Retention sweep config. When enabled (default `true` if a SQL
+     * outbox is in use), a periodic timer prunes old `success` and
+     * `dead` rows from `sys_webhook_delivery`.
+     *
+     * Set `false` to disable (e.g. when using `MemoryWebhookOutbox`).
+     */
+    retention?: boolean | DeliveryRetentionOptions;
 }
 /**
- * WebhooksPlugin — fan out data.record.* events to external HTTP endpoints.
- *
- * @example
- * ```ts
- * kernel.use(new WebhooksPlugin({
- *   sinks: [
- *     { id: 'crm-sync', url: 'https://hooks.example.com/in',
- *       secret: process.env.HOOK_SECRET, objects: ['lead', 'account'] },
- *   ],
- * }));
- * ```
+ * Wires a persistent, cluster-aware webhook outbox into the kernel.
+ *
+ * Registered services:
+ *   - `webhook.outbox`     → `IWebhookOutbox` (enqueue / claim / ack / list)
+ *   - `webhook.dispatcher` → `WebhookDispatcher` (manual `tick()` if needed)
+ *   - `webhook.autoEnqueuer` → `AutoEnqueuer` when auto-enqueue is on
+ *   - `webhook.retention`    → `DeliveryRetentionSweeper` when retention is on
+ *
+ * End-to-end flow once auto-enqueue is enabled:
+ *
+ *   engine.insert('contact', {...})
+ *     → engine publishes data.record.created via IRealtimeService
+ *     → AutoEnqueuer matches active sys_webhook rows in O(1)
+ *     → outbox.enqueue() runs fire-and-forget (not on the write path)
+ *     → dispatcher claims and POSTs (cluster-coordinated)
+ *
+ * **Cluster requirement** — this plugin depends on the cluster service
+ * (`ClusterServicePlugin`). With the default `memory` driver the
+ * dispatcher works correctly inside a single process; with a real driver
+ * (`@objectstack/service-cluster-redis`) it correctly coordinates work
+ * across nodes.
  */
-declare class WebhooksPlugin implements Plugin {
+declare class WebhookOutboxPlugin implements Plugin {
+    private readonly options;
     name: string;
     version: string;
-    type: string;
+    type: "standard";
     dependencies: string[];
-    private readonly options;
-    private subscriptionIds;
-    private realtime?;
-    private sinks;
-    private logger?;
-    constructor(options?: WebhooksPluginOptions);
+    private dispatcher;
+    private autoEnqueuer;
+    private retention;
+    private outboxInstance;
+    constructor(options?: WebhookOutboxPluginOptions);
     init(ctx: PluginContext): Promise<void>;
-    start(ctx: PluginContext): Promise<void>;
-    stop(ctx: PluginContext): Promise<void>;
-    /**
-     * Resolve sinks from constructor options, falling back to env vars when
-     * none provided. Exposed for testing.
-     */
-    private resolveSinks;
-    /**
-     * Dispatch a single event to a sink, with HMAC signing, timeout, and
-     * exponential-backoff retry. Failures past the retry budget are logged
-     * but never thrown — webhook delivery must never break the originating
-     * mutation.
-     */
-    private dispatch;
+    dispose(): Promise<void>;
+    private resolveOutbox;
+    private bootAutoEnqueue;
+    private bootRetention;
+    private tryGetService;
 }
 
-export { type WebhookDeliveryRecord, type WebhookDeliveryStatus, type WebhookSink, WebhooksPlugin, type WebhooksPluginOptions };
+/**
+ * In-memory `IWebhookOutbox` for tests and single-process development.
+ *
+ * Implements the atomic-claim semantics by running its claim/ack logic
+ * synchronously (single-threaded JS event loop) inside one `Map`. Two
+ * `MemoryWebhookOutbox` instances do NOT share state — for the cross-node
+ * test the *same* instance is passed to both dispatchers (simulating one
+ * shared database).
+ *
+ * A production SQL-backed implementation will live in a sibling file and
+ * use `SELECT ... FOR UPDATE SKIP LOCKED`.
+ */
+declare class MemoryWebhookOutbox implements IWebhookOutbox {
+    private readonly rows;
+    /** Dedup index keyed by `${eventId}::${webhookId}` -> row id. */
+    private readonly dedup;
+    enqueue(input: EnqueueInput): Promise<string>;
+    claim(opts: ClaimOptions): Promise<WebhookDelivery[]>;
+    ack(id: string, result: AckResult): Promise<void>;
+    list(filter?: {
+        status?: DeliveryStatus;
+    }): Promise<WebhookDelivery[]>;
+}
+
+/**
+ * Stable, framework-free partition hash. The dispatcher uses this to
+ * assign webhooks to partitions; the in-memory outbox uses the same hash
+ * to filter rows in `claim()`. Both call sites MUST agree, which is why
+ * this is a single shared helper.
+ *
+ * Uses a 32-bit FNV-1a variant — fast, no allocations, deterministic.
+ */
+declare function hashPartition(key: string, count: number): number;
+
+export { AckResult, type AttemptOutcome, AutoEnqueuer, type AutoEnqueuerOptions, ClaimOptions, DEFAULT_TIMEOUT_MS, type DeliveryRetentionOptions, DeliveryRetentionSweeper, DeliveryStatus, type DispatcherOptions, EnqueueInput, type FetchImpl, IWebhookOutbox, MemoryWebhookOutbox, WebhookDelivery, WebhookDispatcher, WebhookOutboxPlugin, type WebhookOutboxPluginOptions, classifyAttempt, hashPartition, nextRetryDelayMs, sendOnce };