@end-close/relay 0.12.0

package/src/engine/relay.ts

@@ -0,0 +1,294 @@
+import { EventEmitter } from 'node:events'
+import { randomUUID } from 'node:crypto'
+import { refEnrichment, relayConfigSchema, type RelayConfig, type RouteConfig } from '../config/schema.js'
+import { deriveKey } from '../crypto/keys.js'
+import { EndCloseClient } from '../forward/endclose-client.js'
+import { Dispatcher, type DispatchCounts } from '../forward/dispatcher.js'
+import type { Enrichment } from '../forward/enrich.js'
+import { mapEvent, type MappedEvent } from '../forward/mapper.js'
+import { hasAdapter } from '../ingest/adapters/registry.js'
+import type { ProcessorAdapter, RawRequest } from '../ingest/adapters/types.js'
+import type { Json } from '../mask/paths.js'
+import { noopLogger, type Logger } from '../logger.js'
+import { sleep } from '../util/strings.js'
+import { aesGcmCodec, plainCodec } from './codec.js'
+import { RelayHooks, type RelayEventName, type RelayHandler } from './hooks.js'
+import { ingestWebhook, type IngestResult } from './ingest.js'
+import { toSecretResolver, type SecretResolver } from './secrets.js'
+import {
+  DEFAULT_DISPATCH,
+  DEFAULT_RETENTION,
+  type DispatchSettings,
+  type RetentionSettings,
+} from './settings.js'
+import {
+  hasAdmin,
+  MemoryControlStore,
+  staticRoutes,
+  type ControlStore,
+  type EventStore,
+  type RouteProvider,
+} from './store.js'
+
+// The embeddable engine: everything the application does between "webhook arrives" and
+// "record accepted by End Close", with storage, secrets, logging and observability
+// supplied by the host.
+
+export interface RelayOptions {
+  /** Route definitions: the same shape as the `routes` block of relay.yaml. */
+  routes: RouteConfig[] | RouteProvider
+  store: EventStore
+  /** Killswitch and per-route pause state. Default: in-memory, nothing paused. */
+  control?: ControlStore
+  /** Where `auth.secret_env` references resolve. */
+  secrets: SecretResolver | Record<string, string>
+  endclose: { apiKey: string; baseUrl?: string; fetch?: typeof fetch }
+  /** Explicit: encrypt buffered payloads at rest under this key, or store them as-is. */
+  encryption: { dataKey: string | Buffer } | 'none'
+  /** Keys the deterministic `hash` transform. Never leaves the host. */
+  maskingKey: string | Buffer
+  dispatch?: Partial<DispatchSettings>
+  /** `false` disables retention pruning entirely. */
+  retention?: Partial<RetentionSettings> | false
+  /** Default: silent. */
+  logger?: Logger | null
+  /** Additional processor adapters keyed by route `source`. */
+  adapters?: Record<string, ProcessorAdapter>
+  /**
+   * Host functions a route's map may name with `enrich: <name>` on `description` or a
+   * `metadata` entry. Each runs in-process per event after mapping, receives the value at
+   * that field's `source`, and returns what is forwarded (validated like any mapped value).
+   * Throw to retry the event with backoff; throw `EnrichmentError` to park it; return
+   * `undefined` to omit the field. Bounded by `dispatch.enrichTimeoutMs` per call.
+   */
+  enrichments?: Record<string, Enrichment>
+  /** Lease owner for claimed batches. Give each long-lived replica a stable id. */
+  instanceId?: string
+  hooks?: RelayHooks
+  /** Supply a pre-built client (the application shares one with telemetry). */
+  client?: EndCloseClient
+}
+
+export type DispatchOnceResult = DispatchCounts
+
+export interface FlushResult extends DispatchOnceResult {
+  /** True when nothing deliverable remained when flush returned. */
+  drained: boolean
+  /**
+   * Why flush stopped early: the deadline passed, forwarding is paused (killswitch or
+   * every due route), or due events belong to routes the provider no longer knows.
+   */
+  reason?: 'timeout' | 'paused' | 'unroutable'
+}
+
+export interface Relay {
+  /** Framework-agnostic webhook entrypoint. */
+  ingest(routeId: string, req: RawRequest): Promise<IngestResult>
+  /** Start the background dispatch loop (long-lived processes). */
+  start(): void
+  /** Stop the loop and drain in-flight work. */
+  stop(): Promise<void>
+  /** Run one dispatch cycle (cron / serverless / tests). */
+  dispatchOnce(opts?: { prune?: boolean }): Promise<DispatchOnceResult>
+  /**
+   * Run dispatch cycles until nothing deliverable remains or `timeoutMs` (default 30 s)
+   * passes, retrying as backoff timers expire. Returns immediately if forwarding is
+   * paused. Events still `retried` when it returns need a later cycle or a durable store.
+   */
+  flush(opts?: { timeoutMs?: number }): Promise<FlushResult>
+  /** Run retention pruning to completion. */
+  prune(): Promise<{ wiped: number; deleted: number }>
+  /**
+   * Map a sample payload through a route without storing or sending anything. Enrichments
+   * are not run: their fields are listed in `report.enriched` and `pending`.
+   */
+  preview(route: RouteConfig, sample: Json, receivedAt?: string): MappedEvent
+  /** Decode a buffered payload. Sensitive: the caller is responsible for auditing. */
+  readPayload(id: string): Promise<Buffer | undefined>
+  on<E extends RelayEventName>(name: E, handler: RelayHandler<E>): () => void
+  readonly store: EventStore
+  readonly control: ControlStore
+}
+
+function toKey(name: string, v: string | Buffer): Buffer {
+  if (typeof v === 'string') return deriveKey(name, v)
+  if (v.length !== 32) throw new Error(`${name} must be a 32-byte Buffer or a string of 16+ chars`)
+  return v
+}
+
+/** Reject routes whose `source` has no adapter (built-in or host-registered). */
+export function assertKnownSources(
+  routes: RouteConfig[],
+  adapters?: Record<string, ProcessorAdapter>,
+): void {
+  for (const r of routes) {
+    if (!hasAdapter(r.source, adapters)) {
+      throw new Error(`route ${r.id}: no adapter for source "${r.source}"`)
+    }
+  }
+}
+
+/** Every `enrich:` reference in a route's map, as [field, enrichment name]. */
+export function routeEnrichments(route: RouteConfig): [field: string, enrichment: string][] {
+  const out: [string, string][] = []
+  const desc = route.map.description === undefined ? undefined : refEnrichment(route.map.description)
+  if (desc !== undefined) out.push(['description', desc])
+  for (const [key, ref] of Object.entries(route.map.metadata)) {
+    const name = refEnrichment(ref)
+    if (name !== undefined) out.push([`metadata.${key}`, name])
+  }
+  return out
+}
+
+/** Reject routes whose map names an enrichment the host has not registered. */
+export function assertKnownEnrichments(
+  routes: RouteConfig[],
+  enrichments?: Record<string, unknown>,
+): void {
+  for (const r of routes) {
+    for (const [field, name] of routeEnrichments(r)) {
+      if (!enrichments || !Object.hasOwn(enrichments, name)) {
+        throw new Error(`route ${r.id}: ${field} references unknown enrichment "${name}"`)
+      }
+    }
+  }
+}
+
+/**
+ * Validate a routes document (parsed YAML or a plain object) into RouteConfig[]. Applies
+ * defaults, the hard-denylist check on metadata names, duplicate-id, unknown-source and
+ * unknown-enrichment checks. Pass the host's extra adapters and enrichments so routes
+ * that use them validate too; with none registered, any `enrich:` reference is rejected.
+ */
+export function parseRoutes(
+  doc: unknown,
+  opts: { adapters?: Record<string, ProcessorAdapter>; enrichments?: Record<string, unknown> } = {},
+): RouteConfig[] {
+  const config: RelayConfig = relayConfigSchema.parse(doc)
+  const seen = new Set<string>()
+  for (const route of config.routes) {
+    if (seen.has(route.id)) throw new Error(`duplicate route id: ${route.id}`)
+    seen.add(route.id)
+  }
+  assertKnownSources(config.routes, opts.adapters)
+  assertKnownEnrichments(config.routes, opts.enrichments)
+  return config.routes
+}
+
+const FLUSH_POLL_MIN_MS = 50
+const FLUSH_POLL_MAX_MS = 1000
+// Asking for "due" events at this time returns every route holding pending/retry rows.
+const FAR_FUTURE = '9999-12-31T23:59:59.999Z'
+
+export function createRelay(opts: RelayOptions): Relay {
+  if (Array.isArray(opts.routes)) {
+    assertKnownSources(opts.routes, opts.adapters)
+    assertKnownEnrichments(opts.routes, opts.enrichments)
+  }
+  const routes = Array.isArray(opts.routes) ? staticRoutes(opts.routes) : opts.routes
+  const control = opts.control ?? new MemoryControlStore()
+  const secrets = toSecretResolver(opts.secrets)
+  const logger = opts.logger ?? noopLogger
+  const hooks = opts.hooks ?? new RelayHooks()
+  const codec =
+    opts.encryption === 'none' ? plainCodec : aesGcmCodec(toKey('dataKey', opts.encryption.dataKey))
+  const maskingKey = toKey('maskingKey', opts.maskingKey)
+  const client =
+    opts.client ??
+    new EndCloseClient(
+      opts.endclose.baseUrl ?? 'https://api.endclose.com/v1',
+      opts.endclose.apiKey,
+      opts.endclose.fetch ?? fetch,
+    )
+  const dispatch: DispatchSettings = { ...DEFAULT_DISPATCH, ...opts.dispatch }
+  const retention = opts.retention === false ? null : { ...DEFAULT_RETENTION, ...opts.retention }
+  const signal = new EventEmitter()
+  const { store } = opts
+
+  const ingestDeps = {
+    store,
+    control,
+    routes,
+    secrets,
+    codec,
+    signal,
+    hooks,
+    logger,
+    ...(opts.adapters ? { adapters: opts.adapters } : {}),
+  }
+
+  const dispatcher = new Dispatcher({
+    store,
+    control,
+    routes,
+    dispatch,
+    retention,
+    client,
+    codec,
+    maskingKey,
+    instanceId: opts.instanceId ?? randomUUID(),
+    signal,
+    hooks,
+    logger,
+    ...(opts.enrichments ? { enrichments: opts.enrichments } : {}),
+  })
+
+  const dispatchOnce = async (o: { prune?: boolean } = {}): Promise<DispatchOnceResult> => {
+    const { delivered, retried, parked } = await dispatcher.runOnce()
+    if (o.prune) await dispatcher.pruneNow()
+    return { delivered, retried, parked }
+  }
+
+  const flush = async ({ timeoutMs = 30_000 } = {}): Promise<FlushResult> => {
+    const deadline = Date.now() + timeoutMs
+    const totals: FlushResult = { delivered: 0, retried: 0, parked: 0, drained: false }
+    let pollMs = FLUSH_POLL_MIN_MS
+    for (;;) {
+      const c = await dispatcher.runOnce()
+      totals.delivered += c.delivered
+      totals.retried += c.retried
+      totals.parked += c.parked
+      if (c.halted) return { ...totals, reason: 'paused' }
+
+      const remaining = deadline - Date.now()
+      if (remaining <= 0) return { ...totals, reason: 'timeout' }
+      const touched = c.delivered + c.retried + c.parked
+      if (touched > 0) continue // there may be more than one batch's worth; go straight back
+
+      const backlog = await store.routesWithDueEvents(FAR_FUTURE)
+      if (backlog.length === 0) return { ...totals, drained: true }
+      // Backlog confined to routes the cycle deliberately skipped will not clear on its
+      // own. Anything else is waiting on a backoff timer.
+      const skipped = new Map(c.skipped.map((s) => [s.routeId, s.reason]))
+      if (backlog.every((r) => skipped.has(r))) {
+        const allPaused = backlog.every((r) => skipped.get(r) === 'paused')
+        return { ...totals, reason: allPaused ? 'paused' : 'unroutable' }
+      }
+
+      await sleep(Math.min(pollMs, remaining))
+      pollMs = Math.min(pollMs * 2, FLUSH_POLL_MAX_MS)
+    }
+  }
+
+  const readPayload = async (id: string): Promise<Buffer | undefined> => {
+    if (!hasAdmin(store)) throw new Error('store does not support inspection (EventStoreAdmin)')
+    const row = await store.getById(id)
+    if (!row || row.payload.length === 0) return undefined
+    return codec.decode(row.payload, row.payload_iv)
+  }
+
+  return {
+    ingest: (routeId, req) => ingestWebhook(ingestDeps, routeId, req),
+    start: () => dispatcher.start(),
+    stop: () => dispatcher.stop(),
+    dispatchOnce,
+    flush,
+    prune: () => dispatcher.pruneNow(),
+    preview: (route, sample, receivedAt = new Date().toISOString()) =>
+      mapEvent(route, sample, receivedAt, maskingKey),
+    readPayload,
+    on: (name, handler) => hooks.on(name, handler),
+    store,
+    control,
+  }
+}

package/src/engine/secrets.ts

@@ -0,0 +1,51 @@
+// Secret resolution for the engine. Route configs reference secrets by NAME only (the
+// `secret_env` field); the host decides where the value comes from. The application uses the
+// process environment; an embedding application may use a static map loaded from its
+// secret manager at startup.
+//
+// Resolution is synchronous by design: it sits on the webhook verification hot path, and
+// secrets should be materialised once at boot rather than fetched per request.
+
+export interface SecretResolver {
+  /** The secret value for `ref`, or undefined when it is not available. */
+  resolve(ref: string): string | undefined
+  /** Whether `ref` is currently available (defaults to `resolve(ref) !== undefined`). */
+  has?(ref: string): boolean
+}
+
+export class SecretUnavailableError extends Error {
+  constructor(readonly ref: string) {
+    super(`missing required secret env var: ${ref}`)
+  }
+}
+
+/** Resolve from an environment-shaped record; empty strings count as unset. */
+export function envSecrets(env: Record<string, string | undefined> = process.env): SecretResolver {
+  return {
+    resolve: (ref) => {
+      const v = env[ref]
+      return v ? v : undefined
+    },
+  }
+}
+
+export function staticSecrets(map: Record<string, string>): SecretResolver {
+  return { resolve: (ref) => map[ref] }
+}
+
+export function toSecretResolver(s: SecretResolver | Record<string, string>): SecretResolver {
+  return typeof (s as SecretResolver).resolve === 'function'
+    ? (s as SecretResolver)
+    : staticSecrets(s as Record<string, string>)
+}
+
+export function hasSecret(resolver: SecretResolver, ref: string): boolean {
+  return resolver.has ? resolver.has(ref) : resolver.resolve(ref) !== undefined
+}
+
+/** Resolve or throw `SecretUnavailableError`. */
+export function requireSecret(resolver: SecretResolver, ref: string): string {
+  const v = resolver.resolve(ref)
+  if (v === undefined) throw new SecretUnavailableError(ref)
+  return v
+}

package/src/engine/settings.ts

@@ -0,0 +1,42 @@
+// Tuning knobs shared by createRelay() and the Dispatcher. One shape, camelCase.
+
+export interface DispatchSettings {
+  /** Records per bulk POST. */
+  batchMax: number
+  /** How often the loop wakes when idle (it also wakes immediately on ingest). */
+  pollIntervalMs: number
+  backoffBaseMs: number
+  backoffCapMs: number
+  /** Retrying events park (never dropped) after this long. */
+  parkAfterMs: number
+  /** How long a claimed batch is protected from recovery by other instances. */
+  leaseMs: number
+  /** How often a running instance sweeps for expired leases left by crashed peers. */
+  recoverIntervalMs: number
+  /**
+   * Upper bound on one `enrich` call. A slower call counts as a transient failure and the
+   * event retries with backoff. Enrichments run one at a time per event, so keep
+   * batchMax × (enrichments per event) × enrichTimeoutMs well under leaseMs.
+   */
+  enrichTimeoutMs: number
+}
+
+export interface RetentionSettings {
+  /** Payloads of delivered/filtered events are wiped after this many days. */
+  deliveredDays: number
+  /** Their rows (the idempotency ledger) are deleted after this many days. */
+  ledgerDays: number
+}
+
+export const DEFAULT_DISPATCH: DispatchSettings = {
+  batchMax: 100,
+  pollIntervalMs: 250,
+  backoffBaseMs: 1000,
+  backoffCapMs: 600_000,
+  parkAfterMs: 7 * 24 * 3600 * 1000,
+  leaseMs: 600_000,
+  recoverIntervalMs: 60_000,
+  enrichTimeoutMs: 5_000,
+}
+
+export const DEFAULT_RETENTION: RetentionSettings = { deliveredDays: 7, ledgerDays: 30 }

package/src/engine/store.ts

@@ -0,0 +1,188 @@
+import type { RouteConfig } from '../config/schema.js'
+
+// The persistence contracts the engine depends on. A host supplies implementations: the
+// application uses SQLite; an embedding application may use its own database. Every
+// method is async so implementations can be backed by a network database.
+
+export type EventStatus =
+  | 'pending'
+  | 'retry'
+  | 'delivering'
+  | 'delivered'
+  | 'parked'
+  | 'dropped_by_filter'
+
+export interface EventRecord {
+  /** Opaque store-assigned id. */
+  id: string
+  route_id: string
+  source: string
+  event_id: string
+  event_type: string | null
+  /** Whatever the PayloadCodec produced; opaque to the store. */
+  payload: Buffer
+  payload_iv: Buffer | null
+  headers_json: string
+  received_at: string
+  status: EventStatus
+  attempts: number
+  next_attempt_at: string | null
+  delivered_at: string | null
+  bulk_request_id: string | null
+  last_error: string | null
+  idempotency_key: string
+  claimed_by: string | null
+  lease_until: string | null
+}
+
+export interface NewEvent {
+  route_id: string
+  source: string
+  event_id: string
+  event_type: string | null
+  payload: Buffer
+  payload_iv: Buffer | null
+  headers_json: string
+  received_at: string
+  status: EventStatus
+  idempotency_key: string
+}
+
+export interface RouteStats {
+  route_id: string
+  counts: Partial<Record<EventStatus, number>>
+  last_delivered_at: string | null
+  oldest_pending_at: string | null
+}
+
+export type EventSummary = Omit<
+  EventRecord,
+  'payload' | 'payload_iv' | 'headers_json' | 'idempotency_key' | 'claimed_by' | 'lease_until'
+>
+
+export type InsertResult = { duplicate: false; id: string } | { duplicate: true }
+
+/** A store operation failed. `op` names the operation for logs and hooks. */
+export class StoreError extends Error {
+  constructor(
+    message: string,
+    readonly op: string,
+    options?: { cause?: unknown },
+  ) {
+    super(message, options)
+    this.name = 'StoreError'
+  }
+}
+
+/**
+ * The store is temporarily unable to serve a request (lock contention, connection
+ * loss). Ingest answers 503 so the processor retries; any other StoreError is 500.
+ */
+export class StoreUnavailableError extends StoreError {
+  constructor(message: string, op: string, options?: { cause?: unknown }) {
+    super(message, op, options)
+    this.name = 'StoreUnavailableError'
+  }
+}
+
+export function storeOp(err: unknown): string | undefined {
+  return err instanceof StoreError ? err.op : undefined
+}
+
+/** What the engine needs from event storage. */
+export interface EventStore {
+  /** Persist a new event; `duplicate: true` when the idempotency key already exists. */
+  insert(e: NewEvent): Promise<InsertResult>
+  /**
+   * Routes with `pending`/`retry` events whose next attempt is at or before `now`.
+   * Called with a far-future `now` it answers "which routes hold any backlog at all".
+   */
+  routesWithDueEvents(now: string): Promise<string[]>
+  /**
+   * Atomically select up to `limit` due events for a route (oldest first), mark them
+   * `delivering` and record the lease. Concurrent callers must never receive overlapping
+   * rows (a SQL store would use SELECT ... FOR UPDATE SKIP LOCKED).
+   */
+  claimDue(routeId: string, now: string, limit: number, lease: Lease): Promise<EventRecord[]>
+  markDelivered(ids: string[], deliveredAt: string, bulkRequestId: string | null): Promise<void>
+  markFailed(ids: string[], nextAttemptAt: string, error: string): Promise<void>
+  markParked(ids: string[], error: string): Promise<void>
+  /** Return the given rows to `retry` if (and only if) they are still `delivering`. */
+  releaseDelivering(ids: string[], nextAttemptAt: string, error: string): Promise<number>
+  /**
+   * Return `delivering` rows whose lease has expired — or that were claimed by `owner`,
+   * so a restarted instance reclaims its own work immediately — to `retry`.
+   */
+  recoverDelivering(now: string, owner?: string): Promise<number>
+  /** Park events that have been retrying longer than `maxAgeMs`. */
+  parkExpired(now: string, maxAgeMs: number): Promise<number>
+  /** One bounded retention step; callers loop until it returns zeros. */
+  pruneBatch(
+    now: string,
+    deliveredDays: number,
+    ledgerDays: number,
+    limit: number,
+  ): Promise<{ wiped: number; deleted: number }>
+  close?(): Promise<void>
+}
+
+export interface Lease {
+  owner: string
+  until: string
+}
+
+/** Optional inspection/replay capability used by an operator surface. */
+export interface EventStoreAdmin {
+  getById(id: string): Promise<EventRecord | undefined>
+  list(filter: { status?: EventStatus; route?: string; limit?: number }): Promise<EventSummary[]>
+  countByStatus(): Promise<Record<string, number>>
+  perRouteStats(): Promise<RouteStats[]>
+  replay(id: string): Promise<boolean>
+  replayAllParked(): Promise<number>
+}
+
+export function hasAdmin(store: EventStore): store is EventStore & EventStoreAdmin {
+  return typeof (store as Partial<EventStoreAdmin>).list === 'function'
+}
+
+export type Killswitch = 'none' | 'pause' | 'panic'
+
+/** Runtime-mutable operator state: the global killswitch and per-route pauses. */
+export interface ControlStore {
+  getKillswitch(): Promise<Killswitch>
+  setKillswitch(state: Killswitch): Promise<void>
+  isRoutePaused(routeId: string): Promise<boolean>
+  setRoutePaused(routeId: string, paused: boolean): Promise<void>
+}
+
+/** Where the engine reads route definitions. Static for an embedding app; live for the application. */
+export interface RouteProvider {
+  get(id: string): Promise<RouteConfig | undefined>
+  all(): Promise<RouteConfig[]>
+}
+
+export function staticRoutes(routes: RouteConfig[]): RouteProvider {
+  const byId = new Map(routes.map((r) => [r.id, r]))
+  return {
+    get: async (id) => byId.get(id),
+    all: async () => [...byId.values()],
+  }
+}
+
+export class MemoryControlStore implements ControlStore {
+  private killswitch: Killswitch = 'none'
+  private paused = new Set<string>()
+  async getKillswitch(): Promise<Killswitch> {
+    return this.killswitch
+  }
+  async setKillswitch(state: Killswitch): Promise<void> {
+    this.killswitch = state
+  }
+  async isRoutePaused(routeId: string): Promise<boolean> {
+    return this.paused.has(routeId)
+  }
+  async setRoutePaused(routeId: string, paused: boolean): Promise<void> {
+    if (paused) this.paused.add(routeId)
+    else this.paused.delete(routeId)
+  }
+}

package/src/forward/backoff.ts

@@ -0,0 +1,10 @@
+// Exponential backoff with ±20% jitter, Sequin-style: base * 2^attempts, capped.
+export function backoffMs(attempts: number, baseMs = 1000, capMs = 600_000): number {
+  const raw = Math.min(baseMs * 2 ** Math.min(attempts, 30), capMs)
+  const jitter = 0.8 + Math.random() * 0.4
+  return Math.round(raw * jitter)
+}
+
+export function nextAttemptAt(attempts: number, baseMs?: number, capMs?: number): string {
+  return new Date(Date.now() + backoffMs(attempts, baseMs, capMs)).toISOString()
+}