@end-close/relay 0.12.0

package/src/engine/ingest.ts

@@ -0,0 +1,194 @@
+import { createHash } from 'node:crypto'
+import type { EventEmitter } from 'node:events'
+import { StoreError, StoreUnavailableError, type ControlStore, type EventStore, type RouteProvider } from './store.js'
+import type { PayloadCodec } from './codec.js'
+import { adapterFor, hasAdapter } from '../ingest/adapters/registry.js'
+import type { ProcessorAdapter, RawRequest } from '../ingest/adapters/types.js'
+import type { Json } from '../mask/paths.js'
+import { RelayHooks, type IngestOutcome } from './hooks.js'
+import { noopLogger, type Logger } from '../logger.js'
+import type { SecretResolver } from './secrets.js'
+import { jsonTopLevelKeys, requestHeaderNames } from '../util/payload-shape.js'
+import { escapeRe } from '../util/strings.js'
+
+// The framework-agnostic ingest path: verify → encrypt → persist → ack. A host adapts its
+// HTTP request into a RawRequest and maps the IngestResult back onto its response.
+
+// Headers persisted alongside the payload for debugging/replay. Auth headers are
+// deliberately excluded — secrets never reach the database.
+const PERSISTED_HEADERS = ['content-type', 'user-agent']
+
+export interface IngestDeps {
+  store: EventStore
+  control: ControlStore
+  routes: RouteProvider
+  secrets: SecretResolver
+  /** How buffered payloads are stored (encrypted or plain). */
+  codec: PayloadCodec
+  /** Emits 'event' whenever a new deliverable event lands, so the dispatcher wakes immediately. */
+  signal?: EventEmitter
+  hooks?: RelayHooks
+  logger?: Logger
+  /** Additional processor adapters keyed by route `source` (override built-ins by name). */
+  adapters?: Record<string, ProcessorAdapter>
+}
+
+export type IngestResultOutcome =
+  | IngestOutcome
+  | 'unknown_route'
+  | 'secret_unavailable'
+  | 'unknown_source'
+  | 'persist_failed'
+  | 'unavailable'
+
+/** HTTP-shaped so any host can answer the processor without knowing the engine's rules. */
+export interface IngestResult {
+  status: 200 | 400 | 401 | 404 | 413 | 500 | 503
+  body: { status: string } | { error: string }
+  outcome: IngestResultOutcome
+  /** Store id of the buffered event; present for `accepted` and `filtered`. Correlate with the `settled` hook. */
+  id?: string
+}
+
+export function eventTypeMatches(patterns: string[], eventType: string | null): boolean {
+  if (patterns.length === 0) return true
+  if (eventType === null) return false
+  return patterns.some((p) =>
+    p.includes('*') ? new RegExp(`^${p.split('*').map(escapeRe).join('.*')}$`).test(eventType) : p === eventType,
+  )
+}
+
+/** Idempotency key for one inbound event: stable across redeliveries of the same processor event. */
+export function eventIdempotencyKey(source: string, eventId: string): string {
+  return 'sha256:' + createHash('sha256').update(`${source}:${eventId}`).digest('hex')
+}
+
+export async function ingestWebhook(
+  deps: IngestDeps,
+  routeId: string,
+  raw: RawRequest,
+): Promise<IngestResult> {
+  const hooks = deps.hooks ?? new RelayHooks()
+  const logger = deps.logger ?? noopLogger
+  try {
+    return await ingestInner(deps, routeId, raw, hooks, logger)
+  } catch (err) {
+    // Any store failure (route lookup, killswitch, insert): 503 when the store says it is
+    // temporarily unavailable so the processor retries, 500 otherwise. Never the raw error.
+    if (!(err instanceof StoreError)) throw err
+    logger.error('ingest store failure', { route: routeId, op: err.op, error: err.message })
+    hooks.emit('error', { kind: 'ingest_persist', error: err, routeId, op: err.op })
+    if (err instanceof StoreUnavailableError) {
+      return { status: 503, body: { error: 'temporarily unavailable' }, outcome: 'unavailable' }
+    }
+    return { status: 500, body: { error: 'internal error' }, outcome: 'persist_failed' }
+  }
+}
+
+async function ingestInner(
+  deps: IngestDeps,
+  routeId: string,
+  raw: RawRequest,
+  hooks: RelayHooks,
+  logger: Logger,
+): Promise<IngestResult> {
+  const { store, control, routes, secrets, codec, signal } = deps
+
+  const route = await routes.get(routeId)
+  if (!route) return { status: 404, body: { error: 'unknown route' }, outcome: 'unknown_route' }
+
+  const { rawBody } = raw
+  const bodyBytes = Buffer.isBuffer(rawBody) ? rawBody.length : 0
+  const ingested = (outcome: IngestOutcome, eventType: string | null = null) =>
+    hooks.emit('ingest', { routeId, outcome, eventType, bodyBytes })
+
+  // Panic refuses at the door; the processor's own retries carry the window.
+  if ((await control.getKillswitch()) === 'panic') {
+    ingested('panic')
+    return { status: 503, body: { error: 'relay is in panic mode' }, outcome: 'panic' }
+  }
+
+  if (!Buffer.isBuffer(rawBody) || rawBody.length === 0) {
+    ingested('rejected_json')
+    return { status: 400, body: { error: 'empty body' }, outcome: 'rejected_json' }
+  }
+  if (rawBody.length > route.max_body_bytes) {
+    ingested('rejected_size')
+    return { status: 413, body: { error: 'body too large' }, outcome: 'rejected_size' }
+  }
+
+  if (!hasAdapter(route.source, deps.adapters)) {
+    logger.error('ingest route has no adapter', { route: routeId, source: route.source })
+    return { status: 500, body: { error: 'internal error' }, outcome: 'unknown_source' }
+  }
+  const adapter = adapterFor(route.source, deps.adapters)
+  const secret = secrets.resolve(route.auth.secret_env)
+  if (secret === undefined) {
+    logger.error('ingest secret unavailable', { route: routeId, secret_env: route.auth.secret_env })
+    return { status: 500, body: { error: 'internal error' }, outcome: 'secret_unavailable' }
+  }
+  const verdict = adapter.verify(raw, route, { secret })
+  if (!verdict.ok) {
+    ingested('rejected_auth')
+    logger.warn('ingest rejected', { route: routeId, reason: verdict.reason })
+    return { status: 401, body: { error: 'verification failed' }, outcome: 'rejected_auth' }
+  }
+
+  let body: Json
+  try {
+    body = JSON.parse(rawBody.toString('utf8')) as Json
+  } catch {
+    ingested('rejected_json')
+    return { status: 400, body: { error: 'invalid JSON' }, outcome: 'rejected_json' }
+  }
+
+  const eventId = adapter.extractEventId(body, raw, route)
+  const eventType = adapter.extractEventType(body, raw, route)
+  const filtered = Boolean(route.events && !eventTypeMatches(route.events, eventType))
+  // Shape-only debug metadata (no values).
+  logger.debug('ingest shape', {
+    route: routeId,
+    event_type: eventType,
+    body_bytes: rawBody.length,
+    remote_ip: raw.remoteIp,
+    header_names: requestHeaderNames(raw.headers as Record<string, unknown>),
+    payload_keys: jsonTopLevelKeys(body),
+  })
+
+  const { payload, iv } = codec.encode(rawBody)
+  const headersJson = JSON.stringify(
+    Object.fromEntries(
+      PERSISTED_HEADERS.map((h) => [h, raw.headers[h]]).filter(([, v]) => v !== undefined),
+    ),
+  )
+
+  const inserted = await store.insert({
+    route_id: routeId,
+    source: route.source,
+    event_id: eventId,
+    event_type: eventType,
+    payload,
+    payload_iv: iv,
+    headers_json: headersJson,
+    received_at: new Date().toISOString(),
+    status: filtered ? 'dropped_by_filter' : 'pending',
+    idempotency_key: eventIdempotencyKey(route.source, eventId),
+  })
+
+  if (inserted.duplicate) {
+    ingested('duplicate', eventType)
+    logger.info('duplicate event acked', { route: routeId, event_type: eventType })
+    return { status: 200, body: { status: 'duplicate' }, outcome: 'duplicate' }
+  }
+
+  ingested(filtered ? 'filtered' : 'accepted', eventType)
+  hooks.emit('stored', { routeId, id: inserted.id, filtered })
+  logger.info('event ingested', { route: routeId, event_type: eventType, filtered })
+  if (!filtered) signal?.emit('event', routeId)
+  return {
+    status: 200,
+    body: { status: filtered ? 'filtered' : 'accepted' },
+    outcome: filtered ? 'filtered' : 'accepted',
+    id: inserted.id,
+  }
+}

package/src/engine/memory-store.ts

@@ -0,0 +1,232 @@
+import type {
+  EventRecord,
+  EventStatus,
+  EventStore,
+  EventStoreAdmin,
+  EventSummary,
+  InsertResult,
+  Lease,
+  NewEvent,
+  RouteStats,
+} from './store.js'
+
+// In-process EventStore for development, tests and single-process experiments. Not
+// durable: everything is lost when the process exits. It is the reference semantics for
+// the store contract (see the contract test suite).
+
+const DUE: EventStatus[] = ['pending', 'retry']
+const TERMINAL: EventStatus[] = ['delivered', 'dropped_by_filter']
+
+export class MemoryEventStore implements EventStore, EventStoreAdmin {
+  private rows = new Map<string, EventRecord>()
+  private byKey = new Map<string, string>()
+  private nextId = 1
+
+  async insert(e: NewEvent): Promise<InsertResult> {
+    if (this.byKey.has(e.idempotency_key)) return { duplicate: true }
+    const id = String(this.nextId++)
+    this.rows.set(id, {
+      ...e,
+      id,
+      attempts: 0,
+      next_attempt_at: e.received_at,
+      delivered_at: null,
+      bulk_request_id: null,
+      last_error: null,
+      claimed_by: null,
+      lease_until: null,
+    })
+    this.byKey.set(e.idempotency_key, id)
+    return { duplicate: false, id }
+  }
+
+  async routesWithDueEvents(now: string): Promise<string[]> {
+    const out = new Set<string>()
+    for (const r of this.rows.values()) if (this.isDue(r, now)) out.add(r.route_id)
+    return [...out]
+  }
+
+  async claimDue(routeId: string, now: string, limit: number, lease: Lease): Promise<EventRecord[]> {
+    const due = [...this.rows.values()]
+      // Map iteration is insertion order, and ids are assigned in insertion order.
+      .filter((r) => r.route_id === routeId && this.isDue(r, now))
+      .slice(0, limit)
+    for (const r of due) {
+      r.status = 'delivering'
+      r.next_attempt_at = now
+      r.claimed_by = lease.owner
+      r.lease_until = lease.until
+    }
+    return due.map((r) => ({ ...r }))
+  }
+
+  async markDelivered(ids: string[], deliveredAt: string, bulkRequestId: string | null): Promise<void> {
+    for (const r of this.pick(ids)) {
+      r.status = 'delivered'
+      r.delivered_at = deliveredAt
+      r.bulk_request_id = bulkRequestId
+      r.last_error = null
+      this.clearLease(r)
+    }
+  }
+
+  async markFailed(ids: string[], nextAttemptAt: string, error: string): Promise<void> {
+    for (const r of this.pick(ids)) {
+      r.status = 'retry'
+      r.attempts += 1
+      r.next_attempt_at = nextAttemptAt
+      r.last_error = error.slice(0, 500)
+      this.clearLease(r)
+    }
+  }
+
+  async markParked(ids: string[], error: string): Promise<void> {
+    for (const r of this.pick(ids)) {
+      r.status = 'parked'
+      r.last_error = error.slice(0, 500)
+      this.clearLease(r)
+    }
+  }
+
+  async releaseDelivering(ids: string[], nextAttemptAt: string, error: string): Promise<number> {
+    let n = 0
+    for (const r of this.pick(ids)) {
+      if (r.status !== 'delivering') continue
+      r.status = 'retry'
+      r.attempts += 1
+      r.next_attempt_at = nextAttemptAt
+      r.last_error = error.slice(0, 500)
+      this.clearLease(r)
+      n++
+    }
+    return n
+  }
+
+  async recoverDelivering(now: string, owner?: string): Promise<number> {
+    let n = 0
+    for (const r of this.rows.values()) {
+      if (r.status !== 'delivering') continue
+      const mine = owner !== undefined && r.claimed_by === owner
+      if (r.lease_until === null || r.lease_until < now || mine) {
+        r.status = 'retry'
+        r.next_attempt_at = now
+        this.clearLease(r)
+        n++
+      }
+    }
+    return n
+  }
+
+  async parkExpired(now: string, maxAgeMs: number): Promise<number> {
+    const cutoff = new Date(Date.parse(now) - maxAgeMs).toISOString()
+    let n = 0
+    for (const r of this.rows.values()) {
+      if (r.status === 'retry' && r.received_at < cutoff) {
+        r.status = 'parked'
+        r.last_error = 'retry window exhausted'
+        n++
+      }
+    }
+    return n
+  }
+
+  async pruneBatch(now: string, deliveredDays: number, ledgerDays: number, limit: number) {
+    const wipeCutoff = new Date(Date.parse(now) - deliveredDays * 86_400_000).toISOString()
+    const deleteCutoff = new Date(Date.parse(now) - ledgerDays * 86_400_000).toISOString()
+    const terminal = [...this.rows.values()].filter((r) => TERMINAL.includes(r.status))
+    const toWipe = terminal.filter((r) => r.received_at < wipeCutoff && r.payload.length > 0).slice(0, limit)
+    if (toWipe.length > 0) {
+      for (const r of toWipe) {
+        r.payload = Buffer.alloc(0)
+        r.payload_iv = null
+        r.headers_json = '{}'
+      }
+      return { wiped: toWipe.length, deleted: 0 }
+    }
+    const toDelete = terminal.filter((r) => r.received_at < deleteCutoff).slice(0, limit)
+    for (const r of toDelete) {
+      this.rows.delete(r.id)
+      this.byKey.delete(r.idempotency_key)
+    }
+    return { wiped: 0, deleted: toDelete.length }
+  }
+
+  // ── admin ──
+  async getById(id: string): Promise<EventRecord | undefined> {
+    const r = this.rows.get(id)
+    return r ? { ...r } : undefined
+  }
+
+  async list(filter: { status?: EventStatus; route?: string; limit?: number }): Promise<EventSummary[]> {
+    return [...this.rows.values()]
+      .filter((r) => (!filter.status || r.status === filter.status) && (!filter.route || r.route_id === filter.route))
+      .sort((a, b) => Number(b.id) - Number(a.id))
+      .slice(0, filter.limit ?? 50)
+      .map(({ payload: _p, payload_iv: _iv, headers_json: _h, idempotency_key: _k, claimed_by: _c, lease_until: _l, ...rest }) => rest)
+  }
+
+  async countByStatus(): Promise<Record<string, number>> {
+    const out: Record<string, number> = {}
+    for (const r of this.rows.values()) out[r.status] = (out[r.status] ?? 0) + 1
+    return out
+  }
+
+  async perRouteStats(): Promise<RouteStats[]> {
+    const byRoute = new Map<string, RouteStats>()
+    for (const r of this.rows.values()) {
+      const s =
+        byRoute.get(r.route_id) ??
+        ({ route_id: r.route_id, counts: {}, last_delivered_at: null, oldest_pending_at: null } as RouteStats)
+      s.counts[r.status] = (s.counts[r.status] ?? 0) + 1
+      if (r.delivered_at && (!s.last_delivered_at || r.delivered_at > s.last_delivered_at)) {
+        s.last_delivered_at = r.delivered_at
+      }
+      if (DUE.includes(r.status) && (!s.oldest_pending_at || r.received_at < s.oldest_pending_at)) {
+        s.oldest_pending_at = r.received_at
+      }
+      byRoute.set(r.route_id, s)
+    }
+    return [...byRoute.values()]
+  }
+
+  async replay(id: string): Promise<boolean> {
+    const r = this.rows.get(id)
+    if (!r || r.status !== 'parked') return false
+    this.requeue(r)
+    return true
+  }
+
+  async replayAllParked(): Promise<number> {
+    let n = 0
+    for (const r of this.rows.values()) {
+      if (r.status !== 'parked') continue
+      this.requeue(r)
+      n++
+    }
+    return n
+  }
+
+  private requeue(r: EventRecord): void {
+    r.status = 'retry'
+    r.attempts = 0
+    r.next_attempt_at = new Date().toISOString()
+    r.last_error = null
+  }
+
+  private isDue(r: EventRecord, now: string): boolean {
+    return DUE.includes(r.status) && r.next_attempt_at !== null && r.next_attempt_at <= now
+  }
+
+  private pick(ids: string[]): EventRecord[] {
+    return ids.map((id) => this.rows.get(id)).filter((r): r is EventRecord => r !== undefined)
+  }
+
+  private clearLease(r: EventRecord): void {
+    r.claimed_by = null
+    r.lease_until = null
+  }
+}
+
+export function memoryStore(): MemoryEventStore {
+  return new MemoryEventStore()
+}