@end-close/relay-sqlite 0.12.0

package/src/repo/events.ts

@@ -0,0 +1,335 @@
+import type { Db } from '../db.js'
+
+import type { EventStatus, RouteStats } from '@end-close/relay'
+
+export type { EventStatus, RouteStats }
+
+/** A raw row of the events table. */
+export interface EventRow {
+  id: number
+  route_id: string
+  source: string
+  event_id: string
+  event_type: string | null
+  payload_enc: Buffer
+  payload_iv: Buffer
+  headers_json: string
+  received_at: string
+  status: EventStatus
+  attempts: number
+  next_attempt_at: string | null
+  delivered_at: string | null
+  bulk_request_id: string | null
+  last_error: string | null
+  idempotency_key: string
+  claimed_by: string | null
+  lease_until: string | null
+}
+
+export interface InsertEvent {
+  route_id: string
+  source: string
+  event_id: string
+  event_type: string | null
+  payload_enc: Buffer
+  payload_iv: Buffer
+  headers_json: string
+  received_at: string
+  status: EventStatus
+  idempotency_key: string
+}
+
+export type EventSummary = Omit<
+  EventRow,
+  'payload_enc' | 'payload_iv' | 'headers_json' | 'idempotency_key' | 'claimed_by' | 'lease_until'
+>
+
+export class EventsRepo {
+  constructor(private db: Db) {}
+
+  /** Returns the new row id, or null if the idempotency key already exists (duplicate delivery). */
+  insert(e: InsertEvent): number | null {
+    const res = this.db
+      .prepare(
+        `INSERT INTO events
+           (route_id, source, event_id, event_type, payload_enc, payload_iv,
+            headers_json, received_at, status, next_attempt_at, idempotency_key)
+         VALUES
+           (@route_id, @source, @event_id, @event_type, @payload_enc, @payload_iv,
+            @headers_json, @received_at, @status, @received_at, @idempotency_key)
+         ON CONFLICT (idempotency_key) DO NOTHING`,
+      )
+      .run(e)
+    return res.changes === 0 ? null : Number(res.lastInsertRowid)
+  }
+
+  /** Claim due events for a route, oldest first, mark them 'delivering' and lease them. */
+  claimDue(routeId: string, now: string, limit: number, owner: string, leaseUntil: string): EventRow[] {
+    const claim = this.db.transaction(() => {
+      const rows = this.db
+        .prepare(
+          `SELECT * FROM events
+           WHERE route_id = ? AND status IN ('pending','retry') AND next_attempt_at <= ?
+           ORDER BY id ASC LIMIT ?`,
+        )
+        .all(routeId, now, limit) as EventRow[]
+      if (rows.length > 0) {
+        const ids = rows.map((r) => r.id)
+        this.db
+          .prepare(
+            `UPDATE events SET status = 'delivering', next_attempt_at = ?, claimed_by = ?, lease_until = ?
+             WHERE id IN (${ids.map(() => '?').join(',')})`,
+          )
+          .run(now, owner, leaseUntil, ...ids)
+        for (const r of rows) {
+          r.status = 'delivering'
+          r.next_attempt_at = now
+          r.claimed_by = owner
+          r.lease_until = leaseUntil
+        }
+      }
+      return rows
+    })
+    return claim()
+  }
+
+  markDelivered(ids: number[], deliveredAt: string, bulkRequestId: string | null): void {
+    if (ids.length === 0) return
+    this.db
+      .prepare(
+        `UPDATE events SET status = 'delivered', delivered_at = ?, bulk_request_id = ?, last_error = NULL,
+           claimed_by = NULL, lease_until = NULL
+         WHERE id IN (${ids.map(() => '?').join(',')})`,
+      )
+      .run(deliveredAt, bulkRequestId, ...ids)
+  }
+
+  markFailed(ids: number[], nextAttemptAt: string, error: string): void {
+    if (ids.length === 0) return
+    this.db
+      .prepare(
+        `UPDATE events SET status = 'retry', attempts = attempts + 1,
+           next_attempt_at = ?, last_error = ?, claimed_by = NULL, lease_until = NULL
+         WHERE id IN (${ids.map(() => '?').join(',')})`,
+      )
+      .run(nextAttemptAt, error.slice(0, 500), ...ids)
+  }
+
+  markParked(ids: number[], error: string): void {
+    if (ids.length === 0) return
+    this.db
+      .prepare(
+        `UPDATE events SET status = 'parked', last_error = ?, claimed_by = NULL, lease_until = NULL
+         WHERE id IN (${ids.map(() => '?').join(',')})`,
+      )
+      .run(error.slice(0, 500), ...ids)
+  }
+
+  /**
+   * Rows stuck in 'delivering' whose lease expired (or that belong to `owner`, or that
+   * predate leases) go back to 'retry'.
+   */
+  recoverDelivering(now: string, owner?: string): number {
+    return this.db
+      .prepare(
+        `UPDATE events SET status = 'retry', next_attempt_at = ?, claimed_by = NULL, lease_until = NULL
+         WHERE status = 'delivering'
+           AND (lease_until IS NULL OR lease_until < ? OR claimed_by = ?)`,
+      )
+      .run(now, now, owner ?? null).changes
+  }
+
+  /**
+   * Return leftover 'delivering' rows to 'retry' without touching already-settled rows.
+   * Used when a dispatch batch throws after claimDue. Increments attempts so a persist
+   * failure after a successful POST backs off instead of tight-looping.
+   */
+  releaseDelivering(ids: number[], nextAttemptAt: string, error: string): number {
+    if (ids.length === 0) return 0
+    return this.db
+      .prepare(
+        `UPDATE events SET status = 'retry', attempts = attempts + 1,
+           next_attempt_at = ?, last_error = ?, claimed_by = NULL, lease_until = NULL
+         WHERE status = 'delivering' AND id IN (${ids.map(() => '?').join(',')})`,
+      )
+      .run(nextAttemptAt, error.slice(0, 500), ...ids).changes
+  }
+
+  routesWithDueEvents(now: string): string[] {
+    return (
+      this.db
+        .prepare(
+          `SELECT DISTINCT route_id FROM events
+           WHERE status IN ('pending','retry') AND next_attempt_at <= ?`,
+        )
+        .all(now) as { route_id: string }[]
+    ).map((r) => r.route_id)
+  }
+
+  countByStatus(): Record<string, number> {
+    const rows = this.db
+      .prepare('SELECT status, COUNT(*) AS n FROM events GROUP BY status')
+      .all() as { status: string; n: number }[]
+    return Object.fromEntries(rows.map((r) => [r.status, r.n]))
+  }
+
+  getById(id: number): EventRow | undefined {
+    return this.db.prepare('SELECT * FROM events WHERE id = ?').get(id) as EventRow | undefined
+  }
+
+  /** Park events that have been retrying longer than maxAgeMs (never silently dropped). */
+  parkExpired(now: string, maxAgeMs: number): number {
+    const cutoff = new Date(Date.parse(now) - maxAgeMs).toISOString()
+    return this.db
+      .prepare(
+        `UPDATE events SET status = 'parked', last_error = 'retry window exhausted'
+         WHERE status = 'retry' AND received_at < ?`,
+      )
+      .run(cutoff).changes
+  }
+
+  perRouteStats(): RouteStats[] {
+    const counts = this.db
+      .prepare('SELECT route_id, status, COUNT(*) AS n FROM events GROUP BY route_id, status')
+      .all() as { route_id: string; status: EventStatus; n: number }[]
+    const extremes = this.db
+      .prepare(
+        `SELECT route_id,
+                MAX(delivered_at) AS last_delivered_at,
+                MIN(CASE WHEN status IN ('pending','retry') THEN received_at END) AS oldest_pending_at
+         FROM events GROUP BY route_id`,
+      )
+      .all() as {
+      route_id: string
+      last_delivered_at: string | null
+      oldest_pending_at: string | null
+    }[]
+    const byRoute = new Map<string, RouteStats>()
+    for (const row of counts) {
+      const stats =
+        byRoute.get(row.route_id) ??
+        ({ route_id: row.route_id, counts: {}, last_delivered_at: null, oldest_pending_at: null } as RouteStats)
+      stats.counts[row.status] = row.n
+      byRoute.set(row.route_id, stats)
+    }
+    for (const row of extremes) {
+      const stats = byRoute.get(row.route_id)
+      if (stats) {
+        stats.last_delivered_at = row.last_delivered_at
+        stats.oldest_pending_at = row.oldest_pending_at
+      }
+    }
+    return [...byRoute.values()]
+  }
+
+  /** Payload-free event listing for the admin plane. */
+  list(filter: { status?: EventStatus; route?: string; limit?: number }): EventSummary[] {
+    const clauses: string[] = []
+    const params: unknown[] = []
+    if (filter.status) {
+      clauses.push('status = ?')
+      params.push(filter.status)
+    }
+    if (filter.route) {
+      clauses.push('route_id = ?')
+      params.push(filter.route)
+    }
+    const where = clauses.length ? `WHERE ${clauses.join(' AND ')}` : ''
+    return this.db
+      .prepare(
+        `SELECT id, route_id, source, event_id, event_type, received_at, status,
+                attempts, next_attempt_at, delivered_at, bulk_request_id, last_error
+         FROM events ${where} ORDER BY id DESC LIMIT ?`,
+      )
+      .all(...params, filter.limit ?? 50) as EventSummary[]
+  }
+
+  /** Re-queue a parked event. Attempts reset so backoff starts fresh. */
+  replay(id: number): boolean {
+    return (
+      this.db
+        .prepare(
+          `UPDATE events SET status = 'retry', attempts = 0, next_attempt_at = ?, last_error = NULL
+           WHERE id = ? AND status = 'parked'`,
+        )
+        .run(new Date().toISOString(), id).changes === 1
+    )
+  }
+
+  replayAllParked(): number {
+    return this.db
+      .prepare(
+        `UPDATE events SET status = 'retry', attempts = 0, next_attempt_at = ?, last_error = NULL
+         WHERE status = 'parked'`,
+      )
+      .run(new Date().toISOString()).changes
+  }
+
+  /**
+   * Retention. Terminal events (delivered / dropped_by_filter) lose their payload after
+   * `deliveredDays` (row kept as the idempotency ledger) and the row itself after
+   * `ledgerDays`. Parked events are never touched — they are unresolved by definition.
+   */
+  prune(now: string, deliveredDays: number, ledgerDays: number): { wiped: number; deleted: number } {
+    let wiped = 0
+    let deleted = 0
+    for (;;) {
+      const batch = this.pruneBatch(now, deliveredDays, ledgerDays, 500)
+      wiped += batch.wiped
+      deleted += batch.deleted
+      if (batch.wiped === 0 && batch.deleted === 0) break
+    }
+    return { wiped, deleted }
+  }
+
+  /**
+   * One short exclusive lock: wipe a limited set of expired payloads, or delete a
+   * limited set of expired ledger rows, not both. Caller loops (and yields) so ingest
+   * and dispatch can run between batches on EFS.
+   */
+  pruneBatch(
+    now: string,
+    deliveredDays: number,
+    ledgerDays: number,
+    limit: number,
+  ): { wiped: number; deleted: number } {
+    const wipeCutoff = new Date(Date.parse(now) - deliveredDays * 86_400_000).toISOString()
+    const deleteCutoff = new Date(Date.parse(now) - ledgerDays * 86_400_000).toISOString()
+    const wipeIds = (
+      this.db
+        .prepare(
+          `SELECT id FROM events
+           WHERE status IN ('delivered','dropped_by_filter')
+             AND received_at < ? AND length(payload_enc) > 0
+           LIMIT ?`,
+        )
+        .all(wipeCutoff, limit) as { id: number }[]
+    ).map((r) => r.id)
+    if (wipeIds.length > 0) {
+      const wiped = this.db
+        .prepare(
+          `UPDATE events SET payload_enc = x'', payload_iv = x'', headers_json = '{}'
+           WHERE id IN (${wipeIds.map(() => '?').join(',')})`,
+        )
+        .run(...wipeIds).changes
+      return { wiped, deleted: 0 }
+    }
+    const deleteIds = (
+      this.db
+        .prepare(
+          `SELECT id FROM events
+           WHERE status IN ('delivered','dropped_by_filter') AND received_at < ?
+           LIMIT ?`,
+        )
+        .all(deleteCutoff, limit) as { id: number }[]
+    ).map((r) => r.id)
+    if (deleteIds.length === 0) return { wiped: 0, deleted: 0 }
+    const deleted = this.db
+      .prepare(`DELETE FROM events WHERE id IN (${deleteIds.map(() => '?').join(',')})`)
+      .run(...deleteIds).changes
+    return { wiped: 0, deleted }
+  }
+}
+
+
+

package/src/repo/kv.ts

@@ -0,0 +1,57 @@
+import type { Killswitch } from '@end-close/relay'
+import type { Db } from '../db.js'
+
+export type GlobalKillswitch = Killswitch
+
+/** kv key prefix for per-route pause flags: `route_paused.<routeId>` = '1'. */
+export const ROUTE_PAUSED_PREFIX = 'route_paused.'
+
+export class KvRepo {
+  constructor(private db: Db) {}
+
+  get(key: string): string | undefined {
+    const row = this.db.prepare('SELECT value FROM kv WHERE key = ?').get(key) as
+      | { value: string }
+      | undefined
+    return row?.value
+  }
+
+  set(key: string, value: string): void {
+    this.db
+      .prepare(
+        `INSERT INTO kv (key, value, updated_at) VALUES (?, ?, ?)
+         ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`,
+      )
+      .run(key, value, new Date().toISOString())
+  }
+
+  delete(key: string): void {
+    this.db.prepare('DELETE FROM kv WHERE key = ?').run(key)
+  }
+
+  globalKillswitch(): GlobalKillswitch {
+    const v = this.get('killswitch.global')
+    return v === 'pause' || v === 'panic' ? v : 'none'
+  }
+
+  setGlobalKillswitch(state: GlobalKillswitch): void {
+    this.set('killswitch.global', state)
+  }
+
+  isRoutePaused(routeId: string): boolean {
+    return this.get(ROUTE_PAUSED_PREFIX + routeId) === '1'
+  }
+
+  setRoutePaused(routeId: string, paused: boolean): void {
+    if (paused) this.set(ROUTE_PAUSED_PREFIX + routeId, '1')
+    else this.delete(ROUTE_PAUSED_PREFIX + routeId)
+  }
+
+  /** Every paused route id in one query. */
+  pausedRoutes(): Set<string> {
+    const rows = this.db
+      .prepare(`SELECT key FROM kv WHERE key LIKE ? AND value = '1'`)
+      .all(ROUTE_PAUSED_PREFIX + '%') as { key: string }[]
+    return new Set(rows.map((r) => r.key.slice(ROUTE_PAUSED_PREFIX.length)))
+  }
+}

package/src/store.ts

@@ -0,0 +1,156 @@
+import type { Db } from './db.js'
+import { EventsRepo, type EventRow } from './repo/events.js'
+import { KvRepo } from './repo/kv.js'
+import { BUSY_RETRY_ATTEMPTS, INGEST_BUSY_RETRY_ATTEMPTS, isSqliteBusy, withBusyRetry } from './busy.js'
+import {
+  noopLogger,
+  StoreError,
+  StoreUnavailableError,
+  type ControlStore,
+  type Logger,
+  type EventRecord,
+  type EventStatus,
+  type EventStore,
+  type EventStoreAdmin,
+  type EventSummary,
+  type InsertResult,
+  type Killswitch,
+  type Lease,
+  type NewEvent,
+  type RouteStats,
+} from '@end-close/relay'
+
+// SQLite implementations of the engine's storage contracts. Lock contention (SQLITE_BUSY,
+// common on network filesystems such as EFS) is retried here and surfaced as
+// StoreUnavailableError; every other failure becomes a StoreError carrying the operation
+// name, with the original error as `cause`.
+
+/** Run one synchronous better-sqlite3 call with busy retry and error classification. */
+export async function runSqlite<T>(
+  op: string,
+  fn: () => T,
+  opts: { attempts?: number; logger?: Logger } = {},
+): Promise<T> {
+  try {
+    return await withBusyRetry(op, fn, { attempts: opts.attempts ?? BUSY_RETRY_ATTEMPTS, logger: opts.logger ?? noopLogger })
+  } catch (err) {
+    if (isSqliteBusy(err)) throw new StoreUnavailableError((err as Error).message, op, { cause: err })
+    if (err instanceof StoreError) throw err
+    throw new StoreError((err as Error).message, op, { cause: err })
+  }
+}
+
+function toRecord(row: EventRow): EventRecord {
+  const { payload_enc, payload_iv, ...rest } = row
+  return {
+    ...rest,
+    id: String(row.id),
+    payload: payload_enc,
+    payload_iv: payload_iv.length === 0 ? null : payload_iv,
+  }
+}
+
+function toSummary<T extends { id: number }>(row: T): Omit<T, 'id'> & { id: string } {
+  return { ...row, id: String(row.id) }
+}
+
+const numericIds = (ids: string[]) => ids.map(Number)
+
+export class SqliteEventStore implements EventStore, EventStoreAdmin {
+  readonly repo: EventsRepo
+  private log: Logger
+
+  constructor(db: Db, opts: { logger?: Logger } = {}) {
+    this.repo = new EventsRepo(db)
+    this.log = opts.logger ?? noopLogger
+  }
+
+  private run<T>(op: string, fn: () => T, attempts?: number): Promise<T> {
+    return runSqlite(op, fn, { logger: this.log, ...(attempts !== undefined ? { attempts } : {}) })
+  }
+
+  async insert(e: NewEvent): Promise<InsertResult> {
+    const { payload, payload_iv, ...rest } = e
+    const row = { ...rest, payload_enc: payload, payload_iv: payload_iv ?? Buffer.alloc(0) }
+    const id = await this.run('insert', () => this.repo.insert(row), INGEST_BUSY_RETRY_ATTEMPTS)
+    return id === null ? { duplicate: true } : { duplicate: false, id: String(id) }
+  }
+  routesWithDueEvents(now: string): Promise<string[]> {
+    return this.run('routesWithDueEvents', () => this.repo.routesWithDueEvents(now))
+  }
+  async claimDue(routeId: string, now: string, limit: number, lease: Lease): Promise<EventRecord[]> {
+    const rows = await this.run('claimDue', () =>
+      this.repo.claimDue(routeId, now, limit, lease.owner, lease.until),
+    )
+    return rows.map(toRecord)
+  }
+  markDelivered(ids: string[], deliveredAt: string, bulkRequestId: string | null): Promise<void> {
+    return this.run('markDelivered', () =>
+      this.repo.markDelivered(numericIds(ids), deliveredAt, bulkRequestId),
+    )
+  }
+  markFailed(ids: string[], nextAttemptAt: string, error: string): Promise<void> {
+    return this.run('markFailed', () => this.repo.markFailed(numericIds(ids), nextAttemptAt, error))
+  }
+  markParked(ids: string[], error: string): Promise<void> {
+    return this.run('markParked', () => this.repo.markParked(numericIds(ids), error))
+  }
+  releaseDelivering(ids: string[], nextAttemptAt: string, error: string): Promise<number> {
+    return this.run('releaseDelivering', () =>
+      this.repo.releaseDelivering(numericIds(ids), nextAttemptAt, error),
+    )
+  }
+  recoverDelivering(now: string, owner?: string): Promise<number> {
+    return this.run('recoverDelivering', () => this.repo.recoverDelivering(now, owner))
+  }
+  parkExpired(now: string, maxAgeMs: number): Promise<number> {
+    return this.run('parkExpired', () => this.repo.parkExpired(now, maxAgeMs))
+  }
+  pruneBatch(now: string, deliveredDays: number, ledgerDays: number, limit: number) {
+    return this.run('prune', () => this.repo.pruneBatch(now, deliveredDays, ledgerDays, limit))
+  }
+
+  // ── admin capability ──
+  async getById(id: string): Promise<EventRecord | undefined> {
+    const row = await this.run('getById', () => this.repo.getById(Number(id)))
+    return row ? toRecord(row) : undefined
+  }
+  async list(filter: { status?: EventStatus; route?: string; limit?: number }): Promise<EventSummary[]> {
+    const rows = await this.run('list', () => this.repo.list(filter))
+    return rows.map(toSummary)
+  }
+  countByStatus(): Promise<Record<string, number>> {
+    return this.run('countByStatus', () => this.repo.countByStatus())
+  }
+  perRouteStats(): Promise<RouteStats[]> {
+    return this.run('perRouteStats', () => this.repo.perRouteStats())
+  }
+  replay(id: string): Promise<boolean> {
+    return this.run('replay', () => this.repo.replay(Number(id)))
+  }
+  replayAllParked(): Promise<number> {
+    return this.run('replayAllParked', () => this.repo.replayAllParked())
+  }
+}
+
+export class SqliteControlStore implements ControlStore {
+  readonly kv: KvRepo
+  private log: Logger
+  constructor(db: Db, opts: { logger?: Logger } = {}) {
+    this.kv = new KvRepo(db)
+    this.log = opts.logger ?? noopLogger
+  }
+  getKillswitch(): Promise<Killswitch> {
+    return runSqlite('killswitch', () => this.kv.globalKillswitch(), { logger: this.log })
+  }
+  setKillswitch(state: Killswitch): Promise<void> {
+    return runSqlite('setKillswitch', () => this.kv.setGlobalKillswitch(state), { logger: this.log })
+  }
+  isRoutePaused(routeId: string): Promise<boolean> {
+    return runSqlite('isPaused', () => this.kv.isRoutePaused(routeId), { logger: this.log })
+  }
+  setRoutePaused(routeId: string, paused: boolean): Promise<void> {
+    return runSqlite('setPaused', () => this.kv.setRoutePaused(routeId, paused), { logger: this.log })
+  }
+}
+