livetap 0.1.0

package/src/server/connections/mqtt.ts

@@ -0,0 +1,104 @@
+/**
+ * MQTT Subscriber — connects to an MQTT broker and writes messages
+ * to a Redis Stream with rolling retention.
+ */
+
+import mqtt from 'mqtt'
+import type Redis from 'ioredis'
+import type { MqttConnectionConfig, Subscriber, ConnectionStatus } from '../types.js'
+
+export interface MqttSubscriberOpts {
+  config: MqttConnectionConfig
+  streamKey: string
+  redis: Redis
+  retentionMs?: number
+  onMessage?: () => void
+}
+
+export class MqttSubscriber implements Subscriber {
+  private client: mqtt.MqttClient | null = null
+  private config: MqttConnectionConfig
+  private streamKey: string
+  private redis: Redis
+  private retentionMs: number
+  private onMessage: (() => void) | undefined
+  private state: ConnectionStatus['runtimeState'] = 'disconnected'
+  private error?: string
+
+  constructor(opts: MqttSubscriberOpts) {
+    this.config = opts.config
+    this.streamKey = opts.streamKey
+    this.redis = opts.redis
+    this.retentionMs = opts.retentionMs ?? 5 * 60 * 1000
+    this.onMessage = opts.onMessage
+  }
+
+  async start(): Promise<void> {
+    const { broker, port, tls, credentials, topics } = this.config
+    const protocol = tls ? 'mqtts' : 'mqtt'
+    const url = `${protocol}://${broker}:${port}`
+
+    this.state = 'disconnected'
+
+    this.client = mqtt.connect(url, {
+      username: credentials.username || undefined,
+      password: credentials.password || undefined,
+      rejectUnauthorized: tls,
+    })
+
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        this.state = 'error'
+        this.error = 'Connection timeout'
+        reject(new Error(`MQTT connection to ${url} timed out`))
+      }, 15_000)
+
+      this.client!.on('connect', () => {
+        clearTimeout(timeout)
+        this.state = 'connected'
+        this.error = undefined
+        this.client!.subscribe(topics, (err) => {
+          if (err) {
+            this.state = 'error'
+            this.error = err.message
+            reject(err)
+          } else {
+            resolve()
+          }
+        })
+      })
+
+      this.client!.on('error', (err) => {
+        this.state = 'error'
+        this.error = err.message
+      })
+
+      this.client!.on('reconnect', () => {
+        this.state = 'reconnecting'
+      })
+
+      this.client!.on('message', async (topic, payload) => {
+        try {
+          await this.redis.xadd(this.streamKey, '*', 'topic', topic, 'payload', payload.toString())
+          const minId = Date.now() - this.retentionMs
+          await this.redis.xtrim(this.streamKey, 'MINID', '~', minId.toString())
+          this.onMessage?.()
+        } catch {
+          // Redis write failure — log but don't crash
+        }
+      })
+    })
+  }
+
+  async stop(): Promise<void> {
+    if (this.client) {
+      await this.client.endAsync()
+      this.client = null
+    }
+    this.state = 'disconnected'
+  }
+
+  getStatus() {
+    return { runtimeState: this.state, error: this.error }
+  }
+}

package/src/server/connections/webhook.ts

@@ -0,0 +1,54 @@
+/**
+ * Webhook Ingestor — HTTP endpoint that accepts POSTs and writes
+ * payloads to a Redis Stream with rolling retention.
+ */
+
+import type Redis from 'ioredis'
+import type { Subscriber, ConnectionStatus } from '../types.js'
+
+export interface WebhookIngestorOpts {
+  streamKey: string
+  redis: Redis
+  retentionMs?: number
+  onMessage?: () => void
+}
+
+export class WebhookIngestor implements Subscriber {
+  private streamKey: string
+  private redis: Redis
+  private retentionMs: number
+  private onMessage: (() => void) | undefined
+  private state: ConnectionStatus['runtimeState'] = 'disconnected'
+
+  constructor(opts: WebhookIngestorOpts) {
+    this.streamKey = opts.streamKey
+    this.redis = opts.redis
+    this.retentionMs = opts.retentionMs ?? 5 * 60 * 1000
+    this.onMessage = opts.onMessage
+  }
+
+  async start(): Promise<void> {
+    this.state = 'connected'
+  }
+
+  async stop(): Promise<void> {
+    this.state = 'disconnected'
+  }
+
+  /**
+   * Ingest a webhook payload. Called by the HTTP API route handler.
+   */
+  async ingest(body: string, headers?: Record<string, string>): Promise<void> {
+    const fields: string[] = ['payload', body]
+    if (headers?.['content-type']) fields.push('content_type', headers['content-type'])
+
+    await this.redis.xadd(this.streamKey, '*', ...fields)
+    const minId = Date.now() - this.retentionMs
+    await this.redis.xtrim(this.streamKey, 'MINID', '~', minId.toString())
+    this.onMessage?.()
+  }
+
+  getStatus() {
+    return { runtimeState: this.state }
+  }
+}

package/src/server/connections/websocket.ts

@@ -0,0 +1,154 @@
+/**
+ * WebSocket Subscriber — connects to a remote WebSocket and writes
+ * messages to a Redis Stream with rolling retention + reconnection.
+ */
+
+import type Redis from 'ioredis'
+import type { WebSocketConnectionConfig, Subscriber, ConnectionStatus } from '../types.js'
+
+export interface WsSubscriberOpts {
+  config: WebSocketConnectionConfig
+  streamKey: string
+  redis: Redis
+  retentionMs?: number
+  onMessage?: () => void
+}
+
+export class WsSubscriber implements Subscriber {
+  private ws: WebSocket | null = null
+  private config: WebSocketConnectionConfig
+  private streamKey: string
+  private redis: Redis
+  private retentionMs: number
+  private onMessage: (() => void) | undefined
+  private state: ConnectionStatus['runtimeState'] = 'disconnected'
+  private error?: string
+  private retryCount = 0
+  private retryTimer: ReturnType<typeof setTimeout> | null = null
+  private pingTimer: ReturnType<typeof setInterval> | null = null
+  private stopped = false
+
+  constructor(opts: WsSubscriberOpts) {
+    this.config = opts.config
+    this.streamKey = opts.streamKey
+    this.redis = opts.redis
+    this.retentionMs = opts.retentionMs ?? 5 * 60 * 1000
+    this.onMessage = opts.onMessage
+  }
+
+  async start(): Promise<void> {
+    this.stopped = false
+    this.connect()
+  }
+
+  private connect() {
+    this.state = this.retryCount === 0 ? 'disconnected' : 'reconnecting'
+
+    this.ws = new WebSocket(this.config.url)
+
+    this.ws.onopen = () => {
+      this.state = 'connected'
+      this.error = undefined
+      this.retryCount = 0
+
+      if (this.config.handshake) {
+        this.ws!.send(this.config.handshake)
+      }
+
+      const pingMs = this.config.pingIntervalMs ?? 30_000
+      if (pingMs > 0) {
+        this.pingTimer = setInterval(() => {
+          if (this.ws?.readyState === WebSocket.OPEN) {
+            this.ws.send('ping')
+          }
+        }, pingMs)
+      }
+    }
+
+    this.ws.onmessage = async (event) => {
+      try {
+        const fields = this.parseMessage(event.data)
+        await this.redis.xadd(this.streamKey, '*', ...Object.entries(fields).flat())
+        const minId = Date.now() - this.retentionMs
+        await this.redis.xtrim(this.streamKey, 'MINID', '~', minId.toString())
+        this.onMessage?.()
+      } catch {
+        // Redis write failure
+      }
+    }
+
+    this.ws.onclose = () => {
+      this.cleanup()
+      if (!this.stopped && this.config.reconnect?.enabled !== false) {
+        this.scheduleReconnect()
+      } else {
+        this.state = 'disconnected'
+      }
+    }
+
+    this.ws.onerror = () => {
+      this.state = 'error'
+    }
+  }
+
+  private parseMessage(data: string | Buffer | ArrayBuffer): Record<string, string> {
+    if (typeof data === 'string') {
+      try {
+        JSON.parse(data) // validate JSON
+        return { payload: data, format: 'json' }
+      } catch {
+        return { payload: data, format: 'text' }
+      }
+    }
+    const buf = data instanceof ArrayBuffer ? Buffer.from(data) : data as Buffer
+    const fmt = this.config.binaryFormat ?? 'base64'
+    if (fmt === 'json') {
+      try {
+        const text = buf.toString('utf-8')
+        JSON.parse(text)
+        return { payload: text, format: 'json' }
+      } catch { /* fall through */ }
+    }
+    if (fmt === 'text') {
+      return { payload: buf.toString('utf-8'), format: 'text' }
+    }
+    return { payload: buf.toString('base64'), format: 'base64' }
+  }
+
+  private scheduleReconnect() {
+    const maxRetries = this.config.reconnect?.maxRetries ?? Infinity
+    if (this.retryCount >= maxRetries) {
+      this.state = 'error'
+      this.error = `Max retries (${maxRetries}) exceeded`
+      return
+    }
+
+    const base = this.config.reconnect?.initialDelayMs ?? 1000
+    const max = this.config.reconnect?.maxDelayMs ?? 30000
+    const delay = Math.min(base * Math.pow(2, this.retryCount), max)
+    const jitter = delay * 0.2 * Math.random()
+
+    this.retryCount++
+    this.state = 'reconnecting'
+    this.retryTimer = setTimeout(() => this.connect(), delay + jitter)
+  }
+
+  private cleanup() {
+    if (this.pingTimer) { clearInterval(this.pingTimer); this.pingTimer = null }
+  }
+
+  async stop(): Promise<void> {
+    this.stopped = true
+    if (this.retryTimer) { clearTimeout(this.retryTimer); this.retryTimer = null }
+    this.cleanup()
+    if (this.ws) {
+      this.ws.close()
+      this.ws = null
+    }
+    this.state = 'disconnected'
+  }
+
+  getStatus() {
+    return { runtimeState: this.state, error: this.error }
+  }
+}

package/src/server/index.ts

@@ -0,0 +1,255 @@
+/**
+ * livetap daemon — HTTP API on :8788
+ * Manages connections, streams, and watchers.
+ */
+
+import { startRedis, type RedisManager } from './redis.js'
+import { ConnectionManager } from './connection-manager.js'
+import { WatcherManager } from './watchers/manager.js'
+import type { WatcherAlert } from './watchers/types.js'
+import type { StreamEntry } from './types.js'
+
+const PORT = parseInt(process.env.LIVETAP_PORT || '8788')
+
+let redis: RedisManager
+let manager: ConnectionManager
+let watchers: WatcherManager
+
+// SSE clients for alert delivery
+const sseClients = new Set<(chunk: string) => void>()
+
+function broadcastSSE(alert: WatcherAlert) {
+  const data = `data: ${JSON.stringify(alert)}\n\n`
+  for (const emit of sseClients) {
+    try { emit(data) } catch { /* client gone */ }
+  }
+}
+
+async function boot() {
+  redis = await startRedis()
+  console.error(`[livetap] Redis started on port ${redis.port}`)
+
+  manager = new ConnectionManager(redis.client, redis.url)
+  watchers = new WatcherManager(redis.client, redis.url, broadcastSSE)
+
+  Bun.serve({
+    port: PORT,
+    hostname: '127.0.0.1',
+    async fetch(req) {
+      const url = new URL(req.url)
+      const method = req.method
+
+      // --- SSE events stream ---
+      if (method === 'GET' && url.pathname === '/events') {
+        const stream = new ReadableStream({
+          start(ctrl) {
+            const encoder = new TextEncoder()
+            ctrl.enqueue(encoder.encode(': connected\n\n'))
+            const emit = (chunk: string) => ctrl.enqueue(encoder.encode(chunk))
+            sseClients.add(emit)
+            req.signal.addEventListener('abort', () => sseClients.delete(emit))
+          },
+        })
+        return new Response(stream, {
+          headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' },
+        })
+      }
+
+      // --- Health ---
+      if (method === 'GET' && url.pathname === '/status') {
+        return json({
+          status: 'running',
+          port: PORT,
+          redisPort: redis.port,
+          connections: manager.list(),
+          uptime: process.uptime(),
+        })
+      }
+
+      // --- Create connection ---
+      if (method === 'POST' && url.pathname === '/connections') {
+        try {
+          const body = await req.json()
+          const record = await manager.create(body.config ?? body, body.name)
+          return json({
+            connectionId: record.id,
+            type: record.config.type,
+            streamKey: record.streamKey,
+            status: 'connected',
+            ...(record.config.type === 'webhook' && {
+              ingestUrl: `http://127.0.0.1:${PORT}/connections/${record.id}/ingest`,
+            }),
+          }, 201)
+        } catch (err) {
+          return json({ error: (err as Error).message }, 400)
+        }
+      }
+
+      // --- List connections ---
+      if (method === 'GET' && url.pathname === '/connections') {
+        return json(manager.list())
+      }
+
+      // --- Connection routes: /connections/:id ---
+      const connMatch = url.pathname.match(/^\/connections\/([^/]+)$/)
+      if (connMatch) {
+        const id = connMatch[1]
+        if (method === 'GET') {
+          const status = manager.getStatus(id)
+          if (!status) return json({ error: 'Connection not found' }, 404)
+          return json(status)
+        }
+        if (method === 'DELETE') {
+          const ok = await manager.destroy(id)
+          if (!ok) return json({ error: 'Connection not found' }, 404)
+          return json({ deleted: id })
+        }
+      }
+
+      // --- Webhook ingest: /connections/:id/ingest ---
+      const ingestMatch = url.pathname.match(/^\/connections\/([^/]+)\/ingest$/)
+      if (ingestMatch && method === 'POST') {
+        const id = ingestMatch[1]
+        const ingestor = manager.getWebhookIngestor(id)
+        if (!ingestor) return json({ error: 'Webhook connection not found' }, 404)
+        const body = await req.text()
+        await ingestor.ingest(body, Object.fromEntries(req.headers.entries()))
+        return json({ ok: true })
+      }
+
+      // --- Read stream: /connections/:id/stream ---
+      const streamMatch = url.pathname.match(/^\/connections\/([^/]+)\/stream$/)
+      if (streamMatch && method === 'GET') {
+        const id = streamMatch[1]
+        const record = manager.get(id)
+        if (!record) return json({ error: 'Connection not found' }, 404)
+
+        const backfillSeconds = parseInt(url.searchParams.get('backfillSeconds') ?? '300')
+        const maxEntries = parseInt(url.searchParams.get('maxEntries') ?? '50')
+
+        try {
+          const entries = await readBackfill(record.streamKey, backfillSeconds, maxEntries)
+          return json({ entries, totalEntries: entries.length })
+        } catch (err) {
+          return json({ error: (err as Error).message }, 500)
+        }
+      }
+
+      // --- Create watcher ---
+      if (method === 'POST' && url.pathname === '/watchers') {
+        try {
+          const body = await req.json()
+          const connId = body.connectionId
+          const record = manager.get(connId)
+          if (!record) return json({ error: 'Connection not found' }, 404)
+
+          const info = await watchers.create(
+            connId,
+            record.streamKey,
+            body.conditions,
+            body.match ?? 'all',
+            body.action ?? 'channel_alert',
+            body.cooldown ?? 60,
+          )
+          return json({ id: info.id, status: info.status, expression: `${info.conditions.map(c => `${c.field} ${c.op} ${c.value}`).join(info.match === 'all' ? ' AND ' : ' OR ')}` }, 201)
+        } catch (err) {
+          return json({ error: (err as Error).message }, 400)
+        }
+      }
+
+      // --- List watchers ---
+      if (method === 'GET' && url.pathname === '/watchers') {
+        const connId = url.searchParams.get('connectionId') || undefined
+        return json(await watchers.list(connId))
+      }
+
+      // --- Watcher routes: /watchers/:id ---
+      const watcherMatch = url.pathname.match(/^\/watchers\/([^/]+)$/)
+      if (watcherMatch) {
+        const id = watcherMatch[1]
+        if (method === 'GET') {
+          const info = await watchers.get(id)
+          if (!info) return json({ error: 'Watcher not found' }, 404)
+          return json(info)
+        }
+        if (method === 'PUT') {
+          const body = await req.json()
+          const info = await watchers.get(id)
+          if (!info) return json({ error: 'Watcher not found' }, 404)
+          const record = manager.get(info.connectionId)
+          const streamKey = record?.streamKey ?? null
+          try {
+            const updated = await watchers.update(id, streamKey, body)
+            return json(updated)
+          } catch (err) {
+            return json({ error: (err as Error).message }, 400)
+          }
+        }
+        if (method === 'DELETE') {
+          const ok = await watchers.delete(id)
+          if (!ok) return json({ error: 'Watcher not found' }, 404)
+          return json({ deleted: id })
+        }
+      }
+
+      // --- Watcher logs: /watchers/:id/logs ---
+      const logsMatch = url.pathname.match(/^\/watchers\/([^/]+)\/logs$/)
+      if (logsMatch && method === 'GET') {
+        const id = logsMatch[1]
+        const lines = parseInt(url.searchParams.get('lines') ?? '50')
+        const logs = await watchers.getLogs(id, lines)
+        return json({ id, logs })
+      }
+
+      // --- Restart watcher: /watchers/:id/restart ---
+      const restartMatch = url.pathname.match(/^\/watchers\/([^/]+)\/restart$/)
+      if (restartMatch && method === 'POST') {
+        const id = restartMatch[1]
+        const info = await watchers.get(id)
+        if (!info) return json({ error: 'Watcher not found' }, 404)
+        const record = manager.get(info.connectionId)
+        const streamKey = record?.streamKey ?? null
+        const ok = await watchers.restart(id, streamKey)
+        if (!ok) return json({ error: 'Watcher not found' }, 404)
+        return json({ restarted: id })
+      }
+
+      return json({ error: 'Not found' }, 404)
+    },
+  })
+
+  console.error(`[livetap] daemon listening on http://127.0.0.1:${PORT}`)
+}
+
+async function readBackfill(streamKey: string, backfillSeconds: number, maxEntries: number): Promise<StreamEntry[]> {
+  const since = (Date.now() - backfillSeconds * 1000).toString()
+  const raw = await redis.client.xrange(streamKey, since, '+', 'COUNT', maxEntries)
+  return raw.map(([id, fieldArray]) => {
+    const fields: Record<string, string> = {}
+    for (let i = 0; i < fieldArray.length; i += 2) {
+      fields[fieldArray[i]] = fieldArray[i + 1]
+    }
+    return { id, fields, ts: parseInt(id.split('-')[0]) }
+  })
+}
+
+function json(data: unknown, status = 200) {
+  return new Response(JSON.stringify(data, null, 2), {
+    status,
+    headers: { 'Content-Type': 'application/json' },
+  })
+}
+
+// Graceful shutdown
+async function shutdown() {
+  console.error('[livetap] shutting down...')
+  await watchers.stopAll()
+  await manager.destroyAll()
+  await redis.stop()
+  process.exit(0)
+}
+
+process.on('SIGTERM', shutdown)
+process.on('SIGINT', shutdown)
+
+boot()

package/src/server/redis.ts

@@ -0,0 +1,62 @@
+/**
+ * Embedded Redis lifecycle manager.
+ * Uses the `redis-server` npm package which bundles a Redis binary.
+ */
+
+import RedisServer from 'redis-server'
+import Redis from 'ioredis'
+
+export interface RedisManager {
+  port: number
+  url: string
+  client: Redis
+  stop(): Promise<void>
+}
+
+/**
+ * Start an embedded redis-server on the given port (or random free port).
+ * Returns a manager with a connected ioredis client.
+ */
+export async function startRedis(preferredPort?: number): Promise<RedisManager> {
+  const port = preferredPort ?? await findFreePort()
+
+  const server = new RedisServer({ port })
+
+  await new Promise<void>((resolve, reject) => {
+    server.open((err: Error | null) => {
+      if (err) reject(err)
+      else resolve()
+    })
+  })
+
+  const url = `redis://127.0.0.1:${port}`
+  const client = new Redis(url, { maxRetriesPerRequest: 3, lazyConnect: false })
+
+  await client.ping()
+
+  return {
+    port,
+    url,
+    client,
+    async stop() {
+      client.disconnect()
+      await new Promise<void>((resolve, reject) => {
+        server.close((err: Error | null) => {
+          if (err) reject(err)
+          else resolve()
+        })
+      })
+    },
+  }
+}
+
+async function findFreePort(): Promise<number> {
+  const server = Bun.listen({
+    hostname: '127.0.0.1',
+    port: 0,
+    socket: { data() {} },
+  })
+  const port = server.port
+  server.stop()
+  return port
+}