livetap 0.1.5 → 0.2.1

package/src/server/connections/webhook.ts

@@ -1,28 +1,28 @@
 /**
  * Webhook Ingestor — HTTP endpoint that accepts POSTs and writes
- * payloads to a Redis Stream with rolling retention.
+ * payloads to a stream with rolling retention.
  */
 
-import type Redis from 'ioredis'
+import type { StreamStore } from '../stream-store.js'
 import type { Subscriber, ConnectionStatus } from '../types.js'
 
 export interface WebhookIngestorOpts {
   streamKey: string
-  redis: Redis
+  store: StreamStore
   retentionMs?: number
   onMessage?: () => void
 }
 
 export class WebhookIngestor implements Subscriber {
   private streamKey: string
-  private redis: Redis
+  private store: StreamStore
   private retentionMs: number
   private onMessage: (() => void) | undefined
   private state: ConnectionStatus['runtimeState'] = 'disconnected'
 
   constructor(opts: WebhookIngestorOpts) {
     this.streamKey = opts.streamKey
-    this.redis = opts.redis
+    this.store = opts.store
     this.retentionMs = opts.retentionMs ?? 5 * 60 * 1000
     this.onMessage = opts.onMessage
   }
@@ -39,12 +39,11 @@ export class WebhookIngestor implements Subscriber {
    * Ingest a webhook payload. Called by the HTTP API route handler.
    */
   async ingest(body: string, headers?: Record<string, string>): Promise<void> {
-    const fields: string[] = ['payload', body]
-    if (headers?.['content-type']) fields.push('content_type', headers['content-type'])
+    const fields: Record<string, string> = { payload: body }
+    if (headers?.['content-type']) fields.content_type = headers['content-type']
 
-    await this.redis.xadd(this.streamKey, '*', ...fields)
-    const minId = Date.now() - this.retentionMs
-    await this.redis.xtrim(this.streamKey, 'MINID', '~', minId.toString())
+    this.store.append(this.streamKey, fields)
+    this.store.trim(this.streamKey, Date.now() - this.retentionMs)
     this.onMessage?.()
   }
 

package/src/server/connections/websocket.ts

@@ -1,15 +1,15 @@
 /**
  * WebSocket Subscriber — connects to a remote WebSocket and writes
- * messages to a Redis Stream with rolling retention + reconnection.
+ * messages to a stream with rolling retention + reconnection.
  */
 
-import type Redis from 'ioredis'
+import type { StreamStore } from '../stream-store.js'
 import type { WebSocketConnectionConfig, Subscriber, ConnectionStatus } from '../types.js'
 
 export interface WsSubscriberOpts {
   config: WebSocketConnectionConfig
   streamKey: string
-  redis: Redis
+  store: StreamStore
   retentionMs?: number
   onMessage?: () => void
 }
@@ -18,7 +18,7 @@ export class WsSubscriber implements Subscriber {
   private ws: WebSocket | null = null
   private config: WebSocketConnectionConfig
   private streamKey: string
-  private redis: Redis
+  private store: StreamStore
   private retentionMs: number
   private onMessage: (() => void) | undefined
   private state: ConnectionStatus['runtimeState'] = 'disconnected'
@@ -31,7 +31,7 @@ export class WsSubscriber implements Subscriber {
   constructor(opts: WsSubscriberOpts) {
     this.config = opts.config
     this.streamKey = opts.streamKey
-    this.redis = opts.redis
+    this.store = opts.store
     this.retentionMs = opts.retentionMs ?? 5 * 60 * 1000
     this.onMessage = opts.onMessage
   }
@@ -65,15 +65,14 @@ export class WsSubscriber implements Subscriber {
       }
     }
 
-    this.ws.onmessage = async (event) => {
+    this.ws.onmessage = (event) => {
       try {
         const fields = this.parseMessage(event.data)
-        await this.redis.xadd(this.streamKey, '*', ...Object.entries(fields).flat())
-        const minId = Date.now() - this.retentionMs
-        await this.redis.xtrim(this.streamKey, 'MINID', '~', minId.toString())
+        this.store.append(this.streamKey, fields)
+        this.store.trim(this.streamKey, Date.now() - this.retentionMs)
         this.onMessage?.()
       } catch {
-        // Redis write failure
+        // Write failure
       }
     }
 

package/src/server/index.ts

@@ -3,7 +3,7 @@
  * Manages connections, streams, and watchers.
  */
 
-import { startRedis, type RedisManager } from './redis.js'
+import { StreamStore } from './stream-store.js'
 import { ConnectionManager } from './connection-manager.js'
 import { WatcherManager } from './watchers/manager.js'
 import type { WatcherAlert } from './watchers/types.js'
@@ -11,7 +11,7 @@ import type { StreamEntry } from './types.js'
 
 const PORT = parseInt(process.env.LIVETAP_PORT || '8788')
 
-let redis: RedisManager
+let store: StreamStore
 let manager: ConnectionManager
 let watchers: WatcherManager
 
@@ -26,11 +26,10 @@ function broadcastSSE(alert: WatcherAlert) {
 }
 
 async function boot() {
-  redis = await startRedis()
-  console.error(`[livetap] Redis started on port ${redis.port}`)
+  store = new StreamStore()
 
-  manager = new ConnectionManager(redis.client, redis.url)
-  watchers = new WatcherManager(redis.client, redis.url, broadcastSSE)
+  manager = new ConnectionManager(store)
+  watchers = new WatcherManager(store, broadcastSSE)
 
   Bun.serve({
     port: PORT,
@@ -60,7 +59,6 @@ async function boot() {
         return json({
           status: 'running',
           port: PORT,
-          redisPort: redis.port,
           connections: manager.list(),
           uptime: process.uptime(),
         })
@@ -128,7 +126,7 @@ async function boot() {
         const maxEntries = parseInt(url.searchParams.get('maxEntries') ?? '50')
 
         try {
-          const entries = await readBackfill(record.streamKey, backfillSeconds, maxEntries)
+          const entries = readBackfill(record.streamKey, backfillSeconds, maxEntries)
           return json({ entries, totalEntries: entries.length })
         } catch (err) {
           return json({ error: (err as Error).message }, 500)
@@ -221,16 +219,13 @@ async function boot() {
   console.error(`[livetap] daemon listening on http://127.0.0.1:${PORT}`)
 }
 
-async function readBackfill(streamKey: string, backfillSeconds: number, maxEntries: number): Promise<StreamEntry[]> {
-  const since = (Date.now() - backfillSeconds * 1000).toString()
-  const raw = await redis.client.xrange(streamKey, since, '+', 'COUNT', maxEntries)
-  return raw.map(([id, fieldArray]) => {
-    const fields: Record<string, string> = {}
-    for (let i = 0; i < fieldArray.length; i += 2) {
-      fields[fieldArray[i]] = fieldArray[i + 1]
-    }
-    return { id, fields, ts: parseInt(id.split('-')[0]) }
-  })
+function readBackfill(streamKey: string, backfillSeconds: number, maxEntries: number): StreamEntry[] {
+  const sinceMs = Date.now() - backfillSeconds * 1000
+  return store.range(streamKey, sinceMs, maxEntries).map((e) => ({
+    id: e.id,
+    fields: e.fields,
+    ts: parseInt(e.id.split('-')[0]),
+  }))
 }
 
 function json(data: unknown, status = 200) {
@@ -245,7 +240,7 @@ async function shutdown() {
   console.error('[livetap] shutting down...')
   await watchers.stopAll()
   await manager.destroyAll()
-  await redis.stop()
+  store.stop()
   process.exit(0)
 }
 

package/src/server/stream-store.ts

@@ -0,0 +1,128 @@
+/**
+ * In-memory stream store — replaces Redis for stream buffering.
+ * Provides append, range, trim, len, del, and subscribe (EventEmitter).
+ */
+
+import { EventEmitter } from 'events'
+
+export interface StreamEntry {
+  id: string // "{timestamp}-{seq}"
+  fields: Record<string, string>
+}
+
+export class StreamStore {
+  private streams = new Map<string, StreamEntry[]>()
+  private seqCounters = new Map<string, number>()
+  private emitter = new EventEmitter()
+
+  constructor() {
+    this.emitter.setMaxListeners(1000)
+  }
+
+  /**
+   * Append an entry to a stream. Returns the generated ID.
+   * Equivalent to Redis XADD streamKey * field1 val1 field2 val2
+   */
+  append(streamKey: string, fields: Record<string, string>): string {
+    const ts = Date.now()
+    const seq = this.seqCounters.get(streamKey) ?? 0
+    this.seqCounters.set(streamKey, seq + 1)
+    const id = `${ts}-${seq}`
+
+    const entry: StreamEntry = { id, fields }
+
+    let stream = this.streams.get(streamKey)
+    if (!stream) {
+      stream = []
+      this.streams.set(streamKey, stream)
+    }
+    stream.push(entry)
+
+    this.emitter.emit(streamKey, entry)
+    return id
+  }
+
+  /**
+   * Read entries from a stream in a time range.
+   * Equivalent to Redis XRANGE streamKey since + COUNT count
+   */
+  range(streamKey: string, sinceMs: number, count?: number): StreamEntry[] {
+    const stream = this.streams.get(streamKey)
+    if (!stream) return []
+
+    const sinceStr = String(sinceMs)
+    const filtered = stream.filter((e) => {
+      const entryTs = e.id.split('-')[0]
+      return entryTs >= sinceStr
+    })
+
+    return count ? filtered.slice(0, count) : filtered
+  }
+
+  /**
+   * Trim entries older than minTs.
+   * Equivalent to Redis XTRIM streamKey MINID ~ minTs
+   */
+  trim(streamKey: string, minTs: number): void {
+    const stream = this.streams.get(streamKey)
+    if (!stream) return
+
+    const minStr = String(minTs)
+    // Find first entry that's >= minTs
+    let cutIndex = 0
+    for (let i = 0; i < stream.length; i++) {
+      const entryTs = stream[i].id.split('-')[0]
+      if (entryTs >= minStr) break
+      cutIndex = i + 1
+    }
+    if (cutIndex > 0) {
+      stream.splice(0, cutIndex)
+    }
+  }
+
+  /**
+   * Get entry count for a stream.
+   * Equivalent to Redis XLEN streamKey
+   */
+  len(streamKey: string): number {
+    return this.streams.get(streamKey)?.length ?? 0
+  }
+
+  /**
+   * Delete a stream entirely.
+   * Equivalent to Redis DEL streamKey
+   */
+  del(streamKey: string): void {
+    this.streams.delete(streamKey)
+    this.seqCounters.delete(streamKey)
+    this.emitter.removeAllListeners(streamKey)
+  }
+
+  /**
+   * Subscribe to new entries on a stream.
+   * Replaces Redis XREAD BLOCK — zero-latency, event-driven.
+   * Returns an unsubscribe function.
+   */
+  subscribe(streamKey: string, callback: (entry: StreamEntry) => void): () => void {
+    this.emitter.on(streamKey, callback)
+    return () => {
+      this.emitter.off(streamKey, callback)
+    }
+  }
+
+  /**
+   * Check if a stream exists.
+   */
+  has(streamKey: string): boolean {
+    return this.streams.has(streamKey)
+  }
+
+  /**
+   * Stop the store — clean up all data and listeners.
+   */
+  stop(): void {
+    this.streams.clear()
+    this.seqCounters.clear()
+    this.emitter.removeAllListeners()
+  }
+}

package/src/server/watchers/engine.ts

@@ -6,10 +6,15 @@
 import type { WatcherCondition, WatcherDefinition } from './types.js'
 
 /**
- * Resolve a dot-separated path into a nested object.
- * Returns undefined if any segment is missing.
+ * Resolve a field path into an object.
+ * First tries the path as a literal key (handles keys like "2.8.0"),
+ * then falls back to dot-separated nested access (handles "sensors.temperature.value").
  */
 export function resolveDotPath(obj: any, path: string): any {
+  if (obj == null) return undefined
+  // Try literal key first (handles OBIS codes like "2.8.0", "32.7.0")
+  if (path in obj) return obj[path]
+  // Fall back to dot-path traversal
   return path.split('.').reduce((o, k) => o?.[k], obj)
 }
 

package/src/server/watchers/manager.ts

@@ -1,13 +1,14 @@
 /**
  * Watcher Manager — CRUD + evaluation loops for expression watchers.
- * Stores definitions in Redis hashes. Runs evaluator loops in-process.
+ * Stores definitions in-memory. Evaluates via StreamStore subscriptions.
  */
 
-import Redis from 'ioredis'
 import { mkdirSync, appendFileSync, statSync, writeFileSync, readFileSync, unlinkSync } from 'fs'
 import { resolve } from 'path'
 import { homedir } from 'os'
+import type { StreamStore, StreamEntry } from '../stream-store.js'
 import type { WatcherCondition, WatcherDefinition, WatcherInfo, WatcherAlert, WatcherAction } from './types.js'
+import { resolveDotPath } from './engine.js'
 import { VALID_OPS } from './types.js'
 import { evaluateWatcher, extractMatchedValues, formatExpression } from './engine.js'
 
@@ -24,15 +25,13 @@ const MAX_LOG_SIZE = 512 * 1024 // 512KB
 export type AlertCallback = (alert: WatcherAlert) => void
 
 export class WatcherManager {
-  private redisUrl: string
-  private loops = new Map<string, { abort: AbortController; reader: Redis }>()
+  private store: StreamStore
+  private subscriptions = new Map<string, () => void>() // watcherId → unsubscribe
   private info = new Map<string, WatcherInfo>()
   private onAlert: AlertCallback
-  private redis: Redis
 
-  constructor(redis: Redis, redisUrl: string, onAlert: AlertCallback) {
-    this.redis = redis
-    this.redisUrl = redisUrl
+  constructor(store: StreamStore, onAlert: AlertCallback) {
+    this.store = store
     this.onAlert = onAlert
     mkdirSync(LOG_DIR, { recursive: true })
   }
@@ -73,9 +72,6 @@ export class WatcherManager {
       updatedAt: now,
     }
 
-    // Store in flat hash (globally unique watcher IDs)
-    await this.redis.hset('livetap:watchers', id, JSON.stringify(def))
-
     const info: WatcherInfo = { ...def, matchCount: 0, entriesChecked: 0 }
     this.info.set(id, info)
 
@@ -135,10 +131,6 @@ export class WatcherManager {
     info.updatedAt = new Date().toISOString()
     info.status = 'running'
 
-    // Persist
-    const def: WatcherDefinition = { ...info }
-    await this.redis.hset('livetap:watchers', watcherId, JSON.stringify(def))
-
     this.writeLog(watcherId, `UPDATED conditions=${formatExpression(info.conditions, info.match)} cooldown=${info.cooldown}s`)
     this.startLoop(watcherId, streamKey, info)
 
@@ -162,7 +154,6 @@ export class WatcherManager {
 
     this.stopLoop(watcherId)
     this.info.delete(watcherId)
-    await this.redis.hdel('livetap:watchers', watcherId)
 
     // Delete log file
     try { unlinkSync(resolve(LOG_DIR, `${watcherId}.log`)) } catch { /* ok */ }
@@ -178,114 +169,93 @@ export class WatcherManager {
   }
 
   private startLoop(watcherId: string, streamKey: string, def: WatcherDefinition) {
-    const abort = new AbortController()
-    const reader = new Redis(this.redisUrl)
-    this.loops.set(watcherId, { abort, reader })
-
     const info = this.info.get(watcherId)!
     let lastAlertTime = 0
     let checkpointTime = Date.now()
-    const fieldNotFoundThrottle = new Map<string, number>() // field → last logged time
+    const fieldNotFoundThrottle = new Map<string, number>()
+
+    const onEntry = (entry: StreamEntry) => {
+      try {
+        info.entriesChecked++
+        info.lastChecked = new Date().toISOString()
+
+        const fields = entry.fields
 
-    const run = async () => {
-      let lastId = '$'
-      while (!abort.signal.aborted) {
+        // Parse payload: use parsed JSON object if available, otherwise raw fields
+        let payload: any
         try {
-          const result = await reader.xread('BLOCK', 2000, 'STREAMS', streamKey, lastId)
-          if (!result || abort.signal.aborted) continue
-
-          for (const [, entries] of result) {
-            for (const [id, fieldArray] of entries) {
-              lastId = id
-              info.entriesChecked++
-              info.lastChecked = new Date().toISOString()
-
-              // Parse fields
-              const fields: Record<string, string> = {}
-              for (let i = 0; i < fieldArray.length; i += 2) {
-                fields[fieldArray[i]] = fieldArray[i + 1]
-              }
-
-              // Parse payload: try JSON first, fall back to raw fields
-              let payload: any
-              try {
-                payload = JSON.parse(fields.payload ?? '{}')
-              } catch {
-                // Plain text — use raw Redis fields as the payload
-                // This allows conditions like { field: "payload", op: "contains", value: "ERROR" }
-                payload = fields
-              }
-
-              // Log missing fields (throttled: once per field per minute)
-              for (const c of def.conditions) {
-                const val = resolveDotPathImport(payload, c.field)
-                if (val === undefined) {
-                  const last = fieldNotFoundThrottle.get(c.field) ?? 0
-                  if (Date.now() - last > 60_000) {
-                    this.writeLog(watcherId, `FIELD_NOT_FOUND ${c.field} in entry ${id}`)
-                    fieldNotFoundThrottle.set(c.field, Date.now())
-                  }
-                }
-              }
-
-              // Evaluate
-              const matched = evaluateWatcher(payload, def)
-              if (matched) {
-                const now = Date.now()
-                if (now - lastAlertTime >= def.cooldown * 1000) {
-                  lastAlertTime = now
-                  info.matchCount++
-                  info.lastMatch = new Date().toISOString()
-
-                  const matchedValues = extractMatchedValues(payload, def.conditions)
-                  const expression = formatExpression(def.conditions, def.match)
-
-                  this.writeLog(watcherId, `MATCH ${Object.entries(matchedValues).map(([k, v]) => `${k}=${v}`).join(' ')} action=${typeof def.action === 'string' ? def.action : JSON.stringify(def.action)}`)
-
-                  const alert: WatcherAlert = {
-                    watcherId,
-                    connectionId: def.connectionId,
-                    expression,
-                    matchedValues,
-                    entry: fields,
-                    ts: now,
-                  }
-
-                  // Execute action
-                  await this.executeAction(def.action, alert)
-                  this.onAlert(alert)
-                } else {
-                  const remaining = Math.round((def.cooldown * 1000 - (Date.now() - lastAlertTime)) / 1000)
-                  this.writeLog(watcherId, `SUPPRESSED ${formatExpression(def.conditions, def.match)} (cooldown ${remaining}s remaining)`)
-                }
-              }
-
-              // Checkpoint every 5 minutes
-              if (Date.now() - checkpointTime > 5 * 60 * 1000) {
-                this.writeLog(watcherId, `CHECKPOINT entries_checked=${info.entriesChecked} matches=${info.matchCount}`)
-                checkpointTime = Date.now()
-              }
+          const parsed = JSON.parse(fields.payload ?? '{}')
+          // Only use parsed result if it's an object (has addressable fields).
+          // Primitives (numbers, strings, booleans) fall through to raw fields
+          // so "payload" remains addressable for regex/contains/numeric conditions.
+          payload = (typeof parsed === 'object' && parsed !== null) ? parsed : fields
+        } catch {
+          // Not valid JSON — use raw fields as the payload
+          payload = fields
+        }
+
+        // Log missing fields (throttled: once per field per minute)
+        for (const c of def.conditions) {
+          const val = resolveDotPath(payload, c.field)
+          if (val === undefined) {
+            const last = fieldNotFoundThrottle.get(c.field) ?? 0
+            if (Date.now() - last > 60_000) {
+              this.writeLog(watcherId, `FIELD_NOT_FOUND ${c.field} in entry ${entry.id}`)
+              fieldNotFoundThrottle.set(c.field, Date.now())
             }
           }
-        } catch (err) {
-          if (!abort.signal.aborted) {
-            this.writeLog(watcherId, `ERROR ${(err as Error).message}`)
-            await new Promise((r) => setTimeout(r, 1000))
+        }
+
+        // Evaluate
+        const matched = evaluateWatcher(payload, def)
+        if (matched) {
+          const now = Date.now()
+          if (now - lastAlertTime >= def.cooldown * 1000) {
+            lastAlertTime = now
+            info.matchCount++
+            info.lastMatch = new Date().toISOString()
+
+            const matchedValues = extractMatchedValues(payload, def.conditions)
+            const expression = formatExpression(def.conditions, def.match)
+
+            this.writeLog(watcherId, `MATCH ${Object.entries(matchedValues).map(([k, v]) => `${k}=${v}`).join(' ')} action=${typeof def.action === 'string' ? def.action : JSON.stringify(def.action)}`)
+
+            const alert: WatcherAlert = {
+              watcherId,
+              connectionId: def.connectionId,
+              expression,
+              matchedValues,
+              entry: fields,
+              ts: now,
+            }
+
+            this.executeAction(def.action, alert)
+            this.onAlert(alert)
+          } else {
+            const remaining = Math.round((def.cooldown * 1000 - (Date.now() - lastAlertTime)) / 1000)
+            this.writeLog(watcherId, `SUPPRESSED ${formatExpression(def.conditions, def.match)} (cooldown ${remaining}s remaining)`)
           }
         }
+
+        // Checkpoint every 5 minutes
+        if (Date.now() - checkpointTime > 5 * 60 * 1000) {
+          this.writeLog(watcherId, `CHECKPOINT entries_checked=${info.entriesChecked} matches=${info.matchCount}`)
+          checkpointTime = Date.now()
+        }
+      } catch (err) {
+        this.writeLog(watcherId, `ERROR ${(err as Error).message}`)
       }
-      reader.disconnect()
     }
 
-    run()
+    const unsub = this.store.subscribe(streamKey, onEntry)
+    this.subscriptions.set(watcherId, unsub)
   }
 
   private stopLoop(watcherId: string) {
-    const loop = this.loops.get(watcherId)
-    if (loop) {
-      loop.abort.abort()
-      loop.reader.disconnect()
-      this.loops.delete(watcherId)
+    const unsub = this.subscriptions.get(watcherId)
+    if (unsub) {
+      unsub()
+      this.subscriptions.delete(watcherId)
     }
     const info = this.info.get(watcherId)
     if (info) info.status = 'stopped'
@@ -342,13 +312,9 @@ export class WatcherManager {
   }
 
   async stopAll() {
-    for (const id of this.loops.keys()) {
+    for (const id of this.subscriptions.keys()) {
       this.stopLoop(id)
     }
   }
 }
 
-// Import from engine to avoid circular
-function resolveDotPathImport(obj: any, path: string): any {
-  return path.split('.').reduce((o, k) => o?.[k], obj)
-}