livetap 0.1.0

package/src/mcp/tools.ts

@@ -0,0 +1,314 @@
+/**
+ * MCP Tool definitions for livetap.
+ * Registers tools on an MCP Server that proxy to the daemon HTTP API.
+ */
+
+import { z } from 'zod'
+import type { Server } from '@modelcontextprotocol/sdk/server/index.js'
+import {
+  ListToolsRequestSchema,
+  CallToolRequestSchema,
+} from '@modelcontextprotocol/sdk/types.js'
+
+export const TOOLS = [
+  {
+    name: 'create_connection',
+    description: 'Create a data connection. MQTT: connects to a broker and subscribes to topics. WebSocket: connects to a remote WS URL. Webhook: creates an HTTP ingest endpoint.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        type: { type: 'string', enum: ['mqtt', 'webhook', 'websocket', 'file'], description: 'Source type', default: 'mqtt' },
+        name: { type: 'string', description: 'Display name for the connection' },
+        broker: { type: 'string', description: 'MQTT broker hostname (required for mqtt)' },
+        port: { type: 'number', description: 'Broker port (default 1883 for mqtt)', default: 1883 },
+        tls: { type: 'boolean', description: 'Use TLS (default false)', default: false },
+        username: { type: 'string', description: 'MQTT username', default: '' },
+        password: { type: 'string', description: 'MQTT password', default: '' },
+        topics: { type: 'array', items: { type: 'string' }, description: 'MQTT topic filters (required for mqtt)' },
+        url: { type: 'string', description: 'WebSocket URL (required for websocket)' },
+        headers: { type: 'object', description: 'WebSocket auth headers' },
+        handshake: { type: 'string', description: 'Message to send after WS connect (e.g. subscription JSON)' },
+        path: { type: 'string', description: 'Absolute file path to tail (required for file type, e.g. "/var/log/app.log")' },
+      },
+    },
+  },
+  {
+    name: 'list_connections',
+    description: 'List all active connections with their status, message rate, and buffered count.',
+    inputSchema: { type: 'object' as const, properties: {} },
+  },
+  {
+    name: 'get_connection',
+    description: 'Get detailed status of a specific connection.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        connectionId: { type: 'string', description: 'The connection ID (e.g. "conn_a1b2c3d4")' },
+      },
+      required: ['connectionId'],
+    },
+  },
+  {
+    name: 'destroy_connection',
+    description: 'Destroy a connection — stops the source subscriber, cleans up the Redis stream.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        connectionId: { type: 'string', description: 'The connection ID to destroy' },
+      },
+      required: ['connectionId'],
+    },
+  },
+  {
+    name: 'read_stream',
+    description: "Read recent entries from a connection's live stream. Use this to inspect what data is flowing through a connection and understand the payload structure before creating watchers.",
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        connectionId: { type: 'string', description: 'The connection ID to read from' },
+        backfillSeconds: { type: 'number', description: 'Include entries from the last N seconds (default 60)', default: 60 },
+        maxEntries: { type: 'number', description: 'Max entries to return (default 10)', default: 10 },
+      },
+      required: ['connectionId'],
+    },
+  },
+  {
+    name: 'create_watcher',
+    description: 'Create an expression-based watcher on a connection. The watcher evaluates structured conditions against each stream entry and fires an alert when conditions match. ALWAYS use read_stream first to understand the data shape and field paths.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        connectionId: { type: 'string', description: 'The connection ID to watch' },
+        conditions: {
+          type: 'array',
+          description: 'Array of conditions: [{field: "dot.path", op: ">", value: 50}]. Supported ops: >, <, >=, <=, ==, !=, contains, matches (regex)',
+          items: {
+            type: 'object',
+            properties: {
+              field: { type: 'string', description: 'Dot-path into the JSON payload (e.g. "sensors.temperature.value")' },
+              op: { type: 'string', enum: ['>', '<', '>=', '<=', '==', '!=', 'contains', 'matches'], description: 'Comparison operator. "matches" takes a regex pattern string.' },
+              value: { description: 'Value to compare against (number, string, or boolean)' },
+            },
+            required: ['field', 'op', 'value'],
+          },
+        },
+        match: { type: 'string', enum: ['all', 'any'], description: 'How to combine conditions: "all" (AND) or "any" (OR). Default: "all"', default: 'all' },
+        action: { description: '"channel_alert" (default), or {webhook: "url"}, or {shell: "command"}', default: 'channel_alert' },
+        cooldown: { type: 'number', description: 'Seconds between repeated alerts. 0 for every match, 60 default. Use 0 for rare events (webhooks), 30-60 for sensors, 300+ for high-frequency streams.', default: 60 },
+      },
+      required: ['connectionId', 'conditions'],
+    },
+  },
+  {
+    name: 'list_watchers',
+    description: 'List watchers. Optionally filter by connectionId. If omitted, lists all watchers.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        connectionId: { type: 'string', description: 'Optional: filter by connection ID' },
+      },
+    },
+  },
+  {
+    name: 'get_watcher',
+    description: 'Get details of a specific watcher including conditions, status, match count, and config.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        watcherId: { type: 'string', description: 'The watcher ID' },
+      },
+      required: ['watcherId'],
+    },
+  },
+  {
+    name: 'get_watcher_logs',
+    description: 'Get evaluation logs from a watcher — shows MATCH, SUPPRESSED, FIELD_NOT_FOUND, and CHECKPOINT events.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        watcherId: { type: 'string', description: 'The watcher ID' },
+        lines: { type: 'number', description: 'Number of log lines to return (default 50)', default: 50 },
+      },
+      required: ['watcherId'],
+    },
+  },
+  {
+    name: 'update_watcher',
+    description: "Update a watcher's conditions, match mode, action, or cooldown. The watcher restarts with the new config.",
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        watcherId: { type: 'string', description: 'The watcher ID' },
+        conditions: { type: 'array', description: 'New conditions array', items: { type: 'object' } },
+        match: { type: 'string', enum: ['all', 'any'] },
+        action: { description: 'New action' },
+        cooldown: { type: 'number', description: 'New cooldown in seconds' },
+      },
+      required: ['watcherId'],
+    },
+  },
+  {
+    name: 'delete_watcher',
+    description: 'Stop and remove a watcher.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        watcherId: { type: 'string', description: 'The watcher ID to delete' },
+      },
+      required: ['watcherId'],
+    },
+  },
+  {
+    name: 'restart_watcher',
+    description: 'Restart a stopped watcher.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        watcherId: { type: 'string', description: 'The watcher ID to restart' },
+      },
+      required: ['watcherId'],
+    },
+  },
+]
+
+function text(content: string) {
+  return { content: [{ type: 'text' as const, text: content }] }
+}
+
+function error(content: string) {
+  return { content: [{ type: 'text' as const, text: content }], isError: true }
+}
+
+/**
+ * Register all livetap MCP tools on the given server.
+ * Tools proxy to the daemon HTTP API at the given base URL.
+ */
+export function registerTools(server: Server, daemonUrl: string) {
+  server.setRequestHandler(ListToolsRequestSchema, async () => ({
+    tools: TOOLS,
+  }))
+
+  server.setRequestHandler(CallToolRequestSchema, async (req) => {
+    const { name, arguments: args } = req.params
+    try {
+      switch (name) {
+        case 'create_connection': {
+          const config: any = { type: args?.type ?? 'mqtt' }
+          if (config.type === 'mqtt') {
+            config.broker = args?.broker
+            config.port = args?.port ?? 1883
+            config.tls = args?.tls ?? false
+            config.credentials = { username: args?.username ?? '', password: args?.password ?? '' }
+            config.topics = args?.topics
+            if (!config.broker || !config.topics?.length) {
+              return error('Error: broker and topics are required for mqtt connections')
+            }
+          } else if (config.type === 'websocket') {
+            config.url = args?.url
+            if (args?.headers) config.headers = args.headers
+            if (args?.handshake) config.handshake = args.handshake
+            if (!config.url) return error('Error: url is required for websocket connections')
+          } else if (config.type === 'file') {
+            config.path = args?.path
+            if (!config.path) return error('Error: path is required for file connections (e.g. "/var/log/app.log")')
+            if (!config.path.startsWith('/')) return error('Error: file path must be absolute')
+          }
+          // webhook needs no extra params
+          const res = await fetch(`${daemonUrl}/connections`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ ...config, name: args?.name }),
+          })
+          return text(await res.text())
+        }
+
+        case 'list_connections': {
+          const res = await fetch(`${daemonUrl}/connections`)
+          return text(await res.text())
+        }
+
+        case 'get_connection': {
+          const res = await fetch(`${daemonUrl}/connections/${args?.connectionId}`)
+          if (!res.ok) return error('Connection not found')
+          return text(await res.text())
+        }
+
+        case 'destroy_connection': {
+          const res = await fetch(`${daemonUrl}/connections/${args?.connectionId}`, { method: 'DELETE' })
+          if (!res.ok) return error('Connection not found')
+          return text(await res.text())
+        }
+
+        case 'read_stream': {
+          const params = new URLSearchParams()
+          if (args?.backfillSeconds) params.set('backfillSeconds', String(args.backfillSeconds))
+          if (args?.maxEntries) params.set('maxEntries', String(args.maxEntries))
+          const res = await fetch(`${daemonUrl}/connections/${args?.connectionId}/stream?${params}`)
+          if (!res.ok) return error('Connection not found')
+          return text(await res.text())
+        }
+
+        case 'create_watcher': {
+          const res = await fetch(`${daemonUrl}/watchers`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+              connectionId: args?.connectionId,
+              conditions: args?.conditions,
+              match: args?.match ?? 'all',
+              action: args?.action ?? 'channel_alert',
+              cooldown: args?.cooldown ?? 60,
+            }),
+          })
+          if (!res.ok) return error(await res.text())
+          return text(await res.text())
+        }
+
+        case 'list_watchers': {
+          const params = args?.connectionId ? `?connectionId=${args.connectionId}` : ''
+          const res = await fetch(`${daemonUrl}/watchers${params}`)
+          return text(await res.text())
+        }
+
+        case 'get_watcher': {
+          const res = await fetch(`${daemonUrl}/watchers/${args?.watcherId}`)
+          if (!res.ok) return error('Watcher not found')
+          return text(await res.text())
+        }
+
+        case 'get_watcher_logs': {
+          const res = await fetch(`${daemonUrl}/watchers/${args?.watcherId}/logs?lines=${args?.lines ?? 50}`)
+          return text(await res.text())
+        }
+
+        case 'update_watcher': {
+          const { watcherId, ...updates } = args as any
+          const res = await fetch(`${daemonUrl}/watchers/${watcherId}`, {
+            method: 'PUT',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(updates),
+          })
+          if (!res.ok) return error(await res.text())
+          return text(await res.text())
+        }
+
+        case 'delete_watcher': {
+          const res = await fetch(`${daemonUrl}/watchers/${args?.watcherId}`, { method: 'DELETE' })
+          if (!res.ok) return error('Watcher not found')
+          return text(await res.text())
+        }
+
+        case 'restart_watcher': {
+          const res = await fetch(`${daemonUrl}/watchers/${args?.watcherId}/restart`, { method: 'POST' })
+          if (!res.ok) return error('Watcher not found')
+          return text(await res.text())
+        }
+
+        default:
+          return error(`Unknown tool: ${name}`)
+      }
+    } catch (err) {
+      return error(`livetap daemon error: ${(err as Error).message}. Is the daemon running?`)
+    }
+  })
+}

package/src/server/connection-manager.ts

@@ -0,0 +1,171 @@
+/**
+ * Connection Manager — creates, tracks, and destroys data source connections.
+ */
+
+import type Redis from 'ioredis'
+import type { ConnectionConfig, ConnectionRecord, ConnectionStatus, Subscriber } from './types.js'
+import { MqttSubscriber } from './connections/mqtt.js'
+import { WebhookIngestor } from './connections/webhook.js'
+import { WsSubscriber } from './connections/websocket.js'
+import { FileSubscriber } from './connections/file.js'
+
+function generateId(): string {
+  const hex = Array.from(crypto.getRandomValues(new Uint8Array(4)))
+    .map((b) => b.toString(16).padStart(2, '0'))
+    .join('')
+  return `conn_${hex}`
+}
+
+export class ConnectionManager {
+  private connections = new Map<string, ConnectionRecord>()
+  private redis: Redis
+  private redisUrl: string
+
+  constructor(redis: Redis, redisUrl: string) {
+    this.redis = redis
+    this.redisUrl = redisUrl
+  }
+
+  async create(config: ConnectionConfig, name?: string): Promise<ConnectionRecord> {
+    const id = generateId()
+    const streamKey = `livetap:stream:${id}`
+    const createdAt = new Date().toISOString()
+
+    const record: ConnectionRecord = {
+      id,
+      config,
+      name,
+      streamKey,
+      createdAt,
+      subscriber: null as unknown as Subscriber,
+      msgCount: 0,
+      msgCountAtLastSample: 0,
+      msgPerSec: 0,
+      bufferedCount: 0,
+    }
+
+    const onMessage = () => { record.msgCount++ }
+
+    // Create appropriate subscriber
+    if (config.type === 'mqtt') {
+      record.subscriber = new MqttSubscriber({
+        config,
+        streamKey,
+        redis: this.redis,
+        onMessage,
+      })
+    } else if (config.type === 'webhook') {
+      record.subscriber = new WebhookIngestor({
+        streamKey,
+        redis: this.redis,
+        onMessage,
+      })
+    } else if (config.type === 'websocket') {
+      record.subscriber = new WsSubscriber({
+        config,
+        streamKey,
+        redis: this.redis,
+        onMessage,
+      })
+    } else if (config.type === 'file') {
+      record.subscriber = new FileSubscriber({
+        config,
+        streamKey,
+        redis: this.redis,
+        onMessage,
+      })
+    } else {
+      throw new Error(`Unsupported connection type: ${(config as any).type}`)
+    }
+
+    await record.subscriber.start()
+
+    // Throughput tracking (every 5s)
+    record.throughputInterval = setInterval(() => {
+      const delta = record.msgCount - record.msgCountAtLastSample
+      record.msgPerSec = Math.round((delta / 5) * 10) / 10
+      record.msgCountAtLastSample = record.msgCount
+    }, 5000)
+
+    // Buffered count tracking (every 5s)
+    record.bufferedInterval = setInterval(async () => {
+      try {
+        record.bufferedCount = await this.redis.xlen(streamKey)
+      } catch { /* ignore */ }
+    }, 5000)
+
+    this.connections.set(id, record)
+    return record
+  }
+
+  list(): ConnectionStatus[] {
+    return Array.from(this.connections.values()).map((r) => this.toStatus(r))
+  }
+
+  get(id: string): ConnectionRecord | undefined {
+    return this.connections.get(id)
+  }
+
+  getStatus(id: string): ConnectionStatus | undefined {
+    const r = this.connections.get(id)
+    return r ? this.toStatus(r) : undefined
+  }
+
+  async destroy(id: string): Promise<boolean> {
+    const record = this.connections.get(id)
+    if (!record) return false
+
+    if (record.throughputInterval) clearInterval(record.throughputInterval)
+    if (record.bufferedInterval) clearInterval(record.bufferedInterval)
+
+    await record.subscriber.stop()
+    await this.redis.del(record.streamKey)
+    this.connections.delete(id)
+    return true
+  }
+
+  async destroyAll(): Promise<void> {
+    for (const id of this.connections.keys()) {
+      await this.destroy(id)
+    }
+  }
+
+  /**
+   * Get the WebhookIngestor for a connection (for routing ingest POSTs).
+   */
+  getWebhookIngestor(id: string): WebhookIngestor | null {
+    const record = this.connections.get(id)
+    if (!record || record.config.type !== 'webhook') return null
+    return record.subscriber as WebhookIngestor
+  }
+
+  private toStatus(r: ConnectionRecord): ConnectionStatus {
+    const sub = r.subscriber.getStatus()
+    const base: ConnectionStatus = {
+      connectionId: r.id,
+      type: r.config.type,
+      name: r.name,
+      summary: this.buildSummary(r),
+      streamKey: r.streamKey,
+      createdAt: r.createdAt,
+      msgPerSec: r.msgPerSec,
+      bufferedCount: r.bufferedCount,
+      runtimeState: sub.runtimeState,
+      error: sub.error,
+    }
+    return base
+  }
+
+  private buildSummary(r: ConnectionRecord): string {
+    switch (r.config.type) {
+      case 'mqtt':
+        return `mqtt://${r.config.broker}:${r.config.port}/${r.config.topics.join(',')}`
+      case 'webhook':
+        return `webhook ingest`
+      case 'websocket':
+        return r.config.url
+      case 'file':
+        return `file://${r.config.path}`
+    }
+  }
+}

package/src/server/connections/file.ts

@@ -0,0 +1,123 @@
+/**
+ * File Tailing Subscriber — watches a file for new lines (tail -f behavior)
+ * and writes each line to a Redis Stream with rolling retention.
+ * Auto-detects JSON vs plain text per line.
+ */
+
+import { watch, type FSWatcher } from 'fs'
+import { open, stat, type FileHandle } from 'fs/promises'
+import type Redis from 'ioredis'
+import type { FileConnectionConfig, Subscriber, ConnectionStatus } from '../types.js'
+
+export interface FileSubscriberOpts {
+  config: FileConnectionConfig
+  streamKey: string
+  redis: Redis
+  retentionMs?: number
+  onMessage?: () => void
+}
+
+export class FileSubscriber implements Subscriber {
+  private config: FileConnectionConfig
+  private streamKey: string
+  private redis: Redis
+  private retentionMs: number
+  private onMessage: (() => void) | undefined
+  private state: ConnectionStatus['runtimeState'] = 'disconnected'
+  private error?: string
+  private fileHandle: FileHandle | null = null
+  private watcher: FSWatcher | null = null
+  private pollTimer: ReturnType<typeof setInterval> | null = null
+  private offset = 0
+  private buffer = ''
+  private stopped = false
+
+  constructor(opts: FileSubscriberOpts) {
+    this.config = opts.config
+    this.streamKey = opts.streamKey
+    this.redis = opts.redis
+    this.retentionMs = opts.retentionMs ?? 5 * 60 * 1000
+    this.onMessage = opts.onMessage
+  }
+
+  async start(): Promise<void> {
+    this.stopped = false
+
+    try {
+      // Open file and seek to end (tail -f: new lines only)
+      const stats = await stat(this.config.path)
+      this.offset = stats.size
+      this.fileHandle = await open(this.config.path, 'r')
+      this.state = 'connected'
+      this.error = undefined
+
+      // Watch for changes with fs.watch + fallback poll
+      try {
+        this.watcher = watch(this.config.path, () => this.readNewLines())
+      } catch {
+        // fs.watch not available on all platforms/filesystems
+      }
+
+      // Poll every 1s as fallback (fs.watch can miss events)
+      this.pollTimer = setInterval(() => this.readNewLines(), 1000)
+
+    } catch (err) {
+      this.state = 'error'
+      this.error = (err as Error).message
+      throw err
+    }
+  }
+
+  private async readNewLines() {
+    if (this.stopped || !this.fileHandle) return
+
+    try {
+      const stats = await stat(this.config.path)
+      if (stats.size <= this.offset) return // no new data
+
+      const readSize = stats.size - this.offset
+      const buf = Buffer.alloc(readSize)
+      const { bytesRead } = await this.fileHandle.read(buf, 0, readSize, this.offset)
+      this.offset += bytesRead
+
+      this.buffer += buf.toString('utf-8', 0, bytesRead)
+
+      // Split into complete lines
+      const lines = this.buffer.split('\n')
+      this.buffer = lines.pop() ?? '' // keep incomplete last line in buffer
+
+      for (const line of lines) {
+        if (!line.trim()) continue // skip empty lines
+
+        // Auto-detect: try JSON, fall back to text
+        let fields: Record<string, string>
+        try {
+          JSON.parse(line)
+          fields = { payload: line, format: 'json' }
+        } catch {
+          fields = { payload: line, format: 'text' }
+        }
+
+        await this.redis.xadd(this.streamKey, '*', ...Object.entries(fields).flat())
+        const minId = Date.now() - this.retentionMs
+        await this.redis.xtrim(this.streamKey, 'MINID', '~', minId.toString())
+        this.onMessage?.()
+      }
+    } catch (err) {
+      // File may have been deleted/rotated
+      this.error = (err as Error).message
+    }
+  }
+
+  async stop(): Promise<void> {
+    this.stopped = true
+    if (this.watcher) { this.watcher.close(); this.watcher = null }
+    if (this.pollTimer) { clearInterval(this.pollTimer); this.pollTimer = null }
+    if (this.fileHandle) { await this.fileHandle.close(); this.fileHandle = null }
+    this.state = 'disconnected'
+  }
+
+  getStatus() {
+    return { runtimeState: this.state, error: this.error }
+  }
+}