@kb-labs/rest-api-app 1.3.0

package/src/routes/logs.ts

@@ -0,0 +1,718 @@
+/**
+ * @module @kb-labs/rest-api-app/routes/logs
+ * Live log streaming and querying endpoints
+ */
+
+import type { FastifyInstance } from 'fastify';
+import type { RestApiConfig } from '@kb-labs/rest-api-core';
+import { platform } from '@kb-labs/core-runtime';
+import type { LogQuery, LogRecord } from '@kb-labs/core-platform';
+import type { EventHub } from '../events/hub';
+
+/**
+ * Map Pino numeric level to string level
+ */
+function mapPinoLevelToString(level: any): string {
+  if (typeof level === 'string') {return level;}
+
+  // Pino levels: 10=trace, 20=debug, 30=info, 40=warn, 50=error, 60=fatal
+  if (level <= 10) {return 'trace';}
+  if (level <= 20) {return 'debug';}
+  if (level <= 30) {return 'info';}
+  if (level <= 40) {return 'warn';}
+  if (level <= 50) {return 'error';}
+  return 'fatal';
+}
+
+/**
+ * Convert internal LogRecord to frontend LogRecord format
+ */
+function toFrontendLogRecord(record: LogRecord): any {
+  // Extract level from fields if it's a Pino log (numeric level)
+  const pinoLevel = record.fields.level;
+  const levelStr = typeof pinoLevel === 'number'
+    ? mapPinoLevelToString(pinoLevel)
+    : record.level;
+
+  // Ensure message is a string
+  const messageStr = typeof record.message === 'string'
+    ? record.message
+    : JSON.stringify(record.message);
+
+  // Remove 'level' and 'time' from fields to avoid overwriting our converted values
+  const { level: _level, time: _time, ...restFields } = record.fields;
+
+  return {
+    id: record.id, // Include ID for navigation to detail page
+    time: new Date(record.timestamp).toISOString(),
+    level: levelStr,
+    msg: messageStr,
+    plugin: record.source,
+    ...restFields,
+  };
+}
+
+/**
+ * Build LLM prompt for log summarization
+ */
+function buildLogSummaryPrompt(
+  question: string,
+  logs: any[],
+  stats: any,
+  includeContext: { errors?: boolean; warnings?: boolean; info?: boolean; metadata?: boolean; stackTraces?: boolean }
+): string {
+  let prompt = `You are analyzing application logs. User question: "${question}"\n\n`;
+
+  // Add statistics
+  prompt += `Statistics:\n`;
+  prompt += `Total logs: ${stats.total}\n`;
+  prompt += `Errors: ${stats.byLevel.error || 0}\n`;
+  prompt += `Warnings: ${stats.byLevel.warn || 0}\n`;
+  prompt += `Info: ${stats.byLevel.info || 0}\n`;
+
+  if (stats.timeRange.from && stats.timeRange.to) {
+    prompt += `Time range: ${stats.timeRange.from} to ${stats.timeRange.to}\n`;
+  }
+
+  if (stats.topErrors.length > 0) {
+    prompt += `\nTop Errors:\n`;
+    stats.topErrors.slice(0, 5).forEach((err: any, idx: number) => {
+      prompt += `${idx + 1}. "${err.message}" (${err.count} occurrences)\n`;
+    });
+  }
+
+  // Filter logs based on includeContext
+  const relevantLogs = logs.filter(log => {
+    if (!includeContext.errors && log.level === 'error') {return false;}
+    if (!includeContext.warnings && log.level === 'warn') {return false;}
+    if (!includeContext.info && log.level === 'info') {return false;}
+    return true;
+  });
+
+  // Add log entries (limit to 100 most recent)
+  prompt += `\nLog Entries (${Math.min(relevantLogs.length, 100)} most recent):\n`;
+  relevantLogs.slice(-100).forEach(log => {
+    prompt += `[${log.time}] ${log.level.toUpperCase()}`;
+
+    if (includeContext.metadata && log.plugin) {
+      prompt += ` [${log.plugin}]`;
+    }
+
+    prompt += `: ${log.msg || '(no message)'}\n`;
+
+    if (includeContext.metadata && (log.traceId || log.executionId)) {
+      if (log.traceId) {prompt += `  traceId: ${log.traceId}\n`;}
+      if (log.executionId) {prompt += `  executionId: ${log.executionId}\n`;}
+    }
+
+    if (includeContext.stackTraces && log.err?.stack) {
+      prompt += `  stack: ${log.err.stack.split('\n').slice(0, 5).join('\n  ')}\n`;
+    }
+  });
+
+  // Add instructions
+  prompt += `\nInstructions:\n`;
+  prompt += `Provide a clear, concise summary answering the user's question as plain text. Focus on:\n`;
+  prompt += `1. What happened (timeline of events)\n`;
+  prompt += `2. Root causes if errors are present\n`;
+  prompt += `3. Patterns or trends you notice\n`;
+  prompt += `4. Actionable recommendations if applicable\n`;
+  prompt += `\nKeep the summary under 300 words. Use simple paragraphs separated by double newlines. Do not use markdown formatting (no **, ##, -, or other markdown syntax). Write in clear, professional language.\n`;
+
+  return prompt;
+}
+
+/**
+ * Extract correlation keys from log for finding related logs
+ */
+function extractCorrelationKeys(log: LogRecord): {
+  requestId?: string;
+  traceId?: string;
+  executionId?: string;
+  sessionId?: string;
+} {
+  return {
+    requestId: log.fields.requestId as string | undefined ??
+               log.fields.reqId as string | undefined,
+    traceId: log.fields.traceId as string | undefined,
+    executionId: log.fields.executionId as string | undefined,
+    sessionId: log.fields.sessionId as string | undefined,
+  };
+}
+
+/**
+ * Find logs related to the given log
+ * Strategy:
+ * 1. Try requestId/traceId/executionId (most specific)
+ * 2. Fallback to time window + same source
+ */
+async function findRelatedLogs(targetLog: LogRecord): Promise<any[]> {
+  const correlationKeys = extractCorrelationKeys(targetLog);
+  const timeWindow = 60000; // 1 minute before/after
+
+  // Strategy 1: Find by correlation IDs (most precise)
+  if (correlationKeys.requestId || correlationKeys.traceId || correlationKeys.executionId) {
+    const relatedLogs: LogRecord[] = [];
+
+    // Query all logs in time window
+    const result = await platform.logs.query({
+      from: targetLog.timestamp - timeWindow,
+      to: targetLog.timestamp + timeWindow,
+    }, {
+      limit: 1000,
+    });
+
+    // Filter in-memory by correlation keys
+    for (const log of result.logs) {
+      if (log.id === targetLog.id) {continue;} // Skip self
+
+      const logKeys = extractCorrelationKeys(log);
+
+      // Match any correlation key
+      if (
+        (correlationKeys.requestId && logKeys.requestId === correlationKeys.requestId) ||
+        (correlationKeys.traceId && logKeys.traceId === correlationKeys.traceId) ||
+        (correlationKeys.executionId && logKeys.executionId === correlationKeys.executionId) ||
+        (correlationKeys.sessionId && logKeys.sessionId === correlationKeys.sessionId)
+      ) {
+        relatedLogs.push(log);
+      }
+    }
+
+    if (relatedLogs.length > 0) {
+      return relatedLogs
+        .sort((a, b) => a.timestamp - b.timestamp)
+        .map(toFrontendLogRecord);
+    }
+  }
+
+  // Strategy 2: Fallback to time window + same source
+  const result = await platform.logs.query({
+    source: targetLog.source,
+    from: targetLog.timestamp - timeWindow,
+    to: targetLog.timestamp + timeWindow,
+  }, {
+    limit: 50,
+  });
+
+  return result.logs
+    .filter(log => log.id !== targetLog.id)
+    .sort((a, b) => a.timestamp - b.timestamp)
+    .map(toFrontendLogRecord);
+}
+
+/**
+ * Generate fallback summary when LLM is unavailable
+ */
+function generateFallbackSummary(stats: any, logs: any[]): string {
+  let summary = `Log Summary\n\n`;
+
+  summary += `Total Logs: ${stats.total}\n\n`;
+
+  // Level breakdown
+  summary += `By Level:\n`;
+  Object.entries(stats.byLevel).forEach(([level, count]) => {
+    summary += `${level}: ${count}\n`;
+  });
+  summary += `\n`;
+
+  // Plugin breakdown
+  if (Object.keys(stats.byPlugin).length > 0) {
+    summary += `By Plugin:\n`;
+    Object.entries(stats.byPlugin)
+      .sort((a: any, b: any) => b[1] - a[1])
+      .slice(0, 5)
+      .forEach(([plugin, count]) => {
+        summary += `${plugin}: ${count}\n`;
+      });
+    summary += `\n`;
+  }
+
+  // Top errors
+  if (stats.topErrors.length > 0) {
+    summary += `Top Errors:\n`;
+    stats.topErrors.slice(0, 5).forEach((err: any, idx: number) => {
+      summary += `${idx + 1}. "${err.message}" (${err.count} times)\n`;
+    });
+    summary += `\n`;
+  }
+
+  // Time range
+  if (stats.timeRange.from && stats.timeRange.to) {
+    summary += `Time Range: ${stats.timeRange.from} to ${stats.timeRange.to}\n\n`;
+  }
+
+  summary += `Note: LLM summarization is not available. This is a basic statistical summary.`;
+
+  return summary;
+}
+
+/**
+ * Register log observability routes
+ */
+export async function registerLogRoutes(
+  server: FastifyInstance,
+  config: RestApiConfig,
+  eventHub: EventHub
+): Promise<void> {
+  /**
+   * GET /api/v1/logs
+   * Query logs with filters
+   */
+  server.get<{
+    Querystring: {
+      from?: string;
+      to?: string;
+      level?: string;
+      plugin?: string;
+      executionId?: string;
+      tenantId?: string;
+      search?: string;
+      limit?: string;
+      offset?: string;
+    };
+  }>('/api/v1/logs', async (request, reply) => {
+    try {
+      // Parse numeric query params (Fastify doesn't parse them automatically)
+      const limit = request.query.limit ? parseInt(request.query.limit, 10) : 100;
+      const offset = request.query.offset ? parseInt(request.query.offset, 10) : 0;
+
+      // Build query from request params
+      const query: LogQuery = {
+        level: request.query.level as any,
+        source: request.query.plugin,
+        from: request.query.from ? new Date(request.query.from).getTime() : undefined,
+        to: request.query.to ? new Date(request.query.to).getTime() : undefined,
+      };
+
+      // Query logs using unified service
+      let result;
+      if (request.query.search) {
+        // Use full-text search if search query provided
+        result = await platform.logs.search(request.query.search, {
+          limit,
+          offset,
+        });
+      } else {
+        // Regular query with filters
+        result = await platform.logs.query(query, {
+          limit,
+          offset,
+        });
+      }
+
+      // Convert to frontend format
+      const frontendLogs = result.logs.map(toFrontendLogRecord);
+
+      // Get stats for response metadata
+      const stats = await platform.logs.getStats();
+
+      return {
+        ok: true,
+        data: {
+          logs: frontendLogs,
+          total: result.total,
+          hasMore: result.hasMore,
+          filters: request.query,
+          source: 'source' in result ? result.source : undefined,
+          stats: {
+            buffer: stats.buffer ? {
+              size: stats.buffer.size,
+              maxSize: stats.buffer.maxSize,
+              oldest: stats.buffer.oldestTimestamp ? new Date(stats.buffer.oldestTimestamp).toISOString() : undefined,
+              newest: stats.buffer.newestTimestamp ? new Date(stats.buffer.newestTimestamp).toISOString() : undefined,
+            } : undefined,
+            persistence: stats.persistence ? {
+              totalLogs: stats.persistence.totalLogs,
+              oldestTimestamp: stats.persistence.oldestTimestamp ? new Date(stats.persistence.oldestTimestamp).toISOString() : undefined,
+              newestTimestamp: stats.persistence.newestTimestamp ? new Date(stats.persistence.newestTimestamp).toISOString() : undefined,
+              sizeBytes: stats.persistence.sizeBytes,
+            } : undefined,
+          },
+        },
+      };
+    } catch (error: any) {
+      return reply.code(503).send({
+        ok: false,
+        error: 'Log query failed',
+        message: error?.message ?? 'Unknown error',
+      });
+    }
+  });
+
+  /**
+   * GET /api/v1/logs/:id
+   * Get single log by ID with related logs
+   */
+  server.get<{
+    Params: { id: string };
+    Querystring: { includeRelated?: string };
+  }>(
+    '/api/v1/logs/:id',
+    async (request, reply) => {
+      try {
+        const log = await platform.logs.getById(request.params.id);
+
+        if (!log) {
+          return reply.code(404).send({
+            ok: false,
+            error: 'Log not found',
+            message: `Log with ID '${request.params.id}' does not exist`,
+          });
+        }
+
+        const frontendLog = toFrontendLogRecord(log);
+
+        // Optionally include related logs
+        let relatedLogs: any[] = [];
+        if (request.query.includeRelated === 'true') {
+          relatedLogs = await findRelatedLogs(log);
+        }
+
+        return {
+          ok: true,
+          data: {
+            log: frontendLog,
+            related: relatedLogs.length > 0 ? relatedLogs : undefined,
+          },
+        };
+      } catch (error: any) {
+        return reply.code(500).send({
+          ok: false,
+          error: 'Failed to fetch log',
+          message: error?.message ?? 'Unknown error',
+        });
+      }
+    }
+  );
+
+  /**
+   * GET /api/v1/logs/:id/related
+   * Get logs related to a specific log (same trace/execution/request)
+   */
+  server.get<{ Params: { id: string } }>(
+    '/api/v1/logs/:id/related',
+    async (request, reply) => {
+      try {
+        const log = await platform.logs.getById(request.params.id);
+
+        if (!log) {
+          return reply.code(404).send({
+            ok: false,
+            error: 'Log not found',
+            message: `Log with ID '${request.params.id}' does not exist`,
+          });
+        }
+
+        const relatedLogs = await findRelatedLogs(log);
+
+        return {
+          ok: true,
+          data: {
+            total: relatedLogs.length,
+            logs: relatedLogs,
+            correlationKeys: extractCorrelationKeys(log),
+          },
+        };
+      } catch (error: any) {
+        return reply.code(500).send({
+          ok: false,
+          error: 'Failed to fetch related logs',
+          message: error?.message ?? 'Unknown error',
+        });
+      }
+    }
+  );
+
+  /**
+   * GET /api/v1/logs/stream
+   * Server-Sent Events stream for real-time logs
+   */
+  server.get('/api/v1/logs/stream', async (request, reply) => {
+    // Check if streaming is supported
+    const caps = platform.logs.getCapabilities();
+    if (!caps.hasStreaming) {
+      return reply.code(503).send({
+        error: 'Log streaming not enabled',
+        message: 'Logger adapter does not support streaming. Enable logRingBuffer in kb.config.json',
+      });
+    }
+
+    // Tell Fastify we're manually managing the response
+    reply.hijack();
+
+    // Track if stream is closed to prevent double-end
+    let streamClosed = false;
+
+    // Explicit CORS headers for EventSource
+    const origin = request.headers.origin;
+    if (origin === 'http://localhost:3000' || origin === 'http://localhost:5173') {
+      reply.raw.setHeader('Access-Control-Allow-Origin', origin);
+      reply.raw.setHeader('Access-Control-Allow-Credentials', 'true');
+    } else {
+      reply.raw.setHeader('Access-Control-Allow-Origin', '*');
+    }
+
+    // Setup SSE headers
+    reply.raw.setHeader('Content-Type', 'text/event-stream');
+    reply.raw.setHeader('Cache-Control', 'no-cache, no-transform');
+    reply.raw.setHeader('Connection', 'keep-alive');
+
+    try {
+      reply.raw.flushHeaders?.();
+      reply.raw.write(': connected\n\n');
+    } catch (err) {
+      // Headers already sent or connection closed
+      streamClosed = true;
+      return;
+    }
+
+    // Subscribe to log stream using unified service
+    const unsubscribe = platform.logs.subscribe((log) => {
+      // Check if connection is still alive before writing
+      if (!streamClosed && !reply.raw.writableEnded && !reply.raw.destroyed) {
+        try {
+          // Frontend expects event name 'log'
+          const frontendLog = toFrontendLogRecord(log);
+          reply.raw.write(`event: log\n`);
+          reply.raw.write(`data: ${JSON.stringify(frontendLog)}\n\n`);
+        } catch (err) {
+          // Connection closed while writing - ignore
+          streamClosed = true;
+        }
+      }
+    });
+
+    // Cleanup on disconnect
+    const cleanup = () => {
+      if (!streamClosed) {
+        streamClosed = true;
+        unsubscribe();
+        try {
+          if (!reply.raw.writableEnded && !reply.raw.destroyed) {
+            reply.raw.end();
+          }
+        } catch (err) {
+          // Already closed - ignore
+        }
+      }
+    };
+
+    request.raw.on('close', cleanup);
+    request.raw.on('error', cleanup);
+
+    // Keep connection alive indefinitely
+    await new Promise<void>(() => {});
+  });
+
+  /**
+   * GET /api/v1/logs/stats
+   * Get combined log statistics from all backends
+   */
+  server.get('/api/v1/logs/stats', async (request, reply) => {
+    try {
+      const stats = await platform.logs.getStats();
+      const caps = platform.logs.getCapabilities();
+
+      return {
+        ok: true,
+        data: {
+          capabilities: caps,
+          buffer: stats.buffer ? {
+            size: stats.buffer.size,
+            maxSize: stats.buffer.maxSize,
+            oldest: stats.buffer.oldestTimestamp ? new Date(stats.buffer.oldestTimestamp).toISOString() : undefined,
+            newest: stats.buffer.newestTimestamp ? new Date(stats.buffer.newestTimestamp).toISOString() : undefined,
+          } : undefined,
+          persistence: stats.persistence ? {
+            totalLogs: stats.persistence.totalLogs,
+            oldest: stats.persistence.oldestTimestamp ? new Date(stats.persistence.oldestTimestamp).toISOString() : undefined,
+            newest: stats.persistence.newestTimestamp ? new Date(stats.persistence.newestTimestamp).toISOString() : undefined,
+            sizeBytes: stats.persistence.sizeBytes,
+          } : undefined,
+        },
+      };
+    } catch (error: any) {
+      return reply.code(500).send({
+        ok: false,
+        error: 'Failed to fetch stats',
+        message: error?.message ?? 'Unknown error',
+      });
+    }
+  });
+
+  /**
+   * POST /api/v1/logs/summarize
+   * AI-powered log summarization (placeholder for future implementation)
+   *
+   * Expected request body:
+   * {
+   *   "timeRange": { "from": "ISO timestamp", "to": "ISO timestamp" },
+   *   "filters": { "level": "error", "plugin": "rest", "traceId": "..." },
+   *   "groupBy": "trace" | "execution" | "plugin",
+   *   "question": "What happened in the last 5 minutes?" (optional)
+   * }
+   */
+  server.post<{
+    Body: {
+      timeRange?: { from?: string; to?: string };
+      filters?: {
+        level?: string;
+        plugin?: string;
+        traceId?: string;
+        executionId?: string;
+      };
+      groupBy?: 'trace' | 'execution' | 'plugin';
+      question?: string;
+      includeContext?: {
+        errors?: boolean;
+        warnings?: boolean;
+        info?: boolean;
+        metadata?: boolean;
+        stackTraces?: boolean;
+      };
+    };
+  }>('/api/v1/logs/summarize', async (request, reply) => {
+    const { timeRange, filters, groupBy, question, includeContext } = request.body;
+
+    try {
+      // Build query from request
+      const query: LogQuery = {
+        level: filters?.level as any,
+        source: filters?.plugin,
+        from: timeRange?.from ? new Date(timeRange.from).getTime() : undefined,
+        to: timeRange?.to ? new Date(timeRange.to).getTime() : undefined,
+      };
+
+      // Query logs using unified service
+      const result = await platform.logs.query(query, { limit: 1000 });
+      const frontendLogs = result.logs.map(toFrontendLogRecord);
+
+    // Additional filters (traceId, executionId)
+    let filteredLogs = frontendLogs;
+    if (filters?.traceId) {
+      filteredLogs = filteredLogs.filter((log) => log.traceId === filters.traceId);
+    }
+    if (filters?.executionId) {
+      filteredLogs = filteredLogs.filter((log) => log.executionId === filters.executionId);
+    }
+
+    // Calculate aggregated statistics
+    const stats = {
+      total: filteredLogs.length,
+      byLevel: {} as Record<string, number>,
+      byPlugin: {} as Record<string, number>,
+      topErrors: [] as Array<{ message: string; count: number }>,
+      timeRange: {
+        from: filteredLogs.length > 0 ? filteredLogs[0]!.time : null,
+        to: filteredLogs.length > 0 ? filteredLogs[filteredLogs.length - 1]!.time : null,
+      },
+    };
+
+    const errorMessages = new Map<string, number>();
+
+    for (const log of filteredLogs) {
+      // Count by level
+      stats.byLevel[log.level] = (stats.byLevel[log.level] || 0) + 1;
+
+      // Count by plugin
+      if (log.plugin) {
+        stats.byPlugin[log.plugin] = (stats.byPlugin[log.plugin] || 0) + 1;
+      }
+
+      // Track error messages
+      if (log.level === 'error' && log.msg) {
+        errorMessages.set(log.msg, (errorMessages.get(log.msg) || 0) + 1);
+      }
+    }
+
+    // Top 10 errors
+    stats.topErrors = Array.from(errorMessages.entries())
+      .map(([message, count]) => ({ message, count }))
+      .sort((a, b) => b.count - a.count)
+      .slice(0, 10);
+
+    // Group logs if requested
+    let groups: Record<string, any[]> | null = null;
+    if (groupBy) {
+      // Map frontend groupBy values to actual log field names
+      const fieldMap: Record<string, string> = {
+        trace: 'traceId',
+        execution: 'executionId',
+        plugin: 'plugin',
+      };
+
+      const groupMap = new Map<string, any[]>();
+      for (const log of filteredLogs) {
+        const fieldName = fieldMap[groupBy] || groupBy;
+        const key = String(log[fieldName] || 'unknown');
+        if (!groupMap.has(key)) {
+          groupMap.set(key, []);
+        }
+        groupMap.get(key)!.push(log);
+      }
+      groups = Object.fromEntries(groupMap.entries());
+    }
+
+    // AI Summarization
+    let aiSummary: string | null = null;
+    let message: string | null = null;
+
+    if (platform.llm && question) {
+      try {
+        const context = includeContext || {
+          errors: true,
+          warnings: true,
+          info: false,
+          metadata: true,
+          stackTraces: true,
+        };
+
+        const prompt = buildLogSummaryPrompt(question, filteredLogs, stats, context);
+
+        const response = await platform.llm.complete(prompt, {
+          temperature: 0.7,
+          maxTokens: 1000,
+          systemPrompt: 'You are a technical log analysis assistant. Provide clear, actionable insights based on application logs.',
+        });
+
+        aiSummary = response.content.trim();
+
+      } catch (error) {
+        // Graceful degradation
+        platform.logger.warn('LLM summarization failed, using fallback', {
+          error: error instanceof Error ? error.message : String(error),
+        });
+        aiSummary = generateFallbackSummary(stats, filteredLogs);
+        message = 'AI summarization unavailable, showing statistical summary';
+      }
+    } else {
+      // No LLM or no question - return fallback
+      aiSummary = generateFallbackSummary(stats, filteredLogs);
+      message = platform.llm ? 'No question provided' : 'LLM not configured';
+    }
+
+      // Return structured data with AI summary
+      return {
+        ok: true,
+        data: {
+          summary: {
+            question: question || 'General log summary',
+            timeRange: stats.timeRange,
+            total: stats.total,
+            stats,
+            groups,
+          },
+          aiSummary,
+          message,
+        },
+      };
+    } catch (error: any) {
+      return reply.code(500).send({
+        ok: false,
+        error: 'Log summarization failed',
+        message: error?.message ?? 'Unknown error',
+      });
+    }
+  });
+}