insforge 0.3.3 → 1.3.0

package/backend/src/{core/logs/providers → providers/logs}/base.provider.ts

@@ -1,21 +1,21 @@
-import { LogSource, AnalyticsLogRecord, LogSourceStats } from '@/types/logs.js';
+import { LogSchema, LogSourceSchema, LogStatsSchema } from '@insforge/shared-schemas';
 
-export interface AnalyticsProvider {
+export interface LogProvider {
   initialize(): Promise<void>;
 
-  getLogSources(): Promise<LogSource[]>;
+  getLogSources(): Promise<LogSourceSchema[]>;
 
   getLogsBySource(
     sourceName: string,
     limit?: number,
     beforeTimestamp?: string
   ): Promise<{
-    logs: AnalyticsLogRecord[];
+    logs: LogSchema[];
     total: number;
     tableName: string;
   }>;
 
-  getLogSourceStats(): Promise<LogSourceStats[]>;
+  getLogSourceStats(): Promise<LogStatsSchema[]>;
 
   searchLogs(
     query: string,
@@ -23,7 +23,7 @@ export interface AnalyticsProvider {
     limit?: number,
     offset?: number
   ): Promise<{
-    logs: (AnalyticsLogRecord & { source: string })[];
+    logs: (LogSchema & { source: string })[];
     total: number;
   }>;
 
@@ -31,7 +31,7 @@ export interface AnalyticsProvider {
 }
 
 // Base class with common functionality
-export abstract class BaseAnalyticsProvider implements AnalyticsProvider {
+export abstract class BaseLogProvider implements LogProvider {
   // Source name mapping for user-friendly display
   protected sourceNameMap: Record<string, string> = {
     'cloudflare.logs.prod': 'insforge.logs',
@@ -59,24 +59,24 @@ export abstract class BaseAnalyticsProvider implements AnalyticsProvider {
   }
 
   abstract initialize(): Promise<void>;
-  abstract getLogSources(): Promise<LogSource[]>;
+  abstract getLogSources(): Promise<LogSourceSchema[]>;
   abstract getLogsBySource(
     sourceName: string,
     limit?: number,
     beforeTimestamp?: string
   ): Promise<{
-    logs: AnalyticsLogRecord[];
+    logs: LogSchema[];
     total: number;
     tableName: string;
   }>;
-  abstract getLogSourceStats(): Promise<LogSourceStats[]>;
+  abstract getLogSourceStats(): Promise<LogStatsSchema[]>;
   abstract searchLogs(
     query: string,
     sourceName?: string,
     limit?: number,
     offset?: number
   ): Promise<{
-    logs: (AnalyticsLogRecord & { source: string })[];
+    logs: (LogSchema & { source: string })[];
     total: number;
   }>;
   abstract close(): Promise<void>;

package/backend/src/{core/logs/providers → providers/logs}/cloudwatch.provider.ts

@@ -4,23 +4,23 @@ import {
   FilterLogEventsCommand,
   StartQueryCommand,
   GetQueryResultsCommand,
+  CreateLogGroupCommand,
+  ResourceAlreadyExistsException,
 } from '@aws-sdk/client-cloudwatch-logs';
-import { LogSource, AnalyticsLogRecord, LogSourceStats } from '@/types/logs.js';
 import logger from '@/utils/logger.js';
-import { BaseAnalyticsProvider } from './base.provider.js';
+import { BaseLogProvider } from './base.provider.js';
+import { AppError } from '@/api/middlewares/error.js';
+import { ERROR_CODES } from '@/types/error-constants.js';
+import { LogSchema, LogSourceSchema, LogStatsSchema } from '@insforge/shared-schemas';
 
-export class CloudWatchProvider extends BaseAnalyticsProvider {
+export class CloudWatchProvider extends BaseLogProvider {
   private cwClient: CloudWatchLogsClient | null = null;
   private cwLogGroup: string | null = null;
   private cwRegion: string | null = null;
 
-  initialize(): Promise<void> {
+  async initialize(): Promise<void> {
     this.cwRegion = process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-2';
-    this.cwLogGroup = process.env.CLOUDWATCH_LOG_GROUP || null;
-
-    if (!this.cwLogGroup) {
-      throw new Error('CLOUDWATCH_LOG_GROUP is required when using CloudWatch analytics');
-    }
+    this.cwLogGroup = process.env.CLOUDWATCH_LOG_GROUP || '/insforge/local';
 
     const cloudwatchOpts: {
       region: string;
@@ -34,8 +34,18 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
     }
 
     this.cwClient = new CloudWatchLogsClient({ ...cloudwatchOpts });
-    logger.info(`Using analytics provider: ${this.cwLogGroup ? 'CloudWatch' : 'LocalDB/Postgres'}`);
-    return Promise.resolve();
+
+    // Create log group if it doesn't exist
+    try {
+      await this.cwClient.send(new CreateLogGroupCommand({ logGroupName: this.cwLogGroup }));
+      logger.info(`Created CloudWatch log group: ${this.cwLogGroup}`);
+    } catch (error) {
+      if (error instanceof ResourceAlreadyExistsException) {
+        logger.info(`CloudWatch log group already exists: ${this.cwLogGroup}`);
+      } else {
+        logger.warn(`Could not create CloudWatch log group: ${error}`);
+      }
+    }
   }
 
   private getSuffixMapping(): Record<string, string> {
@@ -47,9 +57,13 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
     };
   }
 
-  async getLogSources(): Promise<LogSource[]> {
+  async getLogSources(): Promise<LogSourceSchema[]> {
     if (!this.cwLogGroup || !this.cwClient) {
-      throw new Error('CloudWatch not initialized');
+      throw new AppError(
+        'AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY not found in environment variables',
+        500,
+        ERROR_CODES.LOGS_AWS_NOT_CONFIGURED
+      );
     }
     const logGroup = this.cwLogGroup;
     const client = this.cwClient;
@@ -59,13 +73,13 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
     const result = await client.send(cmd);
     const streams = result.logStreams || [];
 
-    const available: LogSource[] = [];
+    const available: LogSourceSchema[] = [];
     let idCounter = 1;
 
     for (const [displayName, suffix] of Object.entries(suffixMapping)) {
       const have = streams.some((s) => (s.logStreamName || '').includes(suffix));
       if (have) {
-        available.push({ id: idCounter++, name: displayName, token: suffix });
+        available.push({ id: String(idCounter++), name: displayName, token: suffix });
       }
     }
 
@@ -77,12 +91,16 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
     limit: number = 100,
     beforeTimestamp?: string
   ): Promise<{
-    logs: AnalyticsLogRecord[];
+    logs: LogSchema[];
     total: number;
     tableName: string;
   }> {
     if (!this.cwLogGroup || !this.cwClient) {
-      throw new Error('CloudWatch not initialized');
+      throw new AppError(
+        'AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY not found in environment variables',
+        500,
+        ERROR_CODES.LOGS_AWS_NOT_CONFIGURED
+      );
     }
     const client = this.cwClient;
     const logGroup = this.cwLogGroup;
@@ -127,7 +145,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
         const fle = await client.send(
           new FilterLogEventsCommand({
             logGroupName: logGroup,
-            logStreamNames: streams.length > 0 ? streams.slice(0, 100) : undefined,
+            logStreamNames: streams.length ? streams.slice(0, 100) : undefined,
             startTime: startMs,
             endTime: endMs,
             nextToken,
@@ -185,7 +203,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
           await sleep(300);
         }
 
-        if (results && results.length > 0) {
+        if (results && results.length) {
           // Convert Insights results to our format
           events = results.map((row) => {
             const obj = Object.fromEntries(row.map((c) => [c.field || '', c.value || '']));
@@ -225,7 +243,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
         const fle = await client.send(
           new FilterLogEventsCommand({
             logGroupName: logGroup,
-            logStreamNames: streams.length > 0 ? streams.slice(0, 100) : undefined,
+            logStreamNames: streams.length ? streams.slice(0, 100) : undefined,
             startTime: fallbackStartMs,
             endTime: beforeMs,
             limit: limit * 2, // Get a bit more to ensure we have enough
@@ -238,7 +256,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
       }
     }
     // Keep CloudWatch's default order (oldest first, newest last)
-    const logs: AnalyticsLogRecord[] = events.map((e) => {
+    const logs: LogSchema[] = events.map((e) => {
       const message = e.message || '';
       let parsed: Record<string, unknown> = {};
       try {
@@ -251,7 +269,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
         id: e.eventId || `${e.logStreamName || ''}-${e.timestamp || ''}`,
         // CloudWatch timestamp is in milliseconds
         timestamp: e.timestamp ? new Date(e.timestamp).toISOString() : new Date().toISOString(),
-        event_message:
+        eventMessage:
           typeof parsed === 'object' && parsed && (parsed as Record<string, unknown>).msg
             ? String((parsed as Record<string, unknown>).msg)
             : typeof message === 'string'
@@ -268,14 +286,18 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
     };
   }
 
-  async getLogSourceStats(): Promise<LogSourceStats[]> {
+  async getLogSourceStats(): Promise<LogStatsSchema[]> {
     if (!this.cwLogGroup || !this.cwClient) {
-      throw new Error('CloudWatch not initialized');
+      throw new AppError(
+        'AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY not found in environment variables',
+        500,
+        ERROR_CODES.LOGS_AWS_NOT_CONFIGURED
+      );
     }
     const client = this.cwClient;
     const logGroup = this.cwLogGroup;
     const sources = await this.getLogSources();
-    const stats: LogSourceStats[] = [];
+    const stats: LogStatsSchema[] = [];
     const suffixMapping = this.getSuffixMapping();
 
     const dls = await client.send(new DescribeLogStreamsCommand({ logGroupName: logGroup }));
@@ -289,7 +311,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
 
       let lastActivity = '';
 
-      if (sourceStreams.length > 0) {
+      if (sourceStreams.length) {
         try {
           // Use EXACTLY the same approach as getLogsBySource to get consistent results
           const endMs = Date.now();
@@ -297,16 +319,13 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
 
           // Use CloudWatch Insights to efficiently get the latest timestamp
           try {
-            const end = Date.now();
-            const start = end - 24 * 60 * 60 * 1000; // Last 24 hours
-
             const insights = `fields @timestamp | filter @logStream like /${suffix}/ | sort @timestamp desc | limit 1`;
 
             const startQuery = await client.send(
               new StartQueryCommand({
                 logGroupName: logGroup,
-                startTime: Math.floor(start / 1000),
-                endTime: Math.floor(end / 1000),
+                startTime: Math.floor(startMs / 1000),
+                endTime: Math.floor(endMs / 1000),
                 queryString: insights,
                 limit: 1,
               })
@@ -326,7 +345,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
               await sleep(200);
             }
 
-            if (results && results.length > 0) {
+            if (results && results.length) {
               const row = results[0];
               const timestampField = row.find((field) => field.field === '@timestamp');
               if (timestampField && timestampField.value) {
@@ -357,7 +376,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
             const fle = await client.send(
               new FilterLogEventsCommand({
                 logGroupName: logGroup,
-                logStreamNames: sourceStreams.length > 0 ? sourceStreams.slice(0, 100) : undefined,
+                logStreamNames: sourceStreams.length ? sourceStreams.slice(0, 100) : undefined,
                 startTime: startMs,
                 endTime: endMs,
                 limit: 1000, // Reasonable limit for fallback
@@ -365,7 +384,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
             );
 
             const events = fle.events || [];
-            if (events.length > 0) {
+            if (events.length) {
               const latestEvent = events[events.length - 1];
               if (latestEvent.timestamp) {
                 lastActivity = new Date(latestEvent.timestamp).toISOString();
@@ -409,11 +428,15 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
     limit: number = 100,
     _offset = 0 // CloudWatch doesn't support offset-based pagination
   ): Promise<{
-    logs: (AnalyticsLogRecord & { source: string })[];
+    logs: (LogSchema & { source: string })[];
     total: number;
   }> {
     if (!this.cwLogGroup || !this.cwClient) {
-      throw new Error('CloudWatch not initialized');
+      throw new AppError(
+        'AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY not found in environment variables',
+        500,
+        ERROR_CODES.LOGS_AWS_NOT_CONFIGURED
+      );
     }
     const client = this.cwClient;
     const logGroup = this.cwLogGroup;
@@ -462,7 +485,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
     const toObj = (row: Array<{ field?: string; value?: string }>) =>
       Object.fromEntries(row.map((c) => [c.field || '', c.value || '']));
 
-    const mapped: (AnalyticsLogRecord & { source: string })[] = rows.map((r) => {
+    const mapped: (LogSchema & { source: string })[] = rows.map((r) => {
       const o = toObj(r);
       const msg = o['@message'] || '';
       let parsed: Record<string, unknown> = {};
@@ -487,7 +510,7 @@ export class CloudWatchProvider extends BaseAnalyticsProvider {
         timestamp: o['@timestamp']
           ? new Date(parseInt(o['@timestamp'])).toISOString()
           : new Date().toISOString(),
-        event_message:
+        eventMessage:
           typeof parsed === 'object' && (parsed as Record<string, unknown>).msg
             ? String((parsed as Record<string, unknown>).msg)
             : typeof msg === 'string'

package/backend/src/providers/logs/local.provider.ts

@@ -0,0 +1,185 @@
+import { promises as fs, createReadStream } from 'fs';
+import { createInterface } from 'readline';
+import path from 'path';
+import { LogSchema, LogSourceSchema, LogStatsSchema } from '@insforge/shared-schemas';
+import { BaseLogProvider } from './base.provider.js';
+import logger from '@/utils/logger.js';
+
+export class LocalFileProvider extends BaseLogProvider {
+  private logsDir: string = '';
+  private logFiles: Record<string, string> = {
+    'insforge.logs': 'insforge.logs.jsonl',
+    'postgres.logs': 'postgres.logs.jsonl',
+    'postgREST.logs': 'postgrest.logs.jsonl',
+    'function.logs': 'function.logs.jsonl',
+  };
+
+  async initialize(): Promise<void> {
+    this.logsDir = process.env.LOGS_DIR || path.resolve(process.cwd(), 'insforge-logs');
+    try {
+      await fs.mkdir(this.logsDir, { recursive: true });
+    } catch {
+      // Directory already exists
+    }
+    logger.info(`File-based analytics initialized at: ${this.logsDir}`);
+  }
+
+  getLogSources(): Promise<LogSourceSchema[]> {
+    const sources: LogSourceSchema[] = [];
+    let id = 1;
+
+    for (const [name, filename] of Object.entries(this.logFiles)) {
+      sources.push({ id: String(id++), name, token: filename });
+    }
+
+    return Promise.resolve(sources);
+  }
+
+  async getLogsBySource(
+    sourceName: string,
+    limit: number = 100,
+    beforeTimestamp?: string
+  ): Promise<{
+    logs: LogSchema[];
+    total: number;
+    tableName: string;
+  }> {
+    const filename = this.logFiles[sourceName];
+    if (!filename) {
+      return { logs: [], total: 0, tableName: sourceName };
+    }
+
+    const filePath = path.join(this.logsDir, filename);
+    const logs = await this.readLogsFromFile(filePath, limit, beforeTimestamp);
+
+    return {
+      logs,
+      total: logs.length,
+      tableName: sourceName,
+    };
+  }
+
+  private async readLogsFromFile(
+    filePath: string,
+    limit: number,
+    beforeTimestamp?: string
+  ): Promise<LogSchema[]> {
+    try {
+      await fs.access(filePath);
+    } catch {
+      return [];
+    }
+
+    const logs: LogSchema[] = [];
+    const beforeMs = beforeTimestamp ? Date.parse(beforeTimestamp) : Date.now();
+
+    const fileStream = createReadStream(filePath);
+    const rl = createInterface({ input: fileStream, crlfDelay: Infinity });
+
+    for await (const line of rl) {
+      if (!line.trim()) {
+        continue;
+      }
+
+      try {
+        const log = JSON.parse(line);
+
+        // Only process Vector-transformed logs (have appname field)
+        if (!log.appname) {
+          continue;
+        }
+
+        const logTime = new Date(log.timestamp).getTime();
+
+        if (logTime < beforeMs) {
+          // For error logs, include error and stack in eventMessage to match CloudWatch display
+          let eventMessage = log.event_message || '';
+          if (log.level === 'error' && log.error) {
+            eventMessage = `${eventMessage}\n\nError: ${log.error}`;
+            if (log.stack) {
+              eventMessage += `\n\nStack Trace:\n${log.stack}`;
+            }
+          }
+
+          logs.push({
+            id: `${logTime}-${Math.random()}`,
+            timestamp: log.timestamp,
+            eventMessage,
+            body: log,
+          });
+        }
+      } catch {
+        // Skip invalid JSON lines
+      }
+    }
+
+    // Return most recent logs up to limit
+    return logs.slice(-limit);
+  }
+
+  async getLogSourceStats(): Promise<LogStatsSchema[]> {
+    const stats: LogStatsSchema[] = [];
+
+    for (const [name, filename] of Object.entries(this.logFiles)) {
+      const filePath = path.join(this.logsDir, filename);
+      try {
+        const fileStats = await fs.stat(filePath);
+        const logs = await this.readLogsFromFile(filePath, 1000);
+
+        stats.push({
+          source: name,
+          count: logs.length,
+          lastActivity: fileStats.mtime.toISOString(),
+        });
+      } catch {
+        // File doesn't exist
+      }
+    }
+
+    return stats;
+  }
+
+  async searchLogs(
+    query: string,
+    sourceName?: string,
+    limit: number = 100,
+    offset: number = 0
+  ): Promise<{
+    logs: (LogSchema & { source: string })[];
+    total: number;
+  }> {
+    const results: (LogSchema & { source: string })[] = [];
+    const searchLower = query.toLowerCase();
+
+    const filesToSearch = sourceName
+      ? [{ name: sourceName, filename: this.logFiles[sourceName] }]
+      : Object.entries(this.logFiles).map(([name, filename]) => ({ name, filename }));
+
+    for (const { name, filename } of filesToSearch) {
+      if (!filename) {
+        continue;
+      }
+
+      const filePath = path.join(this.logsDir, filename);
+      const logs = await this.readLogsFromFile(filePath, 10000);
+
+      for (const log of logs) {
+        const messageMatch = log.eventMessage.toLowerCase().includes(searchLower);
+        const metadataMatch = JSON.stringify(log.body).toLowerCase().includes(searchLower);
+
+        if (messageMatch || metadataMatch) {
+          results.push({ ...log, source: name });
+        }
+      }
+    }
+
+    return {
+      logs: results.slice(offset, offset + limit),
+      total: results.length,
+    };
+  }
+
+  async close(): Promise<void> {
+    // No cleanup needed for file-based provider
+  }
+}