@ai-setting/roy-agent-core 1.0.0

package/src/env/debug/parser/regex-parser.ts

@@ -0,0 +1,196 @@
+/**
+ * @fileoverview Regex Parser for Log Lines
+ * 
+ * Parses log files with trace format:
+ * [TRACE] >>> {method} enter: {args}
+ * [TRACE] <<< {method} quit: {result}
+ * [TRACE] !!! {method} error: {error}
+ */
+
+import type { ParsedLogLine, LogLevel, TraceAction } from '../types';
+
+/**
+ * Regex pattern for parsing trace log lines
+ * 
+ * Format: {Timestamp} [{Level}] [{Source}][{Category}] [TRACE] {Action} {Method} [{requestId}] {actionName} {ActionType}: {Args}
+ * Examples:
+ *   >>> processRequest enter: (user query)
+ *   >>> llm.invoke [requestId=req_abc] enter: ["hello"]
+ *   <<< llm.invoke [requestId=req_abc] quit: {"ok":true}
+ * 
+ * The requestId can appear in either:
+ * 1. Category field: [source][requestId=req_123]
+ * 2. After method name: [requestId=req_abc]
+ */
+const TRACE_LINE_REGEX = /^(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}\.\d{3})\s+\[(\w+)\]\s+\[([^\]]*)\]\[([^\]]*)\]\s+\[TRACE\]\s+(>>>|<<<|!!!)\s+(.+?)(?:\s+\[requestId=([^\]]+)\])?\s+(enter|quit|error):\s+(.*)$/;
+
+/**
+ * Extract requestId from category field (e.g., "[source][requestId=req_123]" -> "req_123")
+ */
+const REQUEST_ID_IN_CATEGORY_REGEX = /\[requestId=([^\]]+)\]/;
+
+/**
+ * Regex Parser for trace log lines
+ */
+export class RegexParser {
+  /**
+   * Parse a single log line
+   */
+  parseLogLine(line: string): ParsedLogLine | null {
+    const trimmedLine = line.trim();
+    
+    if (!trimmedLine) {
+      return null;
+    }
+
+    const match = trimmedLine.match(TRACE_LINE_REGEX);
+    
+    if (!match) {
+      return null;
+    }
+
+    const [, timestamp, level, source, category, action, method, requestId, actionType, rawArgs] = match;
+
+    return {
+      raw: trimmedLine,
+      timestamp: timestamp.replace(' ', 'T'), // Normalize to ISO format
+      level,
+      source,
+      category,
+      action,
+      method,
+      actionType: actionType as TraceAction,
+      rawArgs,
+      requestId: requestId || null, // Store requestId if present
+    };
+  }
+
+  /**
+   * Parse arguments from raw args string
+   */
+  parseArgs(raw: string): unknown | unknown[] | null {
+    if (!raw || !raw.trim()) {
+      return null;
+    }
+
+    const trimmed = raw.trim();
+
+    // Try parsing as JSON
+    try {
+      const parsed = JSON.parse(trimmed);
+      
+      // If it's an array, return as-is
+      if (Array.isArray(parsed)) {
+        return parsed;
+      }
+      
+      // If it's an object, return as-is
+      return parsed;
+    } catch {
+      // Not valid JSON, return as string
+      return trimmed;
+    }
+  }
+
+  /**
+   * Check if a line is a trace line
+   */
+  isTraceLine(line: string): boolean {
+    return line.includes('[TRACE]') && 
+           (line.includes('>>>') || line.includes('<<<') || line.includes('!!!'));
+  }
+
+  /**
+   * Filter lines by log level
+   */
+  filterByLevel(lines: string[], level?: LogLevel): string[] {
+    if (!level) {
+      return lines;
+    }
+
+    const levelPattern = `[${level.toUpperCase()}]`;
+    
+    return lines.filter(line => {
+      // First check if it's a trace line
+      if (!this.isTraceLine(line)) {
+        return false;
+      }
+      // Then check level
+      return line.includes(levelPattern);
+    });
+  }
+
+  /**
+   * Filter lines by function name
+   */
+  filterByFunction(lines: string[], functionName?: string): string[] {
+    if (!functionName) {
+      return lines;
+    }
+
+    return lines.filter(line => {
+      // Check if the function name appears in the trace line
+      return line.includes(functionName);
+    });
+  }
+
+  /**
+   * Extract trace ID from log line
+   * Checks for requestId in two places:
+   * 1. Category field: [source][requestId=req_123]
+   * 2. After method name: [requestId=req_abc]
+   */
+  extractTraceId(line: string): string | null {
+    // First try: look for requestId in category field
+    const categoryMatch = line.match(REQUEST_ID_IN_CATEGORY_REGEX);
+    if (categoryMatch) {
+      return categoryMatch[1];
+    }
+    
+    // Second try: use main regex to extract requestId after method name
+    const match = line.match(TRACE_LINE_REGEX);
+    if (match && match[7]) {
+      return match[7];
+    }
+    
+    return null;
+  }
+
+  /**
+   * Extract trace IDs from multiple lines
+   */
+  extractTraceIds(lines: string[]): Map<string, string[]> {
+    const traceIdMap = new Map<string, string[]>();
+
+    for (const line of lines) {
+      if (!this.isTraceLine(line)) continue;
+
+      const traceId = this.extractTraceId(line);
+      const parsed = this.parseLogLine(line);
+      
+      if (traceId && parsed) {
+        const existing = traceIdMap.get(traceId) || [];
+        existing.push(line);
+        traceIdMap.set(traceId, existing);
+      }
+    }
+
+    return traceIdMap;
+  }
+
+  /**
+   * Parse multiple lines
+   */
+  parseLines(lines: string[]): ParsedLogLine[] {
+    const results: ParsedLogLine[] = [];
+
+    for (const line of lines) {
+      const parsed = this.parseLogLine(line);
+      if (parsed) {
+        results.push(parsed);
+      }
+    }
+
+    return results;
+  }
+}

package/src/env/debug/parser/span-builder.test.ts

@@ -0,0 +1,241 @@
+import { describe, expect, test, beforeEach } from 'bun:test';
+import { SpanBuilder } from './span-builder';
+import type { ParsedLogLine, TraceEntry, TraceTreeNode } from '../types';
+
+describe('SpanBuilder', () => {
+  let builder: SpanBuilder;
+
+  beforeEach(() => {
+    builder = new SpanBuilder();
+  });
+
+  describe('buildTraceEntries', () => {
+    test('should build entries from parsed lines', () => {
+      const lines: ParsedLogLine[] = [
+        {
+          raw: '2026-04-08 19:12:36.134 [INFO] [source][traced:llm] [TRACE] >>> llm.invoke enter: [{}]',
+          timestamp: '2026-04-08T19:12:36.134',
+          level: 'INFO',
+          source: 'source',
+          category: 'traced:llm',
+          action: '>>>',
+          method: 'llm.invoke',
+          actionType: 'enter',
+          rawArgs: '[{}]',
+        },
+        {
+          raw: '2026-04-08 19:12:39.213 [INFO] [source][traced:llm] [TRACE] <<< llm.invoke quit: {}',
+          timestamp: '2026-04-08T19:12:39.213',
+          level: 'INFO',
+          source: 'source',
+          category: 'traced:llm',
+          action: '<<<',
+          method: 'llm.invoke',
+          actionType: 'quit',
+          rawArgs: '{}',
+        },
+      ];
+
+      const entries = builder.buildTraceEntries(lines);
+
+      expect(entries).toHaveLength(2);
+      expect(entries[0].action).toBe('enter');
+      expect(entries[1].action).toBe('quit');
+    });
+
+    test('should extract params from enter action', () => {
+      const lines: ParsedLogLine[] = [
+        {
+          raw: 'test',
+          timestamp: '2026-04-08T19:12:36.134',
+          level: 'INFO',
+          source: 'source',
+          category: 'traced:llm',
+          action: '>>>',
+          method: 'llm.invoke',
+          actionType: 'enter',
+          rawArgs: '[{"messages":[]},{"model":"gpt-4"}]',
+        },
+      ];
+
+      const entries = builder.buildTraceEntries(lines);
+
+      expect(entries[0].params).toEqual([{ messages: [] }, { model: 'gpt-4' }]);
+    });
+
+    test('should extract result from quit action', () => {
+      const lines: ParsedLogLine[] = [
+        {
+          raw: 'test',
+          timestamp: '2026-04-08T19:12:39.213',
+          level: 'INFO',
+          source: 'source',
+          category: 'traced:llm',
+          action: '<<<',
+          method: 'llm.invoke',
+          actionType: 'quit',
+          rawArgs: '{"output":"Hello","latencyMs":100}',
+        },
+      ];
+
+      const entries = builder.buildTraceEntries(lines);
+
+      expect(entries[0].result).toEqual({ output: 'Hello', latencyMs: 100 });
+    });
+
+    test('should extract error from error action', () => {
+      const lines: ParsedLogLine[] = [
+        {
+          raw: 'test',
+          timestamp: '2026-04-08T19:12:39.213',
+          level: 'ERROR',
+          source: 'source',
+          category: 'traced:llm',
+          action: '!!!',
+          method: 'llm.invoke',
+          actionType: 'error',
+          rawArgs: 'Connection timeout',
+        },
+      ];
+
+      const entries = builder.buildTraceEntries(lines);
+
+      expect(entries[0].error).toBe('Connection timeout');
+    });
+
+    test('should extract traceId from category', () => {
+      const lines: ParsedLogLine[] = [
+        {
+          raw: '2026-04-08 19:12:36.134 [INFO] [source][requestId=req_123][traced:llm] [TRACE] >>> llm.invoke enter: [{}]',
+          timestamp: '2026-04-08T19:12:36.134',
+          level: 'INFO',
+          source: 'source',
+          category: 'requestId=req_123][traced:llm',
+          action: '>>>',
+          method: 'llm.invoke',
+          actionType: 'enter',
+          rawArgs: '[{}]',
+        },
+      ];
+
+      const entries = builder.buildTraceEntries(lines);
+
+      expect(entries[0].traceId).toBe('req_123');
+    });
+  });
+
+  describe('calculateDuration', () => {
+    test('should calculate duration between enter and quit', () => {
+      const enter: TraceEntry = {
+        traceId: 'req_123',
+        timestamp: '2026-04-08T19:12:36.134',
+        function: 'llm.invoke',
+        action: 'enter',
+      };
+
+      const quit: TraceEntry = {
+        traceId: 'req_123',
+        timestamp: '2026-04-08T19:12:39.213',
+        function: 'llm.invoke',
+        action: 'quit',
+      };
+
+      const duration = builder.calculateDuration(enter, quit);
+
+      expect(duration).toBe(3079); // ~3079ms
+    });
+
+    test('should return 0 for invalid timestamps', () => {
+      const enter: TraceEntry = {
+        traceId: 'req_123',
+        timestamp: 'invalid',
+        function: 'llm.invoke',
+        action: 'enter',
+      };
+
+      const quit: TraceEntry = {
+        traceId: 'req_123',
+        timestamp: '2026-04-08T19:12:39.213',
+        function: 'llm.invoke',
+        action: 'quit',
+      };
+
+      const duration = builder.calculateDuration(enter, quit);
+
+      expect(duration).toBe(0);
+    });
+  });
+
+  describe('buildCallTree', () => {
+    test('should build single node tree', () => {
+      const entries: TraceEntry[] = [
+        {
+          traceId: 'req_123',
+          timestamp: '2026-04-08T19:12:36.134',
+          function: 'llm.invoke',
+          action: 'enter',
+        },
+        {
+          traceId: 'req_123',
+          timestamp: '2026-04-08T19:12:39.213',
+          function: 'llm.invoke',
+          action: 'quit',
+          durationMs: 3079,
+        },
+      ];
+
+      const tree = builder.buildCallTree(entries);
+
+      expect(tree).not.toBeNull();
+      expect(tree!.function).toBe('llm.invoke');
+      expect(tree!.durationMs).toBe(3079);
+      expect(tree!.children).toHaveLength(0);
+    });
+
+    test('should return null for empty entries', () => {
+      const tree = builder.buildCallTree([]);
+      expect(tree).toBeNull();
+    });
+
+    test('should return null for unmatched enter/quit', () => {
+      const entries: TraceEntry[] = [
+        {
+          traceId: 'req_123',
+          timestamp: '2026-04-08T19:12:36.134',
+          function: 'llm.invoke',
+          action: 'enter',
+        },
+      ];
+
+      const tree = builder.buildCallTree(entries);
+      expect(tree).toBeNull();
+    });
+  });
+
+  describe('pairEnterQuit', () => {
+    test('should pair enter and quit entries', () => {
+      const entries: TraceEntry[] = [
+        {
+          traceId: 'req_123',
+          timestamp: '2026-04-08T19:12:36.134',
+          function: 'llm.invoke',
+          action: 'enter',
+          params: [{ arg1: 'value1' }],
+        },
+        {
+          traceId: 'req_123',
+          timestamp: '2026-04-08T19:12:39.213',
+          function: 'llm.invoke',
+          action: 'quit',
+          result: { output: 'result' },
+        },
+      ];
+
+      const paired = builder.pairEnterQuit(entries);
+
+      expect(paired).toHaveLength(1);
+      expect(paired[0].enter.params).toEqual([{ arg1: 'value1' }]);
+      expect(paired[0].quit.result).toEqual({ output: 'result' });
+    });
+  });
+});