@peakinfer/cli 1.0.133

package/dist/format-normalizer.js

@@ -0,0 +1,1331 @@
+/**
+ * Format Normalizer - Agent-based runtime event format detection and normalization.
+ *
+ * This module implements PRD §6.4: Enable PeakInfer to ingest runtime data from any
+ * observability system, logging framework, or custom format without requiring users
+ * to transform their data first.
+ *
+ * Design Principles (Julie Zhou aligned):
+ * - Behavior First: Detect formats automatically, fallback gracefully
+ * - Clarity Over Cleverness: Clear confidence scores, no silent assumptions
+ * - State Completeness: Handle all format states (known, agent-required, unknown)
+ */
+import { query } from '@anthropic-ai/claude-agent-sdk';
+import { loadPrompt } from './templates.js';
+/**
+ * Extract text content from Claude Agent SDK messages
+ */
+function extractTextFromMessages(messages) {
+    let text = '';
+    for (const msg of messages) {
+        if (msg.type === 'assistant' && msg.message?.content) {
+            for (const block of msg.message.content) {
+                if (block.type === 'text') {
+                    text += block.text;
+                }
+            }
+        }
+    }
+    return text;
+}
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+const SAMPLE_LINES = 20; // Number of lines to sample for detection
+const MIN_CONFIDENCE_THRESHOLD = 0.7; // Minimum confidence for auto-acceptance
+const LLM_MODEL = 'claude-sonnet-4-20250514';
+// Required fields for InferenceEvent
+const REQUIRED_FIELDS = ['id', 'ts', 'provider', 'model', 'input_tokens', 'output_tokens', 'latency_ms'];
+// =============================================================================
+// FORMAT SIGNATURES
+// =============================================================================
+/**
+ * Known format signatures for heuristic detection.
+ * Each signature includes structural patterns that uniquely identify a format.
+ */
+const FORMAT_SIGNATURES = {
+    otel: {
+        patterns: [
+            /resourceSpans/i,
+            /scopeSpans/i,
+            /traceId/i,
+            /spanId/i,
+        ],
+        structuralCheck: (data) => {
+            if (typeof data !== 'object' || data === null)
+                return false;
+            const obj = data;
+            return 'resourceSpans' in obj || 'resource_spans' in obj;
+        },
+        confidence: 0.95,
+    },
+    jaeger: {
+        patterns: [
+            /traceID/, // Jaeger uses capital ID (vs OTEL's traceId)
+            /spanID/, // Jaeger uses capital ID (vs OTEL's spanId)
+            /operationName/,
+            /jaeger/i,
+            /"processes"/, // Jaeger-specific field
+        ],
+        structuralCheck: (data) => {
+            if (typeof data !== 'object' || data === null)
+                return false;
+            const obj = data;
+            // Jaeger format: { data: [{ traceID: ..., processes: ... }] }
+            if ('data' in obj && Array.isArray(obj.data)) {
+                const firstTrace = obj.data[0];
+                // Must have traceID (capital ID) to distinguish from OTEL
+                return firstTrace?.traceID !== undefined && firstTrace?.processes !== undefined;
+            }
+            return false;
+        },
+        confidence: 0.95,
+    },
+    zipkin: {
+        patterns: [
+            /"traceId"/,
+            /"parentId"/,
+            /"localEndpoint"/,
+            /zipkin/i,
+        ],
+        structuralCheck: (data) => {
+            if (!Array.isArray(data))
+                return false;
+            const first = data[0];
+            return first?.traceId !== undefined && first?.localEndpoint !== undefined;
+        },
+        confidence: 0.95,
+    },
+    langsmith: {
+        patterns: [
+            /run_id/,
+            /run_type/,
+            /langsmith/i,
+            /langchain/i,
+        ],
+        structuralCheck: (data) => {
+            if (typeof data !== 'object' || data === null)
+                return false;
+            const obj = data;
+            return 'run_id' in obj || 'runs' in obj;
+        },
+        confidence: 0.90,
+    },
+    litellm: {
+        patterns: [
+            /litellm/i,
+            /call_type/,
+            /api_base/,
+            /response_time_ms/,
+        ],
+        structuralCheck: (data) => {
+            if (typeof data !== 'object' || data === null)
+                return false;
+            const obj = data;
+            // LiteLLM logs have call_type OR api_base OR response_time_ms fields
+            return 'call_type' in obj || 'api_base' in obj || 'response_time_ms' in obj;
+        },
+        confidence: 0.90,
+    },
+    helicone: {
+        patterns: [
+            /helicone/i,
+            /helicone_request_id/,
+            /helicone_response_id/,
+            /helicone_properties/,
+        ],
+        structuralCheck: (data) => {
+            if (typeof data !== 'object' || data === null)
+                return false;
+            const obj = data;
+            // Helicone-specific fields - must have helicone_ prefixed fields
+            return 'helicone_request_id' in obj || 'helicone_response_id' in obj ||
+                'helicone_properties' in obj || 'helicone' in obj;
+        },
+        confidence: 0.85,
+    },
+};
+// =============================================================================
+// PREDEFINED FIELD MAPPINGS
+// =============================================================================
+/**
+ * Predefined field mappings for known formats.
+ * These are high-confidence mappings based on format specifications.
+ */
+const PREDEFINED_MAPPINGS = {
+    otel: [
+        {
+            target: 'id',
+            source_path: '$.resourceSpans[*].scopeSpans[*].spans[*].spanId',
+            extraction_type: 'jsonpath',
+            transform: 'none',
+            confidence: 1.0,
+            evidence: 'OTLP span ID field',
+        },
+        {
+            target: 'ts',
+            source_path: '$.resourceSpans[*].scopeSpans[*].spans[*].startTimeUnixNano',
+            extraction_type: 'jsonpath',
+            transform: 'unix_nano_to_iso',
+            confidence: 1.0,
+            evidence: 'OTLP start time in nanoseconds',
+        },
+        {
+            target: 'provider',
+            source_path: "$.resourceSpans[*].scopeSpans[*].spans[*].attributes[?(@.key=='llm.provider')].value.stringValue",
+            extraction_type: 'jsonpath',
+            transform: 'provider_normalize',
+            confidence: 0.9,
+            evidence: 'LLM semantic convention attribute',
+        },
+        {
+            target: 'model',
+            source_path: "$.resourceSpans[*].scopeSpans[*].spans[*].attributes[?(@.key=='llm.model')].value.stringValue",
+            extraction_type: 'jsonpath',
+            transform: 'none',
+            confidence: 0.9,
+            evidence: 'LLM semantic convention attribute',
+        },
+        {
+            target: 'input_tokens',
+            source_path: "$.resourceSpans[*].scopeSpans[*].spans[*].attributes[?(@.key=='llm.token_count.prompt')].value.intValue",
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.9,
+            evidence: 'LLM semantic convention attribute',
+        },
+        {
+            target: 'output_tokens',
+            source_path: "$.resourceSpans[*].scopeSpans[*].spans[*].attributes[?(@.key=='llm.token_count.completion')].value.intValue",
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.9,
+            evidence: 'LLM semantic convention attribute',
+        },
+        {
+            target: 'latency_ms',
+            source_path: '(endTimeUnixNano - startTimeUnixNano) / 1000000',
+            extraction_type: 'computed',
+            transform: 'none',
+            confidence: 1.0,
+            evidence: 'Computed from OTLP start/end timestamps',
+        },
+    ],
+    jaeger: [
+        {
+            target: 'id',
+            source_path: '$.data[*].spans[*].spanID',
+            extraction_type: 'jsonpath',
+            transform: 'none',
+            confidence: 1.0,
+            evidence: 'Jaeger span ID',
+        },
+        {
+            target: 'ts',
+            source_path: '$.data[*].spans[*].startTime',
+            extraction_type: 'jsonpath',
+            transform: 'unix_ms_to_iso',
+            confidence: 1.0,
+            evidence: 'Jaeger start time in microseconds',
+        },
+        {
+            target: 'provider',
+            source_path: "$.data[*].spans[*].tags[?(@.key=='llm.provider')].value",
+            extraction_type: 'jsonpath',
+            transform: 'provider_normalize',
+            confidence: 0.85,
+            evidence: 'Tag-based provider extraction',
+        },
+        {
+            target: 'model',
+            source_path: "$.data[*].spans[*].tags[?(@.key=='llm.model')].value",
+            extraction_type: 'jsonpath',
+            transform: 'none',
+            confidence: 0.85,
+            evidence: 'Tag-based model extraction',
+        },
+        {
+            target: 'input_tokens',
+            source_path: "$.data[*].spans[*].tags[?(@.key=='llm.input_tokens')].value",
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.85,
+            evidence: 'Tag-based token extraction',
+        },
+        {
+            target: 'output_tokens',
+            source_path: "$.data[*].spans[*].tags[?(@.key=='llm.output_tokens')].value",
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.85,
+            evidence: 'Tag-based token extraction',
+        },
+        {
+            target: 'latency_ms',
+            source_path: '$.data[*].spans[*].duration',
+            extraction_type: 'jsonpath',
+            transform: 'none', // Jaeger duration is already in microseconds, convert to ms
+            confidence: 1.0,
+            evidence: 'Jaeger duration field (microseconds -> ms)',
+        },
+    ],
+    zipkin: [
+        {
+            target: 'id',
+            source_path: '$[*].id',
+            extraction_type: 'jsonpath',
+            transform: 'none',
+            confidence: 1.0,
+            evidence: 'Zipkin span ID',
+        },
+        {
+            target: 'ts',
+            source_path: '$[*].timestamp',
+            extraction_type: 'jsonpath',
+            transform: 'unix_ms_to_iso',
+            confidence: 1.0,
+            evidence: 'Zipkin timestamp in microseconds',
+        },
+        {
+            target: 'provider',
+            source_path: "$[*].tags['llm.provider']",
+            extraction_type: 'jsonpath',
+            transform: 'provider_normalize',
+            confidence: 0.85,
+            evidence: 'Tag-based provider extraction',
+        },
+        {
+            target: 'model',
+            source_path: "$[*].tags['llm.model']",
+            extraction_type: 'jsonpath',
+            transform: 'none',
+            confidence: 0.85,
+            evidence: 'Tag-based model extraction',
+        },
+        {
+            target: 'input_tokens',
+            source_path: "$[*].tags['llm.input_tokens']",
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.85,
+            evidence: 'Tag-based token extraction',
+        },
+        {
+            target: 'output_tokens',
+            source_path: "$[*].tags['llm.output_tokens']",
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.85,
+            evidence: 'Tag-based token extraction',
+        },
+        {
+            target: 'latency_ms',
+            source_path: '$[*].duration',
+            extraction_type: 'jsonpath',
+            transform: 'none', // Zipkin duration is in microseconds
+            confidence: 1.0,
+            evidence: 'Zipkin duration field (microseconds -> ms)',
+        },
+    ],
+    langsmith: [
+        {
+            target: 'id',
+            source_path: 'run_id',
+            extraction_type: 'direct',
+            transform: 'none',
+            confidence: 0.95,
+            evidence: 'LangSmith run ID',
+        },
+        {
+            target: 'ts',
+            source_path: 'start_time',
+            extraction_type: 'direct',
+            transform: 'none', // Already ISO format
+            confidence: 0.95,
+            evidence: 'LangSmith start timestamp',
+        },
+        {
+            target: 'provider',
+            source_path: 'extra.invocation_params.model_provider',
+            extraction_type: 'jsonpath',
+            transform: 'provider_normalize',
+            confidence: 0.8,
+            evidence: 'LangSmith invocation params provider',
+        },
+        {
+            target: 'model',
+            source_path: 'extra.invocation_params.model',
+            extraction_type: 'jsonpath',
+            transform: 'none',
+            confidence: 0.85,
+            evidence: 'LangSmith invocation params model',
+        },
+        {
+            target: 'input_tokens',
+            source_path: 'token_usage.prompt_tokens',
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.9,
+            evidence: 'LangSmith token usage prompt_tokens',
+        },
+        {
+            target: 'output_tokens',
+            source_path: 'token_usage.completion_tokens',
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.9,
+            evidence: 'LangSmith token usage completion_tokens',
+        },
+        {
+            target: 'latency_ms',
+            source_path: 'latency',
+            extraction_type: 'direct',
+            transform: 'duration_to_ms',
+            confidence: 0.9,
+            evidence: 'LangSmith latency field',
+        },
+    ],
+    litellm: [
+        {
+            target: 'id',
+            source_path: 'id',
+            extraction_type: 'direct',
+            transform: 'none',
+            confidence: 0.95,
+            evidence: 'LiteLLM request ID',
+        },
+        {
+            target: 'ts',
+            source_path: 'startTime',
+            extraction_type: 'direct',
+            transform: 'unix_ms_to_iso',
+            confidence: 0.9,
+            evidence: 'LiteLLM start timestamp',
+        },
+        {
+            target: 'provider',
+            source_path: 'model',
+            extraction_type: 'direct',
+            transform: 'provider_normalize', // LiteLLM uses model format like "openai/gpt-4"
+            confidence: 0.85,
+            evidence: 'LiteLLM model field (provider/model format)',
+        },
+        {
+            target: 'model',
+            source_path: 'model',
+            extraction_type: 'direct',
+            transform: 'none',
+            confidence: 0.95,
+            evidence: 'LiteLLM model field',
+        },
+        {
+            target: 'input_tokens',
+            source_path: 'usage.prompt_tokens',
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.95,
+            evidence: 'LiteLLM usage prompt_tokens',
+        },
+        {
+            target: 'output_tokens',
+            source_path: 'usage.completion_tokens',
+            extraction_type: 'jsonpath',
+            transform: 'parse_int',
+            confidence: 0.95,
+            evidence: 'LiteLLM usage completion_tokens',
+        },
+        {
+            target: 'latency_ms',
+            source_path: 'response_time_ms',
+            extraction_type: 'direct',
+            transform: 'none',
+            confidence: 1.0,
+            evidence: 'LiteLLM response_time_ms field',
+        },
+    ],
+    helicone: [
+        {
+            target: 'id',
+            source_path: 'helicone_request_id',
+            extraction_type: 'direct',
+            transform: 'none',
+            confidence: 0.95,
+            evidence: 'Helicone request ID',
+        },
+        {
+            target: 'ts',
+            source_path: 'created_at',
+            extraction_type: 'direct',
+            transform: 'none', // Helicone uses ISO format
+            confidence: 0.9,
+            evidence: 'Helicone created_at timestamp',
+        },
+        {
+            target: 'provider',
+            source_path: 'provider',
+            extraction_type: 'direct',
+            transform: 'provider_normalize',
+            confidence: 0.9,
+            evidence: 'Helicone provider field',
+        },
+        {
+            target: 'model',
+            source_path: 'model',
+            extraction_type: 'direct',
+            transform: 'none',
+            confidence: 0.95,
+            evidence: 'Helicone model field',
+        },
+        {
+            target: 'input_tokens',
+            source_path: 'prompt_tokens',
+            extraction_type: 'direct',
+            transform: 'parse_int',
+            confidence: 0.9,
+            evidence: 'Helicone prompt_tokens',
+        },
+        {
+            target: 'output_tokens',
+            source_path: 'completion_tokens',
+            extraction_type: 'direct',
+            transform: 'parse_int',
+            confidence: 0.9,
+            evidence: 'Helicone completion_tokens',
+        },
+        {
+            target: 'latency_ms',
+            source_path: 'latency_ms',
+            extraction_type: 'direct',
+            transform: 'none',
+            confidence: 1.0,
+            evidence: 'Helicone latency_ms field',
+        },
+    ],
+};
+// =============================================================================
+// FORMAT DETECTION
+// =============================================================================
+/**
+ * Detect the format type of a runtime events file.
+ *
+ * Detection strategy:
+ * 1. Try file extension heuristics
+ * 2. Sample content and check against known signatures
+ * 3. Fall back to agent-based detection for unknown formats
+ */
+export function detectFormat(content, filename) {
+    const lines = content.trim().split('\n').slice(0, SAMPLE_LINES);
+    // First, try to parse as complete JSON (object or array)
+    // This handles single-line JSON arrays which would incorrectly match JSONL
+    let parsedAsWhole;
+    try {
+        parsedAsWhole = JSON.parse(content);
+    }
+    catch {
+        parsedAsWhole = null;
+    }
+    // If it's a JSON array, check for InferenceEvent schema first
+    if (Array.isArray(parsedAsWhole) && parsedAsWhole.length > 0) {
+        const first = parsedAsWhole[0];
+        const hasRequiredFields = REQUIRED_FIELDS.every(f => f in first);
+        if (hasRequiredFields) {
+            return {
+                format_type: 'json_array',
+                confidence: 1.0,
+                evidence: 'JSON array with InferenceEvent schema',
+                sample_size: Math.min(parsedAsWhole.length, SAMPLE_LINES),
+                requires_agent: false,
+            };
+        }
+    }
+    // Check if it's likely JSONL (newline-delimited JSON)
+    // Note: Multi-line content where each line is valid JSON
+    if (lines.length > 1) {
+        const isJSONL = lines.every(line => {
+            const trimmed = line.trim();
+            if (!trimmed)
+                return true; // Empty lines are ok
+            try {
+                JSON.parse(trimmed);
+                return true;
+            }
+            catch {
+                return false;
+            }
+        });
+        if (isJSONL) {
+            // Parse first non-empty line
+            const firstLine = lines.find(l => l.trim());
+            if (firstLine) {
+                try {
+                    const firstEvent = JSON.parse(firstLine);
+                    // Check if JSONL matches InferenceEvent schema
+                    const hasRequiredFields = REQUIRED_FIELDS.every(f => f in firstEvent);
+                    if (hasRequiredFields) {
+                        return {
+                            format_type: 'jsonl',
+                            confidence: 1.0,
+                            evidence: 'JSONL with InferenceEvent schema (all required fields present)',
+                            sample_size: lines.length,
+                            requires_agent: false,
+                        };
+                    }
+                    // Check against known format signatures for JSONL data
+                    // Only match if structuralCheck passes (required for JSONL format detection)
+                    const jsonStr = JSON.stringify(firstEvent);
+                    for (const [formatType, signature] of Object.entries(FORMAT_SIGNATURES)) {
+                        // For JSONL, require structuralCheck to pass (if defined)
+                        if (signature.structuralCheck) {
+                            const structuralMatch = signature.structuralCheck(firstEvent);
+                            if (!structuralMatch)
+                                continue;
+                            const patternMatches = signature.patterns.filter(p => p.test(jsonStr)).length;
+                            const patternRatio = patternMatches / signature.patterns.length;
+                            const confidence = Math.max(0.8, patternRatio) * signature.confidence;
+                            return {
+                                format_type: formatType,
+                                confidence,
+                                evidence: `JSONL with ${formatType} format (structural match, ${patternMatches}/${signature.patterns.length} patterns)`,
+                                sample_size: lines.length,
+                                requires_agent: true,
+                            };
+                        }
+                    }
+                    // JSONL but unknown schema - mark as custom_json requiring agent
+                    return {
+                        format_type: 'custom_json',
+                        confidence: 0.7,
+                        evidence: 'JSONL with custom schema - requires field mapping',
+                        sample_size: lines.length,
+                        requires_agent: true,
+                    };
+                }
+                catch {
+                    // Continue to other detection methods
+                }
+            }
+        }
+    }
+    // Try to parse as JSON (array or object)
+    let parsedData;
+    try {
+        parsedData = JSON.parse(content);
+    }
+    catch {
+        // Not valid JSON, check for CSV/TSV or text logs
+        return detectNonJSONFormat(content, lines, filename);
+    }
+    // Check if it's a JSON array with InferenceEvent schema
+    if (Array.isArray(parsedData) && parsedData.length > 0) {
+        const first = parsedData[0];
+        const hasRequiredFields = REQUIRED_FIELDS.every(f => f in first);
+        if (hasRequiredFields) {
+            return {
+                format_type: 'json_array',
+                confidence: 1.0,
+                evidence: 'JSON array with InferenceEvent schema',
+                sample_size: Math.min(parsedData.length, SAMPLE_LINES),
+                requires_agent: false,
+            };
+        }
+    }
+    // Check against known format signatures
+    // Require structural match for reliable detection
+    const contentStr = JSON.stringify(parsedData);
+    for (const [formatType, signature] of Object.entries(FORMAT_SIGNATURES)) {
+        // Require structuralCheck to pass for format identification
+        if (signature.structuralCheck) {
+            const structuralMatch = signature.structuralCheck(parsedData);
+            if (!structuralMatch)
+                continue;
+            const patternMatches = signature.patterns.filter(p => p.test(contentStr)).length;
+            const patternRatio = patternMatches / signature.patterns.length;
+            const confidence = Math.max(0.8, patternRatio) * signature.confidence;
+            return {
+                format_type: formatType,
+                confidence,
+                evidence: `Matched ${formatType} format (structural match, ${patternMatches}/${signature.patterns.length} patterns)`,
+                sample_size: Array.isArray(parsedData) ? parsedData.length : 1,
+                requires_agent: true, // Known formats still need agent for field mapping
+            };
+        }
+    }
+    // Unknown JSON structure - requires agent normalization
+    return {
+        format_type: 'custom_json',
+        confidence: 0.5,
+        evidence: 'Valid JSON but unknown schema - requires agent normalization',
+        sample_size: Array.isArray(parsedData) ? parsedData.length : 1,
+        requires_agent: true,
+    };
+}
+/**
+ * Detect non-JSON formats (CSV, TSV, text logs).
+ */
+function detectNonJSONFormat(content, lines, filename) {
+    // Check for CSV
+    const firstLine = lines[0];
+    if (firstLine.includes(',')) {
+        const headers = firstLine.split(',').map(h => h.trim().toLowerCase());
+        const hasLLMHeaders = ['provider', 'model', 'latency', 'tokens'].some(h => headers.some(header => header.includes(h)));
+        if (hasLLMHeaders) {
+            return {
+                format_type: 'csv',
+                confidence: 0.9,
+                evidence: 'CSV with LLM-related headers detected',
+                sample_size: lines.length,
+                requires_agent: false,
+            };
+        }
+        return {
+            format_type: 'csv',
+            confidence: 0.7,
+            evidence: 'CSV format detected but headers may need mapping',
+            sample_size: lines.length,
+            requires_agent: true,
+        };
+    }
+    // Check for TSV
+    if (firstLine.includes('\t')) {
+        return {
+            format_type: 'tsv',
+            confidence: 0.8,
+            evidence: 'Tab-separated values detected',
+            sample_size: lines.length,
+            requires_agent: true,
+        };
+    }
+    // Structured text logs
+    const logPatterns = [
+        /^\d{4}-\d{2}-\d{2}/, // ISO date prefix
+        /^\[\d+\]/, // Timestamp prefix
+        /level=(info|warn|error|debug)/i,
+        /provider=\w+/,
+        /model=\w+/,
+    ];
+    const logMatchCount = logPatterns.filter(p => lines.some(line => p.test(line))).length;
+    if (logMatchCount >= 2) {
+        return {
+            format_type: 'custom_text',
+            confidence: 0.6,
+            evidence: `Structured text logs detected (${logMatchCount} patterns matched)`,
+            sample_size: lines.length,
+            requires_agent: true,
+        };
+    }
+    return {
+        format_type: 'unknown',
+        confidence: 0.3,
+        evidence: 'Could not determine format - manual field mapping may be required',
+        sample_size: lines.length,
+        requires_agent: true,
+    };
+}
+// =============================================================================
+// AGENT-BASED NORMALIZATION
+// =============================================================================
+// Load normalization prompt from YAML (with hardcoded fallback)
+function getNormalizationPrompt() {
+    const prompt = loadPrompt('format-normalizer');
+    if (prompt) {
+        return prompt.prompt;
+    }
+    // Fallback to hardcoded prompt if YAML not available
+    return `You are an expert at parsing log formats and trace data. Analyze the following sample data and determine field mappings to the InferenceEvent schema.
+
+The target InferenceEvent schema requires these fields:
+- id (string): Unique event identifier
+- ts (string): ISO 8601 timestamp
+- provider (string): LLM provider (openai, anthropic, google, etc.)
+- model (string): Model name (gpt-4o, claude-3-5-sonnet, etc.)
+- input_tokens (number): Input/prompt token count
+- output_tokens (number): Output/completion token count
+- latency_ms (number): Request latency in milliseconds
+
+Optional fields:
+- streaming (boolean), ttft_ms (number), batch_size (number), cached (boolean), retry_count (number)
+
+For each target field, provide:
+1. The source path/expression to extract the value
+2. The extraction type (direct, jsonpath, regex, computed)
+3. Any transform needed (unix_ms_to_iso, unix_nano_to_iso, parse_int, etc.)
+4. Your confidence (0.0-1.0) in this mapping
+5. Evidence explaining why you chose this mapping
+
+If a field cannot be mapped, indicate it as unmappable with confidence 0.
+
+Respond in JSON format:
+{
+  "format_type": "detected format name",
+  "mappings": [
+    {
+      "target": "field_name",
+      "source_path": "path or expression",
+      "extraction_type": "direct|jsonpath|regex|computed",
+      "transform": "none|unix_ms_to_iso|parse_int|...",
+      "confidence": 0.9,
+      "evidence": "explanation"
+    }
+  ],
+  "unmapped_fields": ["fields that could not be mapped"],
+  "warnings": ["any issues or caveats"]
+}`;
+}
+/**
+ * Use LLM agent to normalize an unknown format.
+ */
+export async function normalizeWithAgent(content, detection, options = {}) {
+    // Check for API key
+    const apiKey = process.env.ANTHROPIC_API_KEY;
+    if (!apiKey) {
+        return createFallbackResult(detection, 'No ANTHROPIC_API_KEY - agent normalization unavailable');
+    }
+    // Sample content for the agent
+    const sampleLines = content.trim().split('\n').slice(0, SAMPLE_LINES);
+    const sampleContent = sampleLines.join('\n');
+    // Build context prompt
+    let contextPrompt = '';
+    if (options.codebase_context) {
+        const scanResult = options.codebase_context;
+        contextPrompt = `\n\nCodebase context available:
+- ${scanResult.files.length} files scanned
+- Languages: ${scanResult.summary.languages.join(', ')}
+- ${scanResult.summary.totalCandidates} potential inference points detected
+
+This may help identify logging patterns and field names used in the application.`;
+    }
+    // User hints
+    let hintsPrompt = '';
+    if (options.format_hint) {
+        hintsPrompt += `\nUser hint: Format appears to be "${options.format_hint}"`;
+    }
+    if (options.field_hints) {
+        hintsPrompt += `\nUser-provided field mappings: ${JSON.stringify(options.field_hints)}`;
+    }
+    try {
+        // Use Claude Agent SDK query() function
+        const agentQuery = query({
+            prompt: `${getNormalizationPrompt()}${contextPrompt}${hintsPrompt}
+
+Detected format: ${detection.format_type} (confidence: ${detection.confidence})
+
+Sample data:
+\`\`\`
+${sampleContent}
+\`\`\``,
+            options: {
+                model: LLM_MODEL,
+                tools: [],
+                permissionMode: 'plan',
+                cwd: process.cwd(),
+            },
+        });
+        // Collect all messages from the async generator
+        const messages = [];
+        for await (const message of agentQuery) {
+            messages.push(message);
+        }
+        // Parse LLM response
+        const responseText = extractTextFromMessages(messages);
+        const jsonMatch = responseText.match(/\{[\s\S]*\}/);
+        if (!jsonMatch) {
+            return createFallbackResult(detection, 'Could not parse agent response');
+        }
+        const agentResult = JSON.parse(jsonMatch[0]);
+        // Validate and build result
+        const mappings = (agentResult.mappings || []).map(m => ({
+            target: m.target,
+            source_path: m.source_path,
+            extraction_type: m.extraction_type,
+            transform: (m.transform || 'none'),
+            confidence: m.confidence,
+            evidence: m.evidence,
+        }));
+        // Check confidence threshold
+        const avgConfidence = mappings.reduce((sum, m) => sum + m.confidence, 0) / mappings.length;
+        const warnings = agentResult.warnings || [];
+        if (avgConfidence < MIN_CONFIDENCE_THRESHOLD && !options.lenient) {
+            warnings.push(`Average mapping confidence (${avgConfidence.toFixed(2)}) is below threshold (${MIN_CONFIDENCE_THRESHOLD}). ` +
+                `Use --lenient flag to accept low-confidence mappings.`);
+        }
+        return {
+            detection: {
+                ...detection,
+                format_type: agentResult.format_type || detection.format_type,
+            },
+            mappings,
+            unmapped_fields: agentResult.unmapped_fields || [],
+            warnings,
+            audit: {
+                normalized_at: new Date().toISOString(),
+                agent_used: true,
+                codebase_context_used: !!options.codebase_context,
+                llm_model: LLM_MODEL,
+            },
+        };
+    }
+    catch (error) {
+        return createFallbackResult(detection, `Agent normalization failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+/**
+ * Create a fallback normalization result when agent is unavailable.
+ */
+function createFallbackResult(detection, warning) {
+    // Use predefined mappings if available
+    const predefinedMappings = PREDEFINED_MAPPINGS[detection.format_type];
+    return {
+        detection,
+        mappings: predefinedMappings || [],
+        unmapped_fields: predefinedMappings ? [] : REQUIRED_FIELDS,
+        warnings: [warning],
+        audit: {
+            normalized_at: new Date().toISOString(),
+            agent_used: false,
+            codebase_context_used: false,
+        },
+    };
+}
+// =============================================================================
+// FIELD EXTRACTION
+// =============================================================================
+/**
+ * Apply a transformation to an extracted value.
+ */
+function applyTransform(value, transform) {
+    if (value === null || value === undefined)
+        return value;
+    switch (transform) {
+        case 'none':
+            return value;
+        case 'unix_ms_to_iso':
+            return new Date(Number(value)).toISOString();
+        case 'unix_s_to_iso':
+            return new Date(Number(value) * 1000).toISOString();
+        case 'unix_nano_to_iso':
+            return new Date(Number(value) / 1_000_000).toISOString();
+        case 'duration_to_ms': {
+            const str = String(value);
+            const match = str.match(/^([\d.]+)(ms|s|m)?$/);
+            if (match) {
+                const num = parseFloat(match[1]);
+                const unit = match[2] || 'ms';
+                switch (unit) {
+                    case 's': return num * 1000;
+                    case 'm': return num * 60000;
+                    default: return num;
+                }
+            }
+            return parseFloat(str);
+        }
+        case 'parse_int':
+            return parseInt(String(value), 10);
+        case 'parse_float':
+            return parseFloat(String(value));
+        case 'lowercase':
+            return String(value).toLowerCase();
+        case 'provider_normalize': {
+            const str = String(value).toLowerCase();
+            // Normalize common provider variations
+            if (str.includes('openai'))
+                return 'openai';
+            if (str.includes('anthropic'))
+                return 'anthropic';
+            if (str.includes('google'))
+                return 'google';
+            if (str.includes('azure'))
+                return 'azure_openai';
+            if (str.includes('bedrock'))
+                return 'bedrock';
+            if (str.includes('together'))
+                return 'together';
+            if (str.includes('groq'))
+                return 'groq';
+            return str;
+        }
+        default:
+            return value;
+    }
+}
+/**
+ * Extract a value from an object using a simple path.
+ * Supports basic dot notation and array access.
+ */
+function extractValue(obj, path) {
+    if (path.startsWith('$.')) {
+        path = path.slice(2);
+    }
+    const parts = path.split(/\.|\[(\d+)\]/).filter(Boolean);
+    let current = obj;
+    for (const part of parts) {
+        if (current === null || current === undefined)
+            return undefined;
+        if (typeof current === 'object') {
+            current = current[part];
+        }
+        else {
+            return undefined;
+        }
+    }
+    return current;
+}
+/**
+ * Extract InferenceEvents from normalized data using field mappings.
+ */
+export function extractEvents(content, normalization) {
+    const events = [];
+    const errors = [];
+    // Parse content based on format type
+    let records;
+    try {
+        const formatType = normalization.detection.format_type;
+        if (formatType === 'jsonl') {
+            records = content.trim().split('\n').map(line => JSON.parse(line));
+        }
+        else if (formatType === 'json_array') {
+            records = JSON.parse(content);
+        }
+        else if (formatType === 'csv' || formatType === 'tsv') {
+            const delimiter = formatType === 'csv' ? ',' : '\t';
+            const lines = content.trim().split('\n');
+            const headers = lines[0].split(delimiter).map(h => h.trim());
+            records = lines.slice(1).map(line => {
+                const values = line.split(delimiter);
+                const obj = {};
+                headers.forEach((h, i) => { obj[h] = values[i]?.trim() || ''; });
+                return obj;
+            });
+        }
+        else {
+            // For complex formats (OTEL, Jaeger, etc.), parse and flatten
+            const data = JSON.parse(content);
+            records = flattenComplexFormat(data, normalization.detection.format_type);
+        }
+    }
+    catch (error) {
+        errors.push(`Failed to parse content: ${error instanceof Error ? error.message : String(error)}`);
+        return { events, errors };
+    }
+    // Extract events using mappings
+    for (let i = 0; i < records.length; i++) {
+        const record = records[i];
+        const event = {};
+        for (const mapping of normalization.mappings) {
+            try {
+                let value;
+                if (mapping.extraction_type === 'computed') {
+                    // Handle computed fields (e.g., latency = end - start)
+                    value = computeValue(record, mapping.source_path);
+                }
+                else if (mapping.extraction_type === 'constant') {
+                    value = mapping.source_path;
+                }
+                else {
+                    value = extractValue(record, mapping.source_path);
+                }
+                if (value !== undefined && value !== null) {
+                    const transformed = applyTransform(value, mapping.transform);
+                    event[mapping.target] = transformed;
+                }
+            }
+            catch (error) {
+                // Skip this field for this record
+            }
+        }
+        // Validate required fields
+        const missingFields = REQUIRED_FIELDS.filter(f => !(f in event));
+        if (missingFields.length === 0) {
+            events.push(event);
+        }
+        else {
+            errors.push(`Record ${i + 1}: Missing required fields: ${missingFields.join(', ')}`);
+        }
+    }
+    return { events, errors };
+}
+/**
+ * Flatten complex nested formats (OTEL, Jaeger, Zipkin) into individual records.
+ */
+function flattenComplexFormat(data, formatType) {
+    if (formatType === 'otel') {
+        return flattenOTEL(data);
+    }
+    else if (formatType === 'jaeger') {
+        return flattenJaeger(data);
+    }
+    else if (formatType === 'zipkin') {
+        // Zipkin is already an array of spans
+        return Array.isArray(data) ? data : [];
+    }
+    // For unknown formats, try to handle arrays or wrap single object
+    if (Array.isArray(data))
+        return data;
+    return [data];
+}
+/**
+ * Flatten OTEL traces into individual spans.
+ */
+function flattenOTEL(data) {
+    const spans = [];
+    const otelData = data;
+    for (const resourceSpan of otelData.resourceSpans || []) {
+        for (const scopeSpan of resourceSpan.scopeSpans || []) {
+            for (const span of scopeSpan.spans || []) {
+                spans.push(span);
+            }
+        }
+    }
+    return spans;
+}
+/**
+ * Flatten Jaeger traces into individual spans.
+ */
+function flattenJaeger(data) {
+    const spans = [];
+    const jaegerData = data;
+    for (const trace of jaegerData.data || []) {
+        for (const span of trace.spans || []) {
+            spans.push(span);
+        }
+    }
+    return spans;
+}
+/**
+ * Compute a derived value from an expression.
+ */
+function computeValue(record, expression) {
+    // Simple expression parser for common patterns
+    // e.g., "(endTimeUnixNano - startTimeUnixNano) / 1000000"
+    const match = expression.match(/\((\w+)\s*-\s*(\w+)\)\s*\/\s*(\d+)/);
+    if (match) {
+        const [, endField, startField, divisor] = match;
+        const endValue = Number(record[endField]);
+        const startValue = Number(record[startField]);
+        if (!isNaN(endValue) && !isNaN(startValue)) {
+            return (endValue - startValue) / Number(divisor);
+        }
+    }
+    // Try direct field access
+    const fieldMatch = expression.match(/^\$\.(\w+)$/);
+    if (fieldMatch) {
+        return Number(record[fieldMatch[1]]);
+    }
+    return undefined;
+}
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+/**
+ * Main entry point: Detect format and normalize runtime events.
+ *
+ * This function implements the complete normalization pipeline:
+ * 1. Detect format type from content
+ * 2. For direct-parse formats (JSONL, JSON array), parse directly
+ * 3. Apply predefined mappings for known complex formats
+ * 4. Use agent for unknown formats (if API key available)
+ */
+export async function normalizeRuntimeEvents(content, options = {}) {
+    // Step 1: Detect format
+    const detection = detectFormat(content, options.format_hint?.toString());
+    // Override with user hint if provided
+    if (options.format_hint) {
+        detection.format_type = options.format_hint;
+        detection.evidence = `User-specified format: ${options.format_hint}`;
+    }
+    // Step 2: For direct-parse formats, parse directly without field mappings
+    if (detection.format_type === 'jsonl' || detection.format_type === 'json_array') {
+        const events = [];
+        const errors = [];
+        try {
+            if (detection.format_type === 'jsonl') {
+                // JSONL: one JSON object per line
+                const lines = content.trim().split('\n');
+                for (let i = 0; i < lines.length; i++) {
+                    const line = lines[i].trim();
+                    if (!line)
+                        continue;
+                    try {
+                        const data = JSON.parse(line);
+                        events.push(validateAndConvertEvent(data, i + 1));
+                    }
+                    catch (e) {
+                        errors.push(`Line ${i + 1}: ${e instanceof Error ? e.message : String(e)}`);
+                    }
+                }
+            }
+            else {
+                // JSON array
+                const data = JSON.parse(content);
+                if (Array.isArray(data)) {
+                    for (let i = 0; i < data.length; i++) {
+                        try {
+                            events.push(validateAndConvertEvent(data[i], i + 1));
+                        }
+                        catch (e) {
+                            errors.push(`Record ${i + 1}: ${e instanceof Error ? e.message : String(e)}`);
+                        }
+                    }
+                }
+            }
+        }
+        catch (e) {
+            errors.push(`Parse error: ${e instanceof Error ? e.message : String(e)}`);
+        }
+        const normalization = {
+            detection,
+            mappings: [],
+            unmapped_fields: [],
+            warnings: errors.length > 0 ? [`${errors.length} records had parsing issues`] : [],
+            audit: {
+                normalized_at: new Date().toISOString(),
+                agent_used: false,
+                codebase_context_used: false,
+            },
+        };
+        return { events, normalization, errors };
+    }
+    // Step 3: Get or generate field mappings for complex formats
+    let normalization;
+    if (!detection.requires_agent || detection.confidence >= 0.95) {
+        // Use predefined mappings for known complex formats
+        const predefinedMappings = PREDEFINED_MAPPINGS[detection.format_type];
+        normalization = {
+            detection,
+            mappings: predefinedMappings || [],
+            unmapped_fields: [],
+            warnings: [],
+            audit: {
+                normalized_at: new Date().toISOString(),
+                agent_used: false,
+                codebase_context_used: false,
+            },
+        };
+    }
+    else {
+        // Agent normalization required
+        normalization = await normalizeWithAgent(content, detection, options);
+    }
+    // Step 4: Extract events using field mappings
+    const { events, errors } = extractEvents(content, normalization);
+    // Add extraction errors to warnings
+    if (errors.length > 0 && errors.length <= 5) {
+        normalization.warnings.push(...errors);
+    }
+    else if (errors.length > 5) {
+        normalization.warnings.push(`${errors.length} records failed extraction (first: ${errors[0]})`);
+    }
+    return { events, normalization, errors };
+}
+/**
+ * Validate and convert raw data to InferenceEvent.
+ * Used for direct-parse formats (JSONL, JSON array).
+ */
+function validateAndConvertEvent(data, recordNum) {
+    if (typeof data !== 'object' || data === null) {
+        throw new Error(`Expected object, got ${typeof data}`);
+    }
+    const obj = data;
+    const errors = [];
+    // Required fields
+    if (typeof obj.id !== 'string')
+        errors.push("Missing 'id'");
+    if (typeof obj.ts !== 'string')
+        errors.push("Missing 'ts'");
+    if (typeof obj.provider !== 'string')
+        errors.push("Missing 'provider'");
+    if (typeof obj.model !== 'string')
+        errors.push("Missing 'model'");
+    if (typeof obj.input_tokens !== 'number')
+        errors.push("Missing 'input_tokens'");
+    if (typeof obj.output_tokens !== 'number')
+        errors.push("Missing 'output_tokens'");
+    if (typeof obj.latency_ms !== 'number')
+        errors.push("Missing 'latency_ms'");
+    if (errors.length > 0) {
+        throw new Error(`Record ${recordNum}: ${errors.join(', ')}`);
+    }
+    return {
+        id: obj.id,
+        ts: obj.ts,
+        provider: obj.provider,
+        model: obj.model,
+        input_tokens: obj.input_tokens,
+        output_tokens: obj.output_tokens,
+        latency_ms: obj.latency_ms,
+        // Optional fields
+        intent: typeof obj.intent === 'string' ? obj.intent : undefined,
+        callsite_id: typeof obj.callsite_id === 'string' ? obj.callsite_id : undefined,
+        streaming: typeof obj.streaming === 'boolean' ? obj.streaming : undefined,
+        ttft_ms: typeof obj.ttft_ms === 'number' ? obj.ttft_ms : undefined,
+        batch_size: typeof obj.batch_size === 'number' ? obj.batch_size : undefined,
+        batch_id: typeof obj.batch_id === 'string' ? obj.batch_id : undefined,
+        cached: typeof obj.cached === 'boolean' ? obj.cached : undefined,
+        retry_count: typeof obj.retry_count === 'number' ? obj.retry_count : undefined,
+        fallback_used: typeof obj.fallback_used === 'boolean' ? obj.fallback_used : undefined,
+        original_model: typeof obj.original_model === 'string' ? obj.original_model : undefined,
+    };
+}
+/**
+ * Get predefined mappings for a known format type.
+ */
+export function getPredefinedMappings(formatType) {
+    return PREDEFINED_MAPPINGS[formatType];
+}
+/**
+ * Check if a format type requires agent normalization.
+ */
+export function requiresAgentNormalization(formatType) {
+    return !['jsonl', 'json_array', 'csv', 'tsv'].includes(formatType);
+}
+/**
+ * Parse and validate a field mapping string.
+ * Format: "target=source,target2=source2,..."
+ * Example: "model=llm_model,latency_ms=duration_ms"
+ */
+export function validateFieldMappings(mappingStr) {
+    const result = {
+        valid: true,
+        mappings: [],
+        errors: [],
+        warnings: [],
+    };
+    if (!mappingStr || mappingStr.trim() === '') {
+        return result; // Empty mapping is valid (no custom mappings)
+    }
+    // Parse mapping string
+    const pairs = mappingStr.split(',').map(p => p.trim()).filter(p => p);
+    for (const pair of pairs) {
+        const parts = pair.split('=');
+        if (parts.length !== 2) {
+            result.errors.push(`Invalid mapping format: "${pair}" (expected target=source)`);
+            result.valid = false;
+            continue;
+        }
+        const [target, source] = parts.map(p => p.trim());
+        if (!target || !source) {
+            result.errors.push(`Empty target or source in mapping: "${pair}"`);
+            result.valid = false;
+            continue;
+        }
+        // Validate target is a known InferenceEvent field
+        const validTargets = [
+            'id', 'ts', 'provider', 'model', 'input_tokens', 'output_tokens',
+            'latency_ms', 'callsite_id', 'streaming', 'cached', 'batch_id',
+            'batch_size', 'retry_count', 'fallback_used', 'error_code', 'error_message'
+        ];
+        if (!validTargets.includes(target)) {
+            result.warnings.push(`Unknown target field "${target}" - may not be used`);
+        }
+        result.mappings.push({ target, source });
+    }
+    // Check for required fields coverage
+    const mappedTargets = new Set(result.mappings.map(m => m.target));
+    const criticalFields = ['model', 'latency_ms'];
+    for (const field of criticalFields) {
+        if (!mappedTargets.has(field)) {
+            result.warnings.push(`Consider mapping "${field}" for better analysis`);
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=format-normalizer.js.map