llmflow 0.3.1

package/otlp-logs.js

@@ -0,0 +1,238 @@
+/**
+ * OTLP Logs Endpoint for LLMFlow
+ * 
+ * Accepts OTLP/HTTP JSON logs and stores them for AI CLI tool observability.
+ * Supports log events from Claude Code, Codex CLI, Gemini CLI, etc.
+ * 
+ * Supports:
+ * - OTLP/HTTP JSON format (Content-Type: application/json)
+ * - event.name extraction for AI CLI tools
+ * - Correlation with traces via trace_id/span_id
+ */
+
+const { v4: uuidv4 } = require('uuid');
+const db = require('./db');
+
+/**
+ * Extract attributes from OTLP KeyValue array format
+ * OTLP attributes are: [{ key: "foo", value: { stringValue: "bar" } }, ...]
+ */
+function extractAttributes(attrs) {
+    if (!attrs || !Array.isArray(attrs)) return {};
+    
+    const result = {};
+    for (const attr of attrs) {
+        const key = attr.key;
+        const val = attr.value;
+        if (!val) continue;
+        
+        if (val.stringValue !== undefined) result[key] = val.stringValue;
+        else if (val.intValue !== undefined) result[key] = parseInt(val.intValue, 10);
+        else if (val.doubleValue !== undefined) result[key] = val.doubleValue;
+        else if (val.boolValue !== undefined) result[key] = val.boolValue;
+        else if (val.arrayValue?.values) {
+            result[key] = val.arrayValue.values.map(v => 
+                v.stringValue ?? v.intValue ?? v.doubleValue ?? v.boolValue ?? null
+            );
+        }
+    }
+    return result;
+}
+
+/**
+ * Convert hex string to normalized format
+ */
+function normalizeId(hexId) {
+    if (!hexId) return null;
+    return hexId.replace(/-/g, '').toLowerCase();
+}
+
+/**
+ * Convert nanoseconds timestamp to milliseconds
+ */
+function nanoToMs(nanoStr) {
+    if (!nanoStr) return Date.now();
+    const nano = BigInt(nanoStr);
+    return Number(nano / BigInt(1000000));
+}
+
+/**
+ * Extract log body from OTLP AnyValue format
+ */
+function extractBody(body) {
+    if (!body) return null;
+    
+    if (body.stringValue !== undefined) return body.stringValue;
+    if (body.intValue !== undefined) return String(body.intValue);
+    if (body.doubleValue !== undefined) return String(body.doubleValue);
+    if (body.boolValue !== undefined) return String(body.boolValue);
+    if (body.arrayValue?.values) {
+        return JSON.stringify(body.arrayValue.values.map(v => 
+            v.stringValue ?? v.intValue ?? v.doubleValue ?? v.boolValue ?? null
+        ));
+    }
+    if (body.kvlistValue?.values) {
+        const obj = {};
+        for (const kv of body.kvlistValue.values) {
+            obj[kv.key] = kv.value?.stringValue ?? kv.value?.intValue ?? kv.value?.doubleValue ?? null;
+        }
+        return JSON.stringify(obj);
+    }
+    if (body.bytesValue) {
+        return `[binary: ${body.bytesValue.length} bytes]`;
+    }
+    
+    return JSON.stringify(body);
+}
+
+/**
+ * Extract event name from log attributes
+ * Common patterns for AI CLI tools
+ */
+function extractLogEventName(attrs) {
+    return attrs['event.name'] 
+        || attrs['log.event.name']
+        || attrs['name']
+        || attrs['event_name']
+        || null;
+}
+
+/**
+ * Map OTLP severity number to text if not provided
+ * Per OTEL spec: https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
+ */
+function getSeverityText(severityNumber, providedText) {
+    if (providedText) return providedText;
+    if (!severityNumber) return null;
+    
+    const severityMap = {
+        1: 'TRACE', 2: 'TRACE', 3: 'TRACE', 4: 'TRACE',
+        5: 'DEBUG', 6: 'DEBUG', 7: 'DEBUG', 8: 'DEBUG',
+        9: 'INFO', 10: 'INFO', 11: 'INFO', 12: 'INFO',
+        13: 'WARN', 14: 'WARN', 15: 'WARN', 16: 'WARN',
+        17: 'ERROR', 18: 'ERROR', 19: 'ERROR', 20: 'ERROR',
+        21: 'FATAL', 22: 'FATAL', 23: 'FATAL', 24: 'FATAL'
+    };
+    
+    return severityMap[severityNumber] || 'UNSPECIFIED';
+}
+
+/**
+ * Process OTLP/HTTP JSON logs request
+ * 
+ * Expected format (OTLP/HTTP JSON):
+ * {
+ *   "resourceLogs": [
+ *     {
+ *       "resource": { "attributes": [...] },
+ *       "scopeLogs": [
+ *         {
+ *           "scope": { "name": "...", "version": "..." },
+ *           "logRecords": [
+ *             {
+ *               "timeUnixNano": "...",
+ *               "observedTimeUnixNano": "...",
+ *               "severityNumber": 9,
+ *               "severityText": "INFO",
+ *               "body": { "stringValue": "..." },
+ *               "attributes": [...],
+ *               "traceId": "hex",
+ *               "spanId": "hex"
+ *             }
+ *           ]
+ *         }
+ *       ]
+ *     }
+ *   ]
+ * }
+ */
+function processOtlpLogs(body) {
+    const results = {
+        accepted: 0,
+        rejected: 0,
+        errors: []
+    };
+    
+    if (!body || !body.resourceLogs) {
+        return results;
+    }
+    
+    for (const resourceLog of body.resourceLogs) {
+        const resourceAttrs = extractAttributes(resourceLog.resource?.attributes);
+        
+        for (const scopeLog of (resourceLog.scopeLogs || [])) {
+            const scopeInfo = scopeLog.scope || {};
+            
+            for (const logRecord of (scopeLog.logRecords || [])) {
+                try {
+                    const attrs = extractAttributes(logRecord.attributes);
+                    const logId = uuidv4();
+                    
+                    db.insertLog({
+                        id: logId,
+                        timestamp: nanoToMs(logRecord.timeUnixNano),
+                        observed_timestamp: nanoToMs(logRecord.observedTimeUnixNano),
+                        severity_number: logRecord.severityNumber || null,
+                        severity_text: getSeverityText(logRecord.severityNumber, logRecord.severityText),
+                        body: extractBody(logRecord.body),
+                        trace_id: normalizeId(logRecord.traceId),
+                        span_id: normalizeId(logRecord.spanId),
+                        event_name: extractLogEventName(attrs),
+                        service_name: resourceAttrs['service.name'] || 'unknown',
+                        scope_name: scopeInfo.name || null,
+                        attributes: attrs,
+                        resource_attributes: resourceAttrs
+                    });
+                    results.accepted++;
+                } catch (err) {
+                    results.rejected++;
+                    results.errors.push(err.message);
+                }
+            }
+        }
+    }
+    
+    return results;
+}
+
+/**
+ * Express middleware for OTLP logs endpoint
+ */
+function createLogsHandler() {
+    return (req, res) => {
+        const contentType = req.headers['content-type'] || '';
+        
+        if (!contentType.includes('application/json')) {
+            return res.status(415).json({
+                error: 'Unsupported Media Type',
+                message: 'Only application/json is supported. Use OTLP/HTTP JSON format.'
+            });
+        }
+        
+        try {
+            const results = processOtlpLogs(req.body);
+            
+            res.status(200).json({
+                partialSuccess: results.rejected > 0 ? {
+                    rejectedLogRecords: results.rejected,
+                    errorMessage: results.errors.slice(0, 5).join('; ')
+                } : undefined
+            });
+        } catch (err) {
+            res.status(500).json({
+                error: 'Internal Server Error',
+                message: err.message
+            });
+        }
+    };
+}
+
+module.exports = {
+    processOtlpLogs,
+    createLogsHandler,
+    extractAttributes,
+    extractBody,
+    extractLogEventName,
+    normalizeId,
+    nanoToMs
+};

package/otlp-metrics.js

@@ -0,0 +1,300 @@
+/**
+ * OTLP Metrics Endpoint for LLMFlow
+ * 
+ * Accepts OTLP/HTTP JSON metrics and stores them for AI CLI tool observability.
+ * Supports metrics from Claude Code, Gemini CLI, etc.
+ * 
+ * Supports:
+ * - OTLP/HTTP JSON format (Content-Type: application/json)
+ * - Sum (Counter), Gauge, Histogram metric types
+ * - Token usage and cost metrics extraction
+ */
+
+const { v4: uuidv4 } = require('uuid');
+const db = require('./db');
+
+/**
+ * Extract attributes from OTLP KeyValue array format
+ */
+function extractAttributes(attrs) {
+    if (!attrs || !Array.isArray(attrs)) return {};
+    
+    const result = {};
+    for (const attr of attrs) {
+        const key = attr.key;
+        const val = attr.value;
+        if (!val) continue;
+        
+        if (val.stringValue !== undefined) result[key] = val.stringValue;
+        else if (val.intValue !== undefined) result[key] = parseInt(val.intValue, 10);
+        else if (val.doubleValue !== undefined) result[key] = val.doubleValue;
+        else if (val.boolValue !== undefined) result[key] = val.boolValue;
+        else if (val.arrayValue?.values) {
+            result[key] = val.arrayValue.values.map(v => 
+                v.stringValue ?? v.intValue ?? v.doubleValue ?? v.boolValue ?? null
+            );
+        }
+    }
+    return result;
+}
+
+/**
+ * Convert nanoseconds timestamp to milliseconds
+ */
+function nanoToMs(nanoStr) {
+    if (!nanoStr) return Date.now();
+    const nano = BigInt(nanoStr);
+    return Number(nano / BigInt(1000000));
+}
+
+/**
+ * Extract value from data point
+ * Handles asInt, asDouble, and various OTLP value formats
+ */
+function extractValue(dataPoint) {
+    if (dataPoint.asInt !== undefined) {
+        return { value_int: parseInt(dataPoint.asInt, 10), value_double: null };
+    }
+    if (dataPoint.asDouble !== undefined) {
+        return { value_int: null, value_double: dataPoint.asDouble };
+    }
+    if (dataPoint.value !== undefined) {
+        if (typeof dataPoint.value === 'number') {
+            if (Number.isInteger(dataPoint.value)) {
+                return { value_int: dataPoint.value, value_double: null };
+            }
+            return { value_int: null, value_double: dataPoint.value };
+        }
+    }
+    return { value_int: null, value_double: null };
+}
+
+/**
+ * Process Sum metric (Counter)
+ * Sum metrics have dataPoints with aggregationTemporality
+ */
+function processSum(metric, resourceAttrs, scopeInfo) {
+    const results = [];
+    const sum = metric.sum;
+    if (!sum || !sum.dataPoints) return results;
+
+    for (const dp of sum.dataPoints) {
+        const attrs = extractAttributes(dp.attributes);
+        const { value_int, value_double } = extractValue(dp);
+        
+        results.push({
+            id: uuidv4(),
+            timestamp: nanoToMs(dp.timeUnixNano),
+            name: metric.name,
+            description: metric.description || null,
+            unit: metric.unit || null,
+            metric_type: 'sum',
+            value_int,
+            value_double,
+            histogram_data: null,
+            service_name: resourceAttrs['service.name'] || 'unknown',
+            scope_name: scopeInfo.name || null,
+            attributes: attrs,
+            resource_attributes: resourceAttrs
+        });
+    }
+    
+    return results;
+}
+
+/**
+ * Process Gauge metric
+ * Gauge metrics represent point-in-time values
+ */
+function processGauge(metric, resourceAttrs, scopeInfo) {
+    const results = [];
+    const gauge = metric.gauge;
+    if (!gauge || !gauge.dataPoints) return results;
+
+    for (const dp of gauge.dataPoints) {
+        const attrs = extractAttributes(dp.attributes);
+        const { value_int, value_double } = extractValue(dp);
+        
+        results.push({
+            id: uuidv4(),
+            timestamp: nanoToMs(dp.timeUnixNano),
+            name: metric.name,
+            description: metric.description || null,
+            unit: metric.unit || null,
+            metric_type: 'gauge',
+            value_int,
+            value_double,
+            histogram_data: null,
+            service_name: resourceAttrs['service.name'] || 'unknown',
+            scope_name: scopeInfo.name || null,
+            attributes: attrs,
+            resource_attributes: resourceAttrs
+        });
+    }
+    
+    return results;
+}
+
+/**
+ * Process Histogram metric
+ * Histogram metrics have buckets and bounds
+ */
+function processHistogram(metric, resourceAttrs, scopeInfo) {
+    const results = [];
+    const histogram = metric.histogram;
+    if (!histogram || !histogram.dataPoints) return results;
+
+    for (const dp of histogram.dataPoints) {
+        const attrs = extractAttributes(dp.attributes);
+        
+        const histogramData = {
+            count: parseInt(dp.count || 0, 10),
+            sum: dp.sum || 0,
+            min: dp.min,
+            max: dp.max,
+            bucketCounts: dp.bucketCounts?.map(c => parseInt(c, 10)) || [],
+            explicitBounds: dp.explicitBounds || []
+        };
+        
+        results.push({
+            id: uuidv4(),
+            timestamp: nanoToMs(dp.timeUnixNano),
+            name: metric.name,
+            description: metric.description || null,
+            unit: metric.unit || null,
+            metric_type: 'histogram',
+            value_int: histogramData.count,
+            value_double: histogramData.sum,
+            histogram_data: histogramData,
+            service_name: resourceAttrs['service.name'] || 'unknown',
+            scope_name: scopeInfo.name || null,
+            attributes: attrs,
+            resource_attributes: resourceAttrs
+        });
+    }
+    
+    return results;
+}
+
+/**
+ * Process a single metric based on its type
+ */
+function processMetric(metric, resourceAttrs, scopeInfo) {
+    if (metric.sum) {
+        return processSum(metric, resourceAttrs, scopeInfo);
+    }
+    if (metric.gauge) {
+        return processGauge(metric, resourceAttrs, scopeInfo);
+    }
+    if (metric.histogram) {
+        return processHistogram(metric, resourceAttrs, scopeInfo);
+    }
+    // Summary and other types - store as gauge with raw data
+    return [];
+}
+
+/**
+ * Process OTLP/HTTP JSON metrics request
+ * 
+ * Expected format (OTLP/HTTP JSON):
+ * {
+ *   "resourceMetrics": [
+ *     {
+ *       "resource": { "attributes": [...] },
+ *       "scopeMetrics": [
+ *         {
+ *           "scope": { "name": "...", "version": "..." },
+ *           "metrics": [
+ *             {
+ *               "name": "metric.name",
+ *               "description": "...",
+ *               "unit": "...",
+ *               "sum": { "dataPoints": [...] }
+ *               // or "gauge": { "dataPoints": [...] }
+ *               // or "histogram": { "dataPoints": [...] }
+ *             }
+ *           ]
+ *         }
+ *       ]
+ *     }
+ *   ]
+ * }
+ */
+function processOtlpMetrics(body) {
+    const results = {
+        accepted: 0,
+        rejected: 0,
+        errors: []
+    };
+    
+    if (!body || !body.resourceMetrics) {
+        return results;
+    }
+    
+    for (const resourceMetric of body.resourceMetrics) {
+        const resourceAttrs = extractAttributes(resourceMetric.resource?.attributes);
+        
+        for (const scopeMetric of (resourceMetric.scopeMetrics || [])) {
+            const scopeInfo = scopeMetric.scope || {};
+            
+            for (const metric of (scopeMetric.metrics || [])) {
+                try {
+                    const dataPoints = processMetric(metric, resourceAttrs, scopeInfo);
+                    
+                    for (const dp of dataPoints) {
+                        db.insertMetric(dp);
+                        results.accepted++;
+                    }
+                } catch (err) {
+                    results.rejected++;
+                    results.errors.push(err.message);
+                }
+            }
+        }
+    }
+    
+    return results;
+}
+
+/**
+ * Express middleware for OTLP metrics endpoint
+ */
+function createMetricsHandler() {
+    return (req, res) => {
+        const contentType = req.headers['content-type'] || '';
+        
+        if (!contentType.includes('application/json')) {
+            return res.status(415).json({
+                error: 'Unsupported Media Type',
+                message: 'Only application/json is supported. Use OTLP/HTTP JSON format.'
+            });
+        }
+        
+        try {
+            const results = processOtlpMetrics(req.body);
+            
+            res.status(200).json({
+                partialSuccess: results.rejected > 0 ? {
+                    rejectedDataPoints: results.rejected,
+                    errorMessage: results.errors.slice(0, 5).join('; ')
+                } : undefined
+            });
+        } catch (err) {
+            res.status(500).json({
+                error: 'Internal Server Error',
+                message: err.message
+            });
+        }
+    };
+}
+
+module.exports = {
+    processOtlpMetrics,
+    createMetricsHandler,
+    extractAttributes,
+    extractValue,
+    processSum,
+    processGauge,
+    processHistogram,
+    nanoToMs
+};