llmflow 0.3.1

package/otlp.js

@@ -0,0 +1,398 @@
+/**
+ * OTLP (OpenTelemetry Protocol) HTTP endpoint for LLMFlow
+ * 
+ * Accepts OTLP/HTTP JSON traces and transforms them to LLMFlow span format.
+ * This allows users with existing OpenTelemetry/OpenLLMetry instrumentation
+ * to export traces directly to LLMFlow.
+ * 
+ * Supports:
+ * - OTLP/HTTP JSON format (Content-Type: application/json)
+ * - gen_ai.* semantic conventions (OpenLLMetry)
+ * - Standard OTEL span attributes
+ */
+
+const db = require('./db');
+const { calculateCost } = require('./pricing');
+
+/**
+ * Map gen_ai.system values to span types
+ */
+const PROVIDER_TO_SPAN_TYPE = {
+    'openai': 'llm',
+    'anthropic': 'llm',
+    'cohere': 'llm',
+    'bedrock': 'llm',
+    'azure': 'llm',
+    'google': 'llm',
+    'ollama': 'llm',
+    'groq': 'llm',
+    'together': 'llm',
+    'mistral': 'llm',
+    'replicate': 'llm',
+};
+
+/**
+ * Map traceloop.span.kind to LLMFlow span types
+ */
+const TRACELOOP_KIND_TO_SPAN_TYPE = {
+    'workflow': 'trace',
+    'task': 'chain',
+    'agent': 'agent',
+    'tool': 'tool',
+};
+
+/**
+ * Convert hex string to standard UUID format if needed
+ */
+function normalizeId(hexId) {
+    if (!hexId) return null;
+    // Remove any existing dashes and lowercase
+    const clean = hexId.replace(/-/g, '').toLowerCase();
+    // If it's already short enough, return as-is
+    if (clean.length <= 32) return clean;
+    return clean;
+}
+
+/**
+ * Extract attributes from OTLP KeyValue array format
+ * OTLP attributes are: [{ key: "foo", value: { stringValue: "bar" } }, ...]
+ */
+function extractAttributes(attrs) {
+    if (!attrs || !Array.isArray(attrs)) return {};
+    
+    const result = {};
+    for (const attr of attrs) {
+        const key = attr.key;
+        const val = attr.value;
+        if (!val) continue;
+        
+        // OTLP value types: stringValue, intValue, doubleValue, boolValue, arrayValue, kvlistValue
+        if (val.stringValue !== undefined) result[key] = val.stringValue;
+        else if (val.intValue !== undefined) result[key] = parseInt(val.intValue, 10);
+        else if (val.doubleValue !== undefined) result[key] = val.doubleValue;
+        else if (val.boolValue !== undefined) result[key] = val.boolValue;
+        else if (val.arrayValue?.values) {
+            result[key] = val.arrayValue.values.map(v => 
+                v.stringValue ?? v.intValue ?? v.doubleValue ?? v.boolValue ?? null
+            );
+        }
+    }
+    return result;
+}
+
+/**
+ * Determine span type from OTEL attributes
+ */
+function determineSpanType(attrs) {
+    // Check for traceloop span kind first (LangChain, etc.)
+    const traceloopKind = attrs['traceloop.span.kind'];
+    if (traceloopKind && TRACELOOP_KIND_TO_SPAN_TYPE[traceloopKind]) {
+        return TRACELOOP_KIND_TO_SPAN_TYPE[traceloopKind];
+    }
+    
+    // Check for gen_ai.system (OpenLLMetry)
+    const genAiSystem = attrs['gen_ai.system'];
+    if (genAiSystem) {
+        return PROVIDER_TO_SPAN_TYPE[genAiSystem.toLowerCase()] || 'llm';
+    }
+    
+    // Check for llm.request.type
+    const llmRequestType = attrs['llm.request.type'];
+    if (llmRequestType) {
+        return 'llm';
+    }
+    
+    // Check for db.system (vector DBs)
+    const dbSystem = attrs['db.system'];
+    if (dbSystem) {
+        const vectorDbs = ['pinecone', 'chroma', 'weaviate', 'qdrant', 'milvus', 'pgvector'];
+        if (vectorDbs.some(v => dbSystem.toLowerCase().includes(v))) {
+            return 'retrieval';
+        }
+    }
+    
+    // Check span name patterns
+    const spanName = attrs._spanName || '';
+    if (spanName.includes('embed')) return 'embedding';
+    if (spanName.includes('retriev') || spanName.includes('search')) return 'retrieval';
+    if (spanName.includes('agent')) return 'agent';
+    if (spanName.includes('tool') || spanName.includes('function')) return 'tool';
+    if (spanName.includes('chain')) return 'chain';
+    
+    return 'custom';
+}
+
+/**
+ * Extract model name from attributes
+ */
+function extractModel(attrs) {
+    return attrs['gen_ai.request.model'] 
+        || attrs['gen_ai.response.model']
+        || attrs['llm.model']
+        || attrs['model']
+        || null;
+}
+
+/**
+ * Extract token usage from attributes
+ */
+function extractTokens(attrs) {
+    return {
+        prompt: attrs['gen_ai.usage.prompt_tokens'] 
+            || attrs['llm.usage.prompt_tokens']
+            || attrs['llm.token_count.prompt']
+            || 0,
+        completion: attrs['gen_ai.usage.completion_tokens'] 
+            || attrs['llm.usage.completion_tokens']
+            || attrs['llm.token_count.completion']
+            || 0,
+        total: attrs['gen_ai.usage.total_tokens']
+            || attrs['llm.usage.total_tokens']
+            || attrs['llm.token_count.total']
+            || 0
+    };
+}
+
+/**
+ * Extract input/output from attributes or events
+ */
+function extractIO(attrs, events) {
+    let input = null;
+    let output = null;
+    
+    // Try gen_ai.prompt / gen_ai.completion (OpenLLMetry)
+    if (attrs['gen_ai.prompt']) {
+        try {
+            input = typeof attrs['gen_ai.prompt'] === 'string' 
+                ? JSON.parse(attrs['gen_ai.prompt']) 
+                : attrs['gen_ai.prompt'];
+        } catch {
+            input = { prompt: attrs['gen_ai.prompt'] };
+        }
+    }
+    
+    if (attrs['gen_ai.completion']) {
+        try {
+            output = typeof attrs['gen_ai.completion'] === 'string' 
+                ? JSON.parse(attrs['gen_ai.completion']) 
+                : attrs['gen_ai.completion'];
+        } catch {
+            output = { completion: attrs['gen_ai.completion'] };
+        }
+    }
+    
+    // Check events for prompt/completion data
+    if (events && events.length > 0) {
+        for (const event of events) {
+            const eventAttrs = extractAttributes(event.attributes);
+            if (event.name === 'gen_ai.content.prompt' || event.name?.includes('prompt')) {
+                input = eventAttrs;
+            }
+            if (event.name === 'gen_ai.content.completion' || event.name?.includes('completion')) {
+                output = eventAttrs;
+            }
+        }
+    }
+    
+    return { input, output };
+}
+
+/**
+ * Convert nanoseconds timestamp to milliseconds
+ */
+function nanoToMs(nanoStr) {
+    if (!nanoStr) return Date.now();
+    const nano = BigInt(nanoStr);
+    return Number(nano / BigInt(1000000));
+}
+
+/**
+ * Transform a single OTLP span to LLMFlow format
+ */
+function transformSpan(span, resourceAttrs, scopeAttrs) {
+    const attrs = {
+        ...extractAttributes(span.attributes),
+        _spanName: span.name,
+    };
+    
+    const traceId = normalizeId(span.traceId);
+    const spanId = normalizeId(span.spanId);
+    const parentId = span.parentSpanId ? normalizeId(span.parentSpanId) : null;
+    
+    const startTimeMs = nanoToMs(span.startTimeUnixNano);
+    const endTimeMs = nanoToMs(span.endTimeUnixNano);
+    const durationMs = endTimeMs - startTimeMs;
+    
+    const spanType = determineSpanType(attrs);
+    const model = extractModel(attrs);
+    const tokens = extractTokens(attrs);
+    const { input, output } = extractIO(attrs, span.events);
+    
+    // Calculate cost if we have model and tokens
+    const estimatedCost = model && (tokens.prompt || tokens.completion)
+        ? calculateCost(model, tokens.prompt, tokens.completion)
+        : 0;
+    
+    // Determine status
+    let status = 200;
+    if (span.status) {
+        // OTEL status: 0=UNSET, 1=OK, 2=ERROR
+        if (span.status.code === 2) {
+            status = 500;
+        }
+    }
+    
+    // Extract provider
+    const provider = attrs['gen_ai.system'] 
+        || attrs['gen_ai.provider.name']
+        || attrs['llm.vendor']
+        || resourceAttrs['service.name']
+        || null;
+    
+    // Extract service name
+    const serviceName = resourceAttrs['service.name'] 
+        || scopeAttrs?.name
+        || 'otel';
+    
+    return {
+        id: spanId,
+        timestamp: startTimeMs,
+        duration_ms: durationMs,
+        provider,
+        model,
+        prompt_tokens: tokens.prompt,
+        completion_tokens: tokens.completion,
+        total_tokens: tokens.total || tokens.prompt + tokens.completion,
+        estimated_cost: estimatedCost,
+        status,
+        error: span.status?.message || attrs['error.message'] || null,
+        request_method: null,
+        request_path: null,
+        request_headers: {},
+        request_body: {},
+        response_status: status,
+        response_headers: {},
+        response_body: {},
+        tags: [],
+        trace_id: traceId,
+        parent_id: parentId,
+        span_type: spanType,
+        span_name: span.name || attrs['traceloop.entity.name'] || spanType,
+        input,
+        output,
+        attributes: {
+            ...attrs,
+            ...resourceAttrs,
+            otel_span_kind: span.kind,
+        },
+        service_name: serviceName
+    };
+}
+
+/**
+ * Process OTLP/HTTP JSON traces request
+ * 
+ * Expected format (OTLP/HTTP JSON):
+ * {
+ *   "resourceSpans": [
+ *     {
+ *       "resource": { "attributes": [...] },
+ *       "scopeSpans": [
+ *         {
+ *           "scope": { "name": "...", "version": "..." },
+ *           "spans": [
+ *             {
+ *               "traceId": "hex",
+ *               "spanId": "hex",
+ *               "parentSpanId": "hex",
+ *               "name": "span name",
+ *               "kind": 1,
+ *               "startTimeUnixNano": "...",
+ *               "endTimeUnixNano": "...",
+ *               "attributes": [...],
+ *               "events": [...],
+ *               "status": { "code": 0 }
+ *             }
+ *           ]
+ *         }
+ *       ]
+ *     }
+ *   ]
+ * }
+ */
+function processOtlpTraces(body) {
+    const results = {
+        accepted: 0,
+        rejected: 0,
+        errors: []
+    };
+    
+    if (!body || !body.resourceSpans) {
+        return results;
+    }
+    
+    for (const resourceSpan of body.resourceSpans) {
+        const resourceAttrs = extractAttributes(resourceSpan.resource?.attributes);
+        
+        for (const scopeSpan of (resourceSpan.scopeSpans || [])) {
+            const scopeAttrs = scopeSpan.scope || {};
+            
+            for (const span of (scopeSpan.spans || [])) {
+                try {
+                    const llmflowSpan = transformSpan(span, resourceAttrs, scopeAttrs);
+                    db.insertTrace(llmflowSpan);
+                    results.accepted++;
+                } catch (err) {
+                    results.rejected++;
+                    results.errors.push(err.message);
+                }
+            }
+        }
+    }
+    
+    return results;
+}
+
+/**
+ * Express middleware for OTLP endpoint
+ */
+function createOtlpHandler() {
+    return (req, res) => {
+        const contentType = req.headers['content-type'] || '';
+        
+        // Only support JSON for now
+        if (!contentType.includes('application/json')) {
+            return res.status(415).json({
+                error: 'Unsupported Media Type',
+                message: 'Only application/json is supported. Use OTLP/HTTP JSON format.'
+            });
+        }
+        
+        try {
+            const results = processOtlpTraces(req.body);
+            
+            // OTLP response format
+            res.status(200).json({
+                partialSuccess: results.rejected > 0 ? {
+                    rejectedSpans: results.rejected,
+                    errorMessage: results.errors.slice(0, 5).join('; ')
+                } : undefined
+            });
+        } catch (err) {
+            res.status(500).json({
+                error: 'Internal Server Error',
+                message: err.message
+            });
+        }
+    };
+}
+
+module.exports = {
+    processOtlpTraces,
+    createOtlpHandler,
+    transformSpan,
+    extractAttributes,
+    determineSpanType,
+    extractModel,
+    extractTokens
+};

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "llmflow",
+  "version": "0.3.1",
+  "description": "See what your LLM calls cost. One command. No signup.",
+  "main": "server.js",
+  "bin": {
+    "llmflow": "./bin/llmflow.js"
+  },
+  "scripts": {
+    "start": "node server.js",
+    "start:verbose": "VERBOSE=1 node server.js",
+    "demo": "node test/demo.js",
+    "demo:many": "node test/demo.js --count=20",
+    "test": "node test/run-tests.js",
+    "test:otlp": "node test/run-tests.js otlp-e2e.js",
+    "test:ws": "node test/run-tests.js websocket-e2e.js",
+    "test:logs": "node test/run-tests.js otlp-logs-e2e.js",
+    "test:metrics": "node test/run-tests.js otlp-metrics-e2e.js",
+    "test:providers": "node test/run-tests.js providers.js",
+    "test:providers-e2e": "node test/run-tests.js providers-e2e.js"
+  },
+  "dependencies": {
+    "better-sqlite3": "^11.0.0",
+    "express": "^4.18.0",
+    "uuid": "^9.0.0",
+    "ws": "^8.18.3"
+  },
+  "files": [
+    "bin/",
+    "providers/",
+    "public/",
+    "server.js",
+    "db.js",
+    "pricing.js",
+    "pricing.fallback.json",
+    "logger.js",
+    "otlp.js",
+    "otlp-logs.js",
+    "otlp-metrics.js",
+    "otlp-export.js"
+  ],
+  "keywords": [
+    "llm",
+    "observability",
+    "tracing",
+    "openai",
+    "anthropic",
+    "langchain",
+    "opentelemetry",
+    "cost-tracking",
+    "ai"
+  ],
+  "author": "Helge Sverre",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/HelgeSverre/llmflow"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}

package/pricing.fallback.json

@@ -0,0 +1,58 @@
+{
+  "gpt-4o": {
+    "input_cost_per_token": 0.0000025,
+    "output_cost_per_token": 0.00001
+  },
+  "gpt-4o-mini": {
+    "input_cost_per_token": 0.00000015,
+    "output_cost_per_token": 0.0000006
+  },
+  "gpt-4-turbo": {
+    "input_cost_per_token": 0.00001,
+    "output_cost_per_token": 0.00003
+  },
+  "gpt-4": {
+    "input_cost_per_token": 0.00003,
+    "output_cost_per_token": 0.00006
+  },
+  "gpt-3.5-turbo": {
+    "input_cost_per_token": 0.0000005,
+    "output_cost_per_token": 0.0000015
+  },
+  "gpt-3.5-turbo-16k": {
+    "input_cost_per_token": 0.000003,
+    "output_cost_per_token": 0.000004
+  },
+  "claude-3-5-sonnet-20241022": {
+    "input_cost_per_token": 0.000003,
+    "output_cost_per_token": 0.000015
+  },
+  "claude-3-5-haiku-20241022": {
+    "input_cost_per_token": 0.0000008,
+    "output_cost_per_token": 0.000004
+  },
+  "claude-3-opus-20240229": {
+    "input_cost_per_token": 0.000015,
+    "output_cost_per_token": 0.000075
+  },
+  "claude-3-sonnet-20240229": {
+    "input_cost_per_token": 0.000003,
+    "output_cost_per_token": 0.000015
+  },
+  "claude-3-haiku-20240307": {
+    "input_cost_per_token": 0.00000025,
+    "output_cost_per_token": 0.00000125
+  },
+  "gemini-1.5-pro": {
+    "input_cost_per_token": 0.00000125,
+    "output_cost_per_token": 0.000005
+  },
+  "gemini-1.5-flash": {
+    "input_cost_per_token": 0.000000075,
+    "output_cost_per_token": 0.0000003
+  },
+  "gemini-2.0-flash-exp": {
+    "input_cost_per_token": 0,
+    "output_cost_per_token": 0
+  }
+}

package/pricing.js

@@ -0,0 +1,154 @@
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+
+const DEFAULT_PRICING_URL = 'https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json';
+const PRICING_URL = process.env.PRICING_URL || DEFAULT_PRICING_URL;
+const REFRESH_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
+const VERBOSE = process.env.VERBOSE === '1' || process.argv.includes('--verbose');
+
+const fallbackPricingPath = path.join(__dirname, 'pricing.fallback.json');
+
+let pricingData = {};
+let lastFetchTime = 0;
+
+function loadFallbackPricing() {
+    try {
+        if (fs.existsSync(fallbackPricingPath)) {
+            pricingData = JSON.parse(fs.readFileSync(fallbackPricingPath, 'utf8'));
+        }
+    } catch (err) {
+        // Silent fail, will use empty pricing
+    }
+}
+
+function fetchJson(url) {
+    return new Promise((resolve, reject) => {
+        const request = https.get(url, { timeout: 10000 }, (res) => {
+            if (res.statusCode !== 200) {
+                reject(new Error(`HTTP ${res.statusCode}`));
+                return;
+            }
+            
+            let data = '';
+            res.on('data', chunk => data += chunk);
+            res.on('end', () => {
+                try {
+                    resolve(JSON.parse(data));
+                } catch (err) {
+                    reject(err);
+                }
+            });
+        });
+        
+        request.on('error', reject);
+        request.on('timeout', () => {
+            request.destroy();
+            reject(new Error('Request timeout'));
+        });
+    });
+}
+
+async function loadPricing() {
+    try {
+        const json = await fetchJson(PRICING_URL);
+        pricingData = json;
+        lastFetchTime = Date.now();
+        if (VERBOSE) {
+            console.log(`\x1b[2m[pricing] Loaded ${Object.keys(pricingData).length} models\x1b[0m`);
+        }
+    } catch (err) {
+        if (Object.keys(pricingData).length === 0) {
+            loadFallbackPricing();
+        }
+    }
+}
+
+function normalizeModelName(model) {
+    if (!model) return null;
+    
+    let normalized = model.toLowerCase().trim();
+    
+    // Remove common prefixes
+    const prefixes = ['openai/', 'anthropic/', 'google/', 'azure/', 'together/'];
+    for (const prefix of prefixes) {
+        if (normalized.startsWith(prefix)) {
+            normalized = normalized.slice(prefix.length);
+            break;
+        }
+    }
+    
+    return normalized;
+}
+
+function findModelPricing(modelName) {
+    if (!modelName || !pricingData) return null;
+    
+    const normalized = normalizeModelName(modelName);
+    if (!normalized) return null;
+    
+    // Try exact match first
+    if (pricingData[normalized]) {
+        return pricingData[normalized];
+    }
+    
+    // Try with provider prefixes
+    const providers = ['openai/', 'azure/', ''];
+    for (const prefix of providers) {
+        const key = prefix + normalized;
+        if (pricingData[key]) {
+            return pricingData[key];
+        }
+    }
+    
+    // Try partial match for versioned models (gpt-4-0125-preview -> gpt-4)
+    const baseName = normalized.split('-').slice(0, 2).join('-');
+    for (const key of Object.keys(pricingData)) {
+        if (key.startsWith(baseName) || key.includes(normalized)) {
+            return pricingData[key];
+        }
+    }
+    
+    return null;
+}
+
+function calculateCost(model, promptTokens, completionTokens) {
+    const pricing = findModelPricing(model);
+    
+    if (!pricing) {
+        // Fallback to default pricing for unknown models
+        return ((promptTokens * 0.001) + (completionTokens * 0.002)) / 1000;
+    }
+    
+    // LiteLLM format: input_cost_per_token, output_cost_per_token (per token, not per 1k)
+    const inputCost = pricing.input_cost_per_token || 0;
+    const outputCost = pricing.output_cost_per_token || pricing.input_cost_per_token || 0;
+    
+    return (promptTokens * inputCost) + (completionTokens * outputCost);
+}
+
+function getPricingInfo(model) {
+    return findModelPricing(model);
+}
+
+function getPricingStats() {
+    return {
+        modelCount: Object.keys(pricingData).length,
+        lastFetchTime,
+        source: PRICING_URL
+    };
+}
+
+// Initialize pricing
+loadFallbackPricing();
+loadPricing();
+
+// Refresh pricing periodically
+setInterval(loadPricing, REFRESH_INTERVAL_MS);
+
+module.exports = {
+    calculateCost,
+    getPricingInfo,
+    getPricingStats,
+    loadPricing
+};