observability-toolkit 1.4.0 → 1.5.0

package/dist/backends/local-jsonl.js

@@ -5,8 +5,11 @@
  * span or log record, not the batched OpenTelemetry export format.
  */
 import { join } from 'path';
+import { convertToOTLPTraces, convertToOTLPLogs, convertToOTLPMetrics, } from '../lib/otlp-export.js';
 import { TELEMETRY_DIR, getTelemetryDirectories, getSpanKind, getStatusCodeName } from '../lib/constants.js';
 import { listFiles, streamJsonl, parseDateFromFilename, getDateString, paginateResults, hasReachedLimit, } from '../lib/file-utils.js';
+import { QueryCache, makeCacheKey } from '../lib/cache.js';
+import { getIndexPath, readIndex, isIndexStale, queryIndex, readLinesByNumber, } from '../lib/indexer.js';
 import { existsSync } from 'fs';
 /**
  * Insert item into a sorted array, maintaining sort order and max size.
@@ -36,6 +39,107 @@ function insertSortedBounded(arr, item, maxSize, compareFn) {
         arr.pop();
     }
 }
+/**
+ * Apply numeric filter conditions to an attributes object.
+ * Returns true if all conditions pass, false otherwise.
+ */
+function applyNumericFilters(attributes, filters) {
+    for (const filter of filters) {
+        const attrValue = attributes?.[filter.attribute];
+        // Skip if attribute doesn't exist or isn't a number
+        if (typeof attrValue !== 'number') {
+            return false;
+        }
+        // Apply the comparison
+        switch (filter.operator) {
+            case 'gt':
+                if (!(attrValue > filter.value))
+                    return false;
+                break;
+            case 'gte':
+                if (!(attrValue >= filter.value))
+                    return false;
+                break;
+            case 'lt':
+                if (!(attrValue < filter.value))
+                    return false;
+                break;
+            case 'lte':
+                if (!(attrValue <= filter.value))
+                    return false;
+                break;
+            case 'eq':
+                if (!(attrValue === filter.value))
+                    return false;
+                break;
+        }
+    }
+    return true;
+}
+/**
+ * Parse time bucket string to milliseconds
+ * Supports: '1m', '5m', '1h', '1d', etc.
+ */
+function parseTimeBucket(bucket) {
+    const match = bucket.match(/^(\d+)(m|h|d)$/);
+    if (!match)
+        return null;
+    const value = parseInt(match[1], 10);
+    const unit = match[2];
+    switch (unit) {
+        case 'm':
+            return value * 60 * 1000; // minutes to ms
+        case 'h':
+            return value * 60 * 60 * 1000; // hours to ms
+        case 'd':
+            return value * 24 * 60 * 60 * 1000; // days to ms
+        default:
+            return null;
+    }
+}
+/**
+ * Floor timestamp to bucket boundary
+ */
+function floorToBucket(timestamp, bucketMs) {
+    const ts = new Date(timestamp).getTime();
+    return Math.floor(ts / bucketMs) * bucketMs;
+}
+/**
+ * Extract a field from an object using dot notation path.
+ * Returns undefined if the path doesn't exist or the object is not traversable.
+ */
+function extractField(obj, path) {
+    const parts = path.split('.');
+    let current = obj;
+    for (const part of parts) {
+        if (current == null || typeof current !== 'object')
+            return undefined;
+        current = current[part];
+    }
+    return current;
+}
+/**
+ * Extract multiple fields from a JSON string body.
+ * Returns undefined if body is not valid JSON or doesn't start with '{'.
+ */
+function extractFieldsFromBody(body, fields) {
+    if (!body.startsWith('{'))
+        return undefined;
+    try {
+        const parsed = JSON.parse(body);
+        const extracted = {};
+        for (const field of fields) {
+            const value = extractField(parsed, field);
+            if (value !== undefined) {
+                extracted[field] = value;
+            }
+        }
+        return Object.keys(extracted).length > 0 ? extracted : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
 /**
  * OTel-compliant severity number mapping
  * https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
@@ -77,6 +181,21 @@ function normalizeSpan(raw) {
     if (raw.resource?.serviceVersion) {
         attributes['service.version'] = raw.resource.serviceVersion;
     }
+    // Normalize span links
+    let links;
+    if (raw.links && raw.links.length > 0) {
+        links = raw.links
+            .filter(link => link.context?.traceId && link.context?.spanId)
+            .map(link => ({
+            traceId: link.context.traceId,
+            spanId: link.context.spanId,
+            attributes: link.attributes,
+        }));
+        // Only include if we have valid links
+        if (links.length === 0) {
+            links = undefined;
+        }
+    }
     return {
         traceId: raw.traceId,
         spanId: raw.spanId,
@@ -89,12 +208,14 @@ function normalizeSpan(raw) {
         status: raw.status?.code !== undefined ? { code: raw.status.code, message: raw.status.message } : undefined,
         statusCode: getStatusCodeName(raw.status?.code),
         attributes,
+        links,
+        instrumentationScope: raw.instrumentationScope,
     };
 }
 /**
  * Convert flat log to normalized LogRecord
  */
-function normalizeLog(raw) {
+function normalizeLog(raw, extractFields) {
     const attributes = { ...raw.attributes };
     if (raw.resource?.serviceName) {
         attributes['service.name'] = raw.resource.serviceName;
@@ -110,16 +231,63 @@ function normalizeLog(raw) {
     }
     const severity = raw.severityText || raw.severity || 'INFO';
     const severityNumber = SEVERITY_MAP[severity.toUpperCase()];
+    const body = raw.body || '';
+    // Extract fields from JSON body if requested
+    let extractedFields;
+    if (extractFields && extractFields.length > 0) {
+        extractedFields = extractFieldsFromBody(body, extractFields);
+    }
     return {
         timestamp,
         severity,
         severityNumber,
-        body: raw.body || '',
+        body,
         traceId: raw.traceId,
         spanId: raw.spanId,
         attributes,
+        instrumentationScope: raw.instrumentationScope,
+        extractedFields,
     };
 }
+/**
+ * Convert OTel aggregation temporality to string representation
+ * OTel spec: 0=UNSPECIFIED, 1=DELTA, 2=CUMULATIVE
+ * https://opentelemetry.io/docs/specs/otel/metrics/data-model/#temporality
+ */
+function normalizeAggregationTemporality(value) {
+    if (value === undefined)
+        return undefined;
+    // Handle string values
+    if (typeof value === 'string') {
+        const upper = value.toUpperCase();
+        if (upper === 'DELTA' || upper === 'CUMULATIVE' || upper === 'UNSPECIFIED') {
+            return upper;
+        }
+        return 'UNSPECIFIED';
+    }
+    // Handle numeric values (OTel spec)
+    switch (value) {
+        case 1:
+            return 'DELTA';
+        case 2:
+            return 'CUMULATIVE';
+        case 0:
+        default:
+            return 'UNSPECIFIED';
+    }
+}
+/**
+ * Normalize a flat exemplar timestamp to ISO string
+ */
+function normalizeExemplarTimestamp(ts, metricTimestamp) {
+    if (!ts)
+        return metricTimestamp;
+    if (typeof ts === 'string')
+        return ts;
+    // Convert [seconds, nanoseconds] to ISO string
+    const ms = ts[0] * 1000 + Math.floor(ts[1] / 1_000_000);
+    return new Date(ms).toISOString();
+}
 /**
  * Convert flat metric to normalized MetricDataPoint
  */
@@ -128,12 +296,40 @@ function normalizeMetric(raw) {
     if (raw.resource?.serviceName) {
         attributes['service.name'] = raw.resource.serviceName;
     }
+    // Build histogram data if present
+    let histogram;
+    if (raw.histogram && raw.type === 'histogram') {
+        histogram = {
+            buckets: raw.histogram.buckets.map(b => ({
+                le: b.le,
+                count: b.count,
+            })),
+            sum: raw.histogram.sum,
+            count: raw.histogram.count,
+        };
+    }
+    // Normalize exemplars if present
+    let exemplars;
+    if (raw.exemplars && raw.exemplars.length > 0) {
+        exemplars = raw.exemplars.map(e => ({
+            timestamp: normalizeExemplarTimestamp(e.timestamp, raw.timestamp),
+            value: e.value,
+            traceId: e.traceId,
+            spanId: e.spanId,
+            attributes: e.attributes,
+        }));
+    }
+    // Normalize aggregation temporality
+    const aggregationTemporality = normalizeAggregationTemporality(raw.aggregationTemporality);
     return {
         timestamp: raw.timestamp,
         name: raw.name,
         value: raw.value,
         unit: raw.unit,
         attributes,
+        histogram,
+        exemplars,
+        aggregationTemporality,
     };
 }
 /**
@@ -163,163 +359,325 @@ function getFilesInRange(dir, pattern, startDate, endDate) {
 export class LocalJsonlBackend {
     name = 'local-jsonl';
     telemetryDir;
-    constructor(telemetryDir) {
+    traceCache = new QueryCache();
+    logCache = new QueryCache();
+    metricCache = new QueryCache();
+    llmEventCache = new QueryCache();
+    useIndexes;
+    constructor(telemetryDir, useIndexes = true) {
         this.telemetryDir = telemetryDir || TELEMETRY_DIR;
+        this.useIndexes = useIndexes;
+    }
+    /**
+     * Clear all query caches
+     */
+    clearCache() {
+        this.traceCache.clear();
+        this.logCache.clear();
+        this.metricCache.clear();
+        this.llmEventCache.clear();
+    }
+    /**
+     * Try to use an index for a file, returning matching line numbers or null if full scan needed
+     */
+    tryUseIndex(file, _type, indexOptions) {
+        if (!this.useIndexes)
+            return null;
+        const idxPath = getIndexPath(file);
+        const index = readIndex(idxPath);
+        if (!index)
+            return null;
+        if (isIndexStale(index, file))
+            return null;
+        return queryIndex(index, indexOptions);
     }
     async queryTraces(options) {
+        // Check cache first
+        const cacheKey = makeCacheKey('traces', options);
+        const cached = this.traceCache.get(cacheKey);
+        if (cached)
+            return cached;
         const files = getFilesInRange(this.telemetryDir, /traces-\d{4}-\d{2}-\d{2}\.jsonl$/, options.startDate, options.endDate);
         const results = [];
         const limit = options.limit || 100;
         const offset = options.offset || 0;
+        // Compile regex once outside the loop, handling invalid patterns gracefully
+        let spanNameRegex = null;
+        if (options.spanNameRegex) {
+            try {
+                spanNameRegex = new RegExp(options.spanNameRegex);
+            }
+            catch {
+                console.warn(`Invalid spanNameRegex pattern: ${options.spanNameRegex}`);
+            }
+        }
+        // Build index query options for indexable filters
+        const indexOptions = {
+            traceId: options.traceId,
+            spanName: options.spanName,
+            serviceName: options.serviceName,
+        };
+        // Helper to apply non-indexable filters to a span
+        const applyFilters = (span) => {
+            // Regex filter (not indexable)
+            if (spanNameRegex && !spanNameRegex.test(span.name))
+                return false;
+            if (options.excludeSpanName && span.name.includes(options.excludeSpanName))
+                return false;
+            if (options.minDurationMs && (span.durationMs || 0) < options.minDurationMs)
+                return false;
+            if (options.maxDurationMs && (span.durationMs || Infinity) > options.maxDurationMs)
+                return false;
+            // Apply attribute filter
+            if (options.attributeFilter) {
+                for (const [key, value] of Object.entries(options.attributeFilter)) {
+                    if (span.attributes?.[key] !== value)
+                        return false;
+                }
+            }
+            // Apply attributeExists filter
+            if (options.attributeExists) {
+                for (const key of options.attributeExists) {
+                    if (span.attributes?.[key] === undefined)
+                        return false;
+                }
+            }
+            // Apply attributeNotExists filter
+            if (options.attributeNotExists) {
+                for (const key of options.attributeNotExists) {
+                    if (span.attributes?.[key] !== undefined)
+                        return false;
+                }
+            }
+            // Apply numeric filter conditions
+            if (options.numericFilter && options.numericFilter.length > 0) {
+                if (!applyNumericFilters(span.attributes, options.numericFilter))
+                    return false;
+            }
+            return true;
+        };
         for (const file of files) {
-            // Stream flat span records (one per line) to avoid loading entire file into memory
-            for await (const raw of streamJsonl(file)) {
-                const span = normalizeSpan(raw);
-                if (!span)
-                    continue;
-                // Apply filters
-                if (options.traceId && span.traceId !== options.traceId)
-                    continue;
-                if (options.spanName && !span.name.includes(options.spanName))
-                    continue;
-                if (options.excludeSpanName && span.name.includes(options.excludeSpanName))
-                    continue;
-                if (options.minDurationMs && (span.durationMs || 0) < options.minDurationMs)
-                    continue;
-                if (options.maxDurationMs && (span.durationMs || Infinity) > options.maxDurationMs)
-                    continue;
-                if (options.serviceName) {
-                    const svc = span.attributes?.['service.name'];
-                    if (svc !== options.serviceName)
+            // Try to use index for pre-filtering
+            const matchingLines = this.tryUseIndex(file, 'traces', indexOptions);
+            if (matchingLines !== null) {
+                // Use indexed query - read only matching lines
+                const rawRecords = await readLinesByNumber(file, matchingLines);
+                for (const raw of rawRecords) {
+                    const span = normalizeSpan(raw);
+                    if (!span)
                         continue;
-                }
-                // Apply attribute filter
-                if (options.attributeFilter) {
-                    let matches = true;
-                    for (const [key, value] of Object.entries(options.attributeFilter)) {
-                        if (span.attributes?.[key] !== value) {
-                            matches = false;
-                            break;
-                        }
-                    }
-                    if (!matches)
+                    if (!applyFilters(span))
                         continue;
-                }
-                // Apply attributeExists filter - all specified attributes must exist
-                if (options.attributeExists) {
-                    let allExist = true;
-                    for (const key of options.attributeExists) {
-                        if (span.attributes?.[key] === undefined) {
-                            allExist = false;
-                            break;
-                        }
+                    results.push(span);
+                    if (hasReachedLimit(results.length, offset, limit)) {
+                        const paginated = paginateResults(results, offset, limit);
+                        this.traceCache.set(cacheKey, paginated);
+                        return paginated;
                     }
-                    if (!allExist)
-                        continue;
                 }
-                // Apply attributeNotExists filter - exclude if any specified attribute exists
-                if (options.attributeNotExists) {
-                    let anyExist = false;
-                    for (const key of options.attributeNotExists) {
-                        if (span.attributes?.[key] !== undefined) {
-                            anyExist = true;
-                            break;
-                        }
+            }
+            else {
+                // Fall back to full file scan
+                for await (const raw of streamJsonl(file)) {
+                    const span = normalizeSpan(raw);
+                    if (!span)
+                        continue;
+                    // Apply indexable filters (since no index was used)
+                    if (options.traceId && span.traceId !== options.traceId)
+                        continue;
+                    if (options.spanName && !span.name.includes(options.spanName))
+                        continue;
+                    if (options.serviceName) {
+                        const svc = span.attributes?.['service.name'];
+                        if (svc !== options.serviceName)
+                            continue;
                     }
-                    if (anyExist)
+                    if (!applyFilters(span))
                         continue;
-                }
-                results.push(span);
-                if (hasReachedLimit(results.length, offset, limit)) {
-                    return paginateResults(results, offset, limit);
+                    results.push(span);
+                    if (hasReachedLimit(results.length, offset, limit)) {
+                        const paginated = paginateResults(results, offset, limit);
+                        this.traceCache.set(cacheKey, paginated);
+                        return paginated;
+                    }
                 }
             }
         }
-        return paginateResults(results, offset, limit);
+        const paginated = paginateResults(results, offset, limit);
+        this.traceCache.set(cacheKey, paginated);
+        return paginated;
     }
     async queryLogs(options) {
+        // Check cache first
+        const cacheKey = makeCacheKey('logs', options);
+        const cached = this.logCache.get(cacheKey);
+        if (cached)
+            return cached;
         const files = getFilesInRange(this.telemetryDir, /logs-\d{4}-\d{2}-\d{2}\.jsonl$/, options.startDate, options.endDate);
         const results = [];
         const limit = options.limit || 100;
         const offset = options.offset || 0;
+        // Build index query options for indexable filters
+        const indexOptions = {
+            traceId: options.traceId,
+            severity: options.severity,
+        };
+        // Helper to apply non-indexable filters to a log
+        const applyFilters = (log) => {
+            if (options.search && !log.body.toLowerCase().includes(options.search.toLowerCase()))
+                return false;
+            // Apply boolean search with multiple terms
+            if (options.searchTerms && options.searchTerms.length > 0) {
+                const bodyLower = log.body.toLowerCase();
+                const operator = options.searchOperator || 'AND';
+                if (operator === 'AND') {
+                    const allMatch = options.searchTerms.every(term => bodyLower.includes(term.toLowerCase()));
+                    if (!allMatch)
+                        return false;
+                }
+                else {
+                    const anyMatch = options.searchTerms.some(term => bodyLower.includes(term.toLowerCase()));
+                    if (!anyMatch)
+                        return false;
+                }
+            }
+            if (options.excludeSearch && log.body.toLowerCase().includes(options.excludeSearch.toLowerCase()))
+                return false;
+            // Apply attributeExists filter
+            if (options.attributeExists) {
+                for (const key of options.attributeExists) {
+                    if (log.attributes?.[key] === undefined)
+                        return false;
+                }
+            }
+            // Apply attributeNotExists filter
+            if (options.attributeNotExists) {
+                for (const key of options.attributeNotExists) {
+                    if (log.attributes?.[key] !== undefined)
+                        return false;
+                }
+            }
+            // Apply numeric filter conditions
+            if (options.numericFilter && options.numericFilter.length > 0) {
+                if (!applyNumericFilters(log.attributes, options.numericFilter))
+                    return false;
+            }
+            return true;
+        };
         for (const file of files) {
-            // Stream flat log records (one per line) to avoid loading entire file into memory
-            for await (const raw of streamJsonl(file)) {
-                const log = normalizeLog(raw);
-                if (!log)
-                    continue;
-                // Apply filters
-                if (options.severity && log.severity.toUpperCase() !== options.severity.toUpperCase())
-                    continue;
-                if (options.traceId && log.traceId !== options.traceId)
-                    continue;
-                if (options.search && !log.body.toLowerCase().includes(options.search.toLowerCase()))
-                    continue;
-                if (options.excludeSearch && log.body.toLowerCase().includes(options.excludeSearch.toLowerCase()))
-                    continue;
-                // Apply attributeExists filter - all specified attributes must exist
-                if (options.attributeExists) {
-                    let allExist = true;
-                    for (const key of options.attributeExists) {
-                        if (log.attributes?.[key] === undefined) {
-                            allExist = false;
-                            break;
-                        }
-                    }
-                    if (!allExist)
+            // Try to use index for pre-filtering
+            const matchingLines = this.tryUseIndex(file, 'logs', indexOptions);
+            if (matchingLines !== null) {
+                // Use indexed query - read only matching lines
+                const rawRecords = await readLinesByNumber(file, matchingLines);
+                for (const raw of rawRecords) {
+                    const log = normalizeLog(raw, options.extractFields);
+                    if (!log)
                         continue;
-                }
-                // Apply attributeNotExists filter - exclude if any specified attribute exists
-                if (options.attributeNotExists) {
-                    let anyExist = false;
-                    for (const key of options.attributeNotExists) {
-                        if (log.attributes?.[key] !== undefined) {
-                            anyExist = true;
-                            break;
-                        }
-                    }
-                    if (anyExist)
+                    if (!applyFilters(log))
                         continue;
+                    results.push(log);
+                    if (hasReachedLimit(results.length, offset, limit)) {
+                        const paginated = paginateResults(results, offset, limit);
+                        this.logCache.set(cacheKey, paginated);
+                        return paginated;
+                    }
                 }
-                results.push(log);
-                if (hasReachedLimit(results.length, offset, limit)) {
-                    return paginateResults(results, offset, limit);
+            }
+            else {
+                // Fall back to full file scan
+                for await (const raw of streamJsonl(file)) {
+                    const log = normalizeLog(raw, options.extractFields);
+                    if (!log)
+                        continue;
+                    // Apply indexable filters (since no index was used)
+                    if (options.severity && log.severity.toUpperCase() !== options.severity.toUpperCase())
+                        continue;
+                    if (options.traceId && log.traceId !== options.traceId)
+                        continue;
+                    if (!applyFilters(log))
+                        continue;
+                    results.push(log);
+                    if (hasReachedLimit(results.length, offset, limit)) {
+                        const paginated = paginateResults(results, offset, limit);
+                        this.logCache.set(cacheKey, paginated);
+                        return paginated;
+                    }
                 }
             }
         }
-        return paginateResults(results, offset, limit);
+        const paginated = paginateResults(results, offset, limit);
+        this.logCache.set(cacheKey, paginated);
+        return paginated;
     }
     async queryMetrics(options) {
+        // Check cache first
+        const cacheKey = makeCacheKey('metrics', options);
+        const cached = this.metricCache.get(cacheKey);
+        if (cached)
+            return cached;
         const files = getFilesInRange(this.telemetryDir, /metrics-\d{4}-\d{2}-\d{2}\.jsonl$/, options.startDate, options.endDate);
         const results = [];
         const limit = options.limit || 100;
         const offset = options.offset || 0;
+        // Build index query options for indexable filters
+        const indexOptions = {
+            metricName: options.metricName,
+        };
         outer: for (const file of files) {
-            // Stream flat metric records (one per line) to avoid loading entire file into memory
-            for await (const raw of streamJsonl(file)) {
-                const point = normalizeMetric(raw);
-                if (!point)
-                    continue;
-                // Apply filters
-                if (options.metricName && !point.name.includes(options.metricName))
-                    continue;
-                results.push(point);
-                if (hasReachedLimit(results.length, offset, limit)) {
-                    break outer;
+            // Try to use index for pre-filtering
+            const matchingLines = this.tryUseIndex(file, 'metrics', indexOptions);
+            if (matchingLines !== null) {
+                // Use indexed query - read only matching lines
+                const rawRecords = await readLinesByNumber(file, matchingLines);
+                for (const raw of rawRecords) {
+                    const point = normalizeMetric(raw);
+                    if (!point)
+                        continue;
+                    results.push(point);
+                    if (hasReachedLimit(results.length, offset, limit)) {
+                        break outer;
+                    }
+                }
+            }
+            else {
+                // Fall back to full file scan
+                for await (const raw of streamJsonl(file)) {
+                    const point = normalizeMetric(raw);
+                    if (!point)
+                        continue;
+                    // Apply filters (since no index was used)
+                    if (options.metricName && !point.name.includes(options.metricName))
+                        continue;
+                    results.push(point);
+                    if (hasReachedLimit(results.length, offset, limit)) {
+                        break outer;
+                    }
                 }
             }
         }
         // Apply aggregation if requested
         if (options.aggregation && results.length > 0) {
-            return this.aggregate(results, options.aggregation, options.groupBy);
+            const aggregated = this.aggregate(results, options.aggregation, options.groupBy, options.timeBucket);
+            this.metricCache.set(cacheKey, aggregated);
+            return aggregated;
         }
-        return paginateResults(results, offset, limit);
+        const paginated = paginateResults(results, offset, limit);
+        this.metricCache.set(cacheKey, paginated);
+        return paginated;
     }
-    aggregate(points, aggregation, groupBy) {
-        // Group by metric name and optional attributes
+    aggregate(points, aggregation, groupBy, timeBucket) {
+        // Parse time bucket if provided
+        const bucketMs = timeBucket ? parseTimeBucket(timeBucket) : null;
+        // Group by metric name, optional attributes, and time bucket
         const groups = new Map();
         for (const point of points) {
             let key = point.name;
+            // Add time bucket to key if specified
+            if (bucketMs) {
+                const bucketTimestamp = floorToBucket(point.timestamp, bucketMs);
+                key += `|_bucket=${bucketTimestamp}`;
+            }
             if (groupBy) {
                 for (const attr of groupBy) {
                     const val = point.attributes?.[attr];
@@ -352,57 +710,150 @@ export class LocalJsonlBackend {
                 case 'count':
                     value = values.length;
                     break;
+                case 'p50':
+                    value = this.calculatePercentile(values, 50);
+                    break;
+                case 'p95':
+                    value = this.calculatePercentile(values, 95);
+                    break;
+                case 'p99':
+                    value = this.calculatePercentile(values, 99);
+                    break;
+                case 'rate':
+                    value = this.calculateRate(group);
+                    break;
+            }
+            // Use bucket timestamp if time bucketing is enabled
+            let timestamp = group[group.length - 1].timestamp;
+            if (bucketMs) {
+                const bucketTimestamp = floorToBucket(group[0].timestamp, bucketMs);
+                timestamp = new Date(bucketTimestamp).toISOString();
             }
             results.push({
-                timestamp: group[group.length - 1].timestamp,
+                timestamp,
                 name: group[0].name,
                 value,
                 unit: group[0].unit,
                 attributes: group[0].attributes,
             });
         }
+        // Sort results by timestamp when using time buckets
+        if (bucketMs) {
+            results.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
+        }
         return results;
     }
+    calculatePercentile(values, percentile) {
+        if (values.length === 0)
+            return 0;
+        const sorted = [...values].sort((a, b) => a - b);
+        const index = Math.ceil((percentile / 100) * sorted.length) - 1;
+        return sorted[Math.max(0, index)];
+    }
+    /**
+     * Calculate rate of change per second.
+     * Rate = (last_value - first_value) / duration_in_seconds
+     * Edge cases: single value returns 0, same timestamp returns 0
+     */
+    calculateRate(group) {
+        if (group.length < 2)
+            return 0;
+        // Sort by timestamp
+        const sorted = [...group].sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
+        const firstPoint = sorted[0];
+        const lastPoint = sorted[sorted.length - 1];
+        const firstTime = new Date(firstPoint.timestamp).getTime();
+        const lastTime = new Date(lastPoint.timestamp).getTime();
+        const durationMs = lastTime - firstTime;
+        // Avoid division by zero when timestamps are the same
+        if (durationMs === 0)
+            return 0;
+        const durationSeconds = durationMs / 1000;
+        return (lastPoint.value - firstPoint.value) / durationSeconds;
+    }
     async queryLLMEvents(options) {
+        // Check cache first
+        const cacheKey = makeCacheKey('llm-events', options);
+        const cached = this.llmEventCache.get(cacheKey);
+        if (cached)
+            return cached;
         const files = getFilesInRange(this.telemetryDir, /llm-events-\d{4}-\d{2}-\d{2}\.jsonl$/, options.startDate, options.endDate);
         const results = [];
         const limit = options.limit || 100;
         const offset = options.offset || 0;
+        // Build index query options - eventName maps to spanName in index
+        const indexOptions = {
+            spanName: options.eventName,
+        };
+        // Helper to apply non-indexable filters to an event
+        const applyFilters = (event) => {
+            if (options.model) {
+                const model = event.attributes?.['gen_ai.request.model'] || event.attributes?.['model'];
+                if (model !== options.model)
+                    return false;
+            }
+            if (options.provider) {
+                const provider = event.attributes?.['gen_ai.system'] || event.attributes?.['provider'];
+                if (provider !== options.provider)
+                    return false;
+            }
+            if (options.search) {
+                const searchLower = options.search.toLowerCase();
+                const attrStr = JSON.stringify(event.attributes).toLowerCase();
+                if (!attrStr.includes(searchLower) && !event.name.toLowerCase().includes(searchLower))
+                    return false;
+            }
+            return true;
+        };
         for (const file of files) {
-            // Stream LLM event records (one per line) to avoid loading entire file into memory
-            for await (const event of streamJsonl(file)) {
-                if (!event.timestamp || !event.name)
-                    continue;
-                // Apply filters
-                if (options.eventName && !event.name.includes(options.eventName))
-                    continue;
-                if (options.model) {
-                    const model = event.attributes?.['gen_ai.request.model'] || event.attributes?.['model'];
-                    if (model !== options.model)
+            // Try to use index for pre-filtering
+            const matchingLines = this.tryUseIndex(file, 'llm-events', indexOptions);
+            if (matchingLines !== null) {
+                // Use indexed query - read only matching lines
+                const rawRecords = await readLinesByNumber(file, matchingLines);
+                for (const event of rawRecords) {
+                    if (!event.timestamp || !event.name)
                         continue;
-                }
-                if (options.provider) {
-                    const provider = event.attributes?.['gen_ai.system'] || event.attributes?.['provider'];
-                    if (provider !== options.provider)
+                    if (!applyFilters(event))
                         continue;
+                    results.push({
+                        timestamp: event.timestamp,
+                        name: event.name,
+                        attributes: event.attributes,
+                    });
+                    if (hasReachedLimit(results.length, offset, limit)) {
+                        const paginated = paginateResults(results, offset, limit);
+                        this.llmEventCache.set(cacheKey, paginated);
+                        return paginated;
+                    }
                 }
-                if (options.search) {
-                    const searchLower = options.search.toLowerCase();
-                    const attrStr = JSON.stringify(event.attributes).toLowerCase();
-                    if (!attrStr.includes(searchLower) && !event.name.toLowerCase().includes(searchLower))
+            }
+            else {
+                // Fall back to full file scan
+                for await (const event of streamJsonl(file)) {
+                    if (!event.timestamp || !event.name)
                         continue;
-                }
-                results.push({
-                    timestamp: event.timestamp,
-                    name: event.name,
-                    attributes: event.attributes,
-                });
-                if (hasReachedLimit(results.length, offset, limit)) {
-                    return paginateResults(results, offset, limit);
+                    // Apply indexable filters (since no index was used)
+                    if (options.eventName && !event.name.includes(options.eventName))
+                        continue;
+                    if (!applyFilters(event))
+                        continue;
+                    results.push({
+                        timestamp: event.timestamp,
+                        name: event.name,
+                        attributes: event.attributes,
+                    });
+                    if (hasReachedLimit(results.length, offset, limit)) {
+                        const paginated = paginateResults(results, offset, limit);
+                        this.llmEventCache.set(cacheKey, paginated);
+                        return paginated;
+                    }
                 }
             }
         }
-        return paginateResults(results, offset, limit);
+        const paginated = paginateResults(results, offset, limit);
+        this.llmEventCache.set(cacheKey, paginated);
+        return paginated;
     }
     async healthCheck() {
         if (!existsSync(this.telemetryDir)) {
@@ -428,6 +879,27 @@ export class LocalJsonlBackend {
             message: `Found: ${found} for ${today}`,
         };
     }
+    /**
+     * Export traces in OTLP JSON format
+     */
+    async exportTracesOTLP(options) {
+        const traces = await this.queryTraces(options);
+        return convertToOTLPTraces(traces);
+    }
+    /**
+     * Export logs in OTLP JSON format
+     */
+    async exportLogsOTLP(options) {
+        const logs = await this.queryLogs(options);
+        return convertToOTLPLogs(logs);
+    }
+    /**
+     * Export metrics in OTLP JSON format
+     */
+    async exportMetricsOTLP(options) {
+        const metrics = await this.queryMetrics(options);
+        return convertToOTLPMetrics(metrics);
+    }
 }
 /**
  * Multi-directory backend that queries all telemetry directories
@@ -437,9 +909,9 @@ export class MultiDirectoryBackend {
     name = 'multi-directory';
     backends;
     directories;
-    constructor(cwd) {
+    constructor(cwd, useIndexes = true) {
         this.directories = getTelemetryDirectories(cwd);
-        this.backends = this.directories.map(d => new LocalJsonlBackend(d.path));
+        this.backends = this.directories.map(d => new LocalJsonlBackend(d.path, useIndexes));
     }
     getDirectories() {
         return this.directories;
@@ -509,5 +981,26 @@ export class MultiDirectoryBackend {
             directories: dirStatuses,
         };
     }
+    /**
+     * Export traces in OTLP JSON format
+     */
+    async exportTracesOTLP(options) {
+        const traces = await this.queryTraces(options);
+        return convertToOTLPTraces(traces);
+    }
+    /**
+     * Export logs in OTLP JSON format
+     */
+    async exportLogsOTLP(options) {
+        const logs = await this.queryLogs(options);
+        return convertToOTLPLogs(logs);
+    }
+    /**
+     * Export metrics in OTLP JSON format
+     */
+    async exportMetricsOTLP(options) {
+        const metrics = await this.queryMetrics(options);
+        return convertToOTLPMetrics(metrics);
+    }
 }
 //# sourceMappingURL=local-jsonl.js.map