observability-toolkit 1.8.5 → 2.0.0

package/dist/backends/local-jsonl.js

@@ -5,15 +5,19 @@
  * span or log record, not the batched OpenTelemetry export format.
  */
 import { join } from 'path';
-import { GENAI_EVALUATION_ATTRIBUTES, } from './index.js';
+import { GENAI_EVALUATION_ATTRIBUTES, GENAI_AGENT_ATTRIBUTES, AGENT_JUDGE_ATTRIBUTES, } from './index.js';
 import { convertToOTLPTraces, convertToOTLPLogs, convertToOTLPMetrics, } from '../lib/otlp-export.js';
-import { TELEMETRY_DIR, getTelemetryDirectories, getSpanKind, getStatusCodeName } from '../lib/constants.js';
+import { TELEMETRY_DIR, getTelemetryDirectories, getSpanKind, getStatusCodeName, WORST_FILES_LIMIT } from '../lib/constants.js';
 import { listFiles, streamJsonl, parseDateFromFilename, getDateString, paginateResults, hasReachedLimit, } from '../lib/file-utils.js';
 import { QueryCache, makeCacheKey } from '../lib/cache.js';
+import { recordQueryDuration, recordCacheHit } from '../lib/metrics.js';
+import { Histogram } from '../lib/histogram.js';
 import { getIndexPath, readIndex, isIndexStale, queryIndex, readLinesByNumber, } from '../lib/indexer.js';
 import { sanitizePath } from '../lib/error-sanitizer.js';
 import { CircuitBreaker } from '../lib/circuit-breaker.js';
+import { withSpan } from '../lib/instrumentation.js';
 import { existsSync } from 'fs';
+import { ParseStatsTracker } from '../lib/parse-stats.js';
 function startTiming() {
     const start = performance.now();
     return {
@@ -49,7 +53,9 @@ function getCachedRegex(pattern) {
         return regex;
     }
     catch {
-        console.warn(`Invalid regex pattern: ${pattern}`);
+        // Security: Truncate pattern in log to prevent log injection/info disclosure
+        const truncatedPattern = pattern.length > 50 ? pattern.slice(0, 50) + '...' : pattern;
+        console.warn(`Invalid regex pattern: ${truncatedPattern.replace(/[\n\r]/g, ' ')}`);
         return null;
     }
 }
@@ -216,6 +222,82 @@ function extractValidEvaluatorType(value) {
  * Convert flat evaluation to normalized EvaluationResult
  * Includes runtime validation for data integrity (P0-2 fix)
  */
+/**
+ * Safely extract StepScore array from raw attribute value.
+ * Returns undefined if not a valid array or if parsing fails.
+ */
+function extractStepScores(value) {
+    if (!value)
+        return undefined;
+    // If it's a string, try to parse as JSON
+    let arr;
+    if (typeof value === 'string') {
+        try {
+            arr = JSON.parse(value);
+            if (!Array.isArray(arr))
+                return undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    else if (Array.isArray(value)) {
+        arr = value;
+    }
+    else {
+        return undefined;
+    }
+    // Validate each element and filter invalid entries
+    return arr
+        .filter((item) => {
+        if (!item || typeof item !== 'object')
+            return false;
+        const obj = item;
+        return ((typeof obj.step === 'string' || typeof obj.step === 'number') &&
+            typeof obj.score === 'number' &&
+            Number.isFinite(obj.score));
+    })
+        .slice(0, 1000); // Respect MAX_STEP_SCORES limit
+}
+/**
+ * Safely extract ToolVerification array from raw attribute value.
+ * Returns undefined if not a valid array or if parsing fails.
+ */
+function extractToolVerifications(value) {
+    if (!value)
+        return undefined;
+    // If it's a string, try to parse as JSON
+    let arr;
+    if (typeof value === 'string') {
+        try {
+            arr = JSON.parse(value);
+            if (!Array.isArray(arr))
+                return undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    else if (Array.isArray(value)) {
+        arr = value;
+    }
+    else {
+        return undefined;
+    }
+    // Validate each element and filter invalid entries
+    return arr
+        .filter((item) => {
+        if (!item || typeof item !== 'object')
+            return false;
+        const obj = item;
+        return (typeof obj.toolName === 'string' &&
+            typeof obj.toolCorrect === 'boolean' &&
+            typeof obj.argsCorrect === 'boolean' &&
+            typeof obj.score === 'number' &&
+            Number.isFinite(obj.score));
+    })
+        .slice(0, 500); // Respect MAX_TOOL_VERIFICATIONS limit
+}
 function normalizeEvaluation(raw) {
     const attrs = raw.attributes;
     // evaluationName is required per OTel spec - must be non-empty string
@@ -239,6 +321,12 @@ function normalizeEvaluation(raw) {
         traceId: raw.traceId,
         spanId: raw.spanId,
         sessionId: extractValidString(attrs?.['session.id']),
+        // Agent-as-Judge fields (Section 10.7)
+        agentId: extractValidString(attrs?.[GENAI_AGENT_ATTRIBUTES.AGENT_ID]),
+        agentName: extractValidString(attrs?.[GENAI_AGENT_ATTRIBUTES.AGENT_NAME]),
+        stepScores: extractStepScores(attrs?.[AGENT_JUDGE_ATTRIBUTES.STEP_SCORES]),
+        toolVerifications: extractToolVerifications(attrs?.[AGENT_JUDGE_ATTRIBUTES.TOOL_VERIFICATIONS]),
+        trajectoryLength: extractValidNumber(attrs?.[AGENT_JUDGE_ATTRIBUTES.TRAJECTORY_LENGTH]),
     };
 }
 /**
@@ -460,17 +548,27 @@ function getFilesInRange(dir, pattern, startDate, endDate) {
 export class LocalJsonlBackend {
     name = 'local-jsonl';
     telemetryDir;
-    traceCache = new QueryCache();
-    logCache = new QueryCache();
-    metricCache = new QueryCache();
-    llmEventCache = new QueryCache();
-    evaluationCache = new QueryCache();
+    traceCache = new QueryCache(100, 60000, 'traces');
+    logCache = new QueryCache(100, 60000, 'logs');
+    metricCache = new QueryCache(100, 60000, 'metrics');
+    llmEventCache = new QueryCache(100, 60000, 'llmEvents');
+    evaluationCache = new QueryCache(100, 60000, 'evaluations');
     useIndexes;
     circuitBreaker;
+    parseStatsTracker;
+    /** Histograms for tracking query latency per type */
+    queryHistograms = {
+        traces: new Histogram(),
+        logs: new Histogram(),
+        metrics: new Histogram(),
+        llmEvents: new Histogram(),
+        evaluations: new Histogram(),
+    };
     constructor(telemetryDir, useIndexes = true) {
         this.telemetryDir = telemetryDir || TELEMETRY_DIR;
         this.useIndexes = useIndexes;
         this.circuitBreaker = new CircuitBreaker({ name: 'local-file-io' });
+        this.parseStatsTracker = new ParseStatsTracker();
     }
     /**
      * Get circuit breaker state (for health check and testing)
@@ -506,6 +604,41 @@ export class LocalJsonlBackend {
             evaluations: this.evaluationCache.getStats(),
         };
     }
+    /**
+     * Get parse statistics for JSONL file processing.
+     * Returns aggregate stats including success rate and worst files.
+     */
+    getParseStats() {
+        return this.parseStatsTracker.getAggregateStats();
+    }
+    /**
+     * Reset parse statistics (useful for testing)
+     */
+    resetParseStats() {
+        this.parseStatsTracker.reset();
+    }
+    /**
+     * Get query latency statistics per query type
+     */
+    getQueryStats() {
+        return {
+            traces: this.queryHistograms.traces.getStats(),
+            logs: this.queryHistograms.logs.getStats(),
+            metrics: this.queryHistograms.metrics.getStats(),
+            llmEvents: this.queryHistograms.llmEvents.getStats(),
+            evaluations: this.queryHistograms.evaluations.getStats(),
+        };
+    }
+    /**
+     * Reset query latency histograms (useful for testing)
+     */
+    resetQueryStats() {
+        this.queryHistograms.traces.reset();
+        this.queryHistograms.logs.reset();
+        this.queryHistograms.metrics.reset();
+        this.queryHistograms.llmEvents.reset();
+        this.queryHistograms.evaluations.reset();
+    }
     /**
      * Try to use an index for a file, returning matching line numbers or null if full scan needed
      */
@@ -521,365 +654,438 @@ export class LocalJsonlBackend {
         return queryIndex(index, indexOptions);
     }
     async queryTraces(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryTraces took ${durationMs.toFixed(1)}ms`);
-            }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
-            return [];
-        }
-        // Check cache first
-        const cacheKey = makeCacheKey('traces', options);
-        const cached = this.traceCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
-        }
-        const files = getFilesInRange(this.telemetryDir, /traces-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Use cached regex to avoid recompilation for frequently-used patterns
-        const spanNameRegex = options.spanNameRegex
-            ? getCachedRegex(options.spanNameRegex)
-            : null;
-        // Build index query options for indexable filters
-        const indexOptions = {
-            traceId: options.traceId,
-            spanName: options.spanName,
-            serviceName: options.serviceName,
-        };
-        // Helper to apply non-indexable filters to a span
-        const applyFilters = (span) => {
-            // Regex filter (not indexable)
-            if (spanNameRegex && !spanNameRegex.test(span.name))
-                return false;
-            if (options.excludeSpanName && span.name.includes(options.excludeSpanName))
-                return false;
-            if (options.minDurationMs && (span.durationMs || 0) < options.minDurationMs)
-                return false;
-            if (options.maxDurationMs && (span.durationMs || Infinity) > options.maxDurationMs)
-                return false;
-            // Apply attribute filter
-            if (options.attributeFilter) {
-                for (const [key, value] of Object.entries(options.attributeFilter)) {
-                    if (span.attributes?.[key] !== value)
-                        return false;
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.traces', () => ({
+            'obs_toolkit.query.type': 'traces',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? 100,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('traces', durationMs, 'local');
+                this.queryHistograms.traces.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    console.warn(`[obs-toolkit] Slow query: queryTraces took ${durationMs.toFixed(1)}ms`);
                 }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-            // Apply attributeExists filter
-            if (options.attributeExists) {
-                for (const key of options.attributeExists) {
-                    if (span.attributes?.[key] === undefined)
-                        return false;
+            // Check cache first
+            const cacheKey = makeCacheKey('traces', options);
+            const cached = this.traceCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('traces');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
                 }
+                return cached;
             }
-            // Apply attributeNotExists filter
-            if (options.attributeNotExists) {
-                for (const key of options.attributeNotExists) {
-                    if (span.attributes?.[key] !== undefined)
+            const files = getFilesInRange(this.telemetryDir, /traces-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
+            const results = [];
+            const limit = options.limit || 100;
+            const offset = options.offset || 0;
+            // Use cached regex to avoid recompilation for frequently-used patterns
+            const spanNameRegex = options.spanNameRegex
+                ? getCachedRegex(options.spanNameRegex)
+                : null;
+            // Build index query options for indexable filters
+            const indexOptions = {
+                traceId: options.traceId,
+                spanName: options.spanName,
+                serviceName: options.serviceName,
+            };
+            // Helper to apply non-indexable filters to a span
+            const applyFilters = (s) => {
+                // Regex filter (not indexable)
+                if (spanNameRegex && !spanNameRegex.test(s.name))
+                    return false;
+                if (options.excludeSpanName && s.name.includes(options.excludeSpanName))
+                    return false;
+                if (options.minDurationMs && (s.durationMs || 0) < options.minDurationMs)
+                    return false;
+                if (options.maxDurationMs && (s.durationMs || Infinity) > options.maxDurationMs)
+                    return false;
+                // Apply attribute filter
+                if (options.attributeFilter) {
+                    for (const [key, value] of Object.entries(options.attributeFilter)) {
+                        if (s.attributes?.[key] !== value)
+                            return false;
+                    }
+                }
+                // Apply attributeExists filter
+                if (options.attributeExists) {
+                    for (const key of options.attributeExists) {
+                        if (s.attributes?.[key] === undefined)
+                            return false;
+                    }
+                }
+                // Apply attributeNotExists filter
+                if (options.attributeNotExists) {
+                    for (const key of options.attributeNotExists) {
+                        if (s.attributes?.[key] !== undefined)
+                            return false;
+                    }
+                }
+                // Apply numeric filter conditions
+                if (options.numericFilter && options.numericFilter.length > 0) {
+                    if (!applyNumericFilters(s.attributes, options.numericFilter))
                         return false;
                 }
-            }
-            // Apply numeric filter conditions
-            if (options.numericFilter && options.numericFilter.length > 0) {
-                if (!applyNumericFilters(span.attributes, options.numericFilter))
+                // OTel GenAI agent/tool filters
+                if (options.agentId && s.attributes?.['gen_ai.agent.id'] !== options.agentId)
                     return false;
-            }
-            // OTel GenAI agent/tool filters
-            if (options.agentId && span.attributes?.['gen_ai.agent.id'] !== options.agentId)
-                return false;
-            if (options.agentName && span.attributes?.['gen_ai.agent.name'] !== options.agentName)
-                return false;
-            if (options.toolName && span.attributes?.['gen_ai.tool.name'] !== options.toolName)
-                return false;
-            if (options.toolCallId && span.attributes?.['gen_ai.tool.call.id'] !== options.toolCallId)
-                return false;
-            if (options.toolType && span.attributes?.['gen_ai.tool.type'] !== options.toolType)
-                return false;
-            if (options.operationName && span.attributes?.['gen_ai.operation.name'] !== options.operationName)
-                return false;
-            return true;
-        };
-        try {
-            for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'traces', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const raw of rawRecords) {
-                        const span = normalizeSpan(raw);
-                        if (!span)
-                            continue;
-                        if (!applyFilters(span))
-                            continue;
-                        results.push(span);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.traceCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+                if (options.agentName && s.attributes?.['gen_ai.agent.name'] !== options.agentName)
+                    return false;
+                if (options.toolName && s.attributes?.['gen_ai.tool.name'] !== options.toolName)
+                    return false;
+                if (options.toolCallId && s.attributes?.['gen_ai.tool.call.id'] !== options.toolCallId)
+                    return false;
+                if (options.toolType && s.attributes?.['gen_ai.tool.type'] !== options.toolType)
+                    return false;
+                if (options.operationName && s.attributes?.['gen_ai.operation.name'] !== options.operationName)
+                    return false;
+                return true;
+            };
+            try {
+                for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'traces', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const raw of rawRecords) {
+                            const traceSpan = normalizeSpan(raw);
+                            if (!traceSpan)
+                                continue;
+                            if (!applyFilters(traceSpan))
+                                continue;
+                            results.push(traceSpan);
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.traceCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
                         }
                     }
-                }
-                else {
-                    // Fall back to full file scan
-                    for await (const raw of streamJsonl(file)) {
-                        const span = normalizeSpan(raw);
-                        if (!span)
-                            continue;
-                        // Apply indexable filters (since no index was used)
-                        if (options.traceId && span.traceId !== options.traceId)
-                            continue;
-                        if (options.spanName && !span.name.includes(options.spanName))
-                            continue;
-                        if (options.serviceName) {
-                            const svc = span.attributes?.['service.name'];
-                            if (svc !== options.serviceName)
+                    else {
+                        // Fall back to full file scan
+                        for await (const raw of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            const traceSpan = normalizeSpan(raw);
+                            if (!traceSpan)
                                 continue;
-                        }
-                        if (!applyFilters(span))
-                            continue;
-                        results.push(span);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.traceCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+                            // Apply indexable filters (since no index was used)
+                            if (options.traceId && traceSpan.traceId !== options.traceId)
+                                continue;
+                            if (options.spanName && !traceSpan.name.includes(options.spanName))
+                                continue;
+                            if (options.serviceName) {
+                                const svc = traceSpan.attributes?.['service.name'];
+                                if (svc !== options.serviceName)
+                                    continue;
+                            }
+                            if (!applyFilters(traceSpan))
+                                continue;
+                            results.push(traceSpan);
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.traceCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
                         }
                     }
                 }
+                const paginated = paginateResults(results, offset, limit);
+                this.traceCache.set(cacheKey, paginated);
+                this.circuitBreaker.recordSuccess();
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.traceCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
-        }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
-        }
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
+            }
+        });
     }
     async queryLogs(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryLogs took ${durationMs.toFixed(1)}ms`);
-            }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
-            return [];
-        }
-        // Check cache first
-        const cacheKey = makeCacheKey('logs', options);
-        const cached = this.logCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
-        }
-        const files = getFilesInRange(this.telemetryDir, /logs-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Build index query options for indexable filters
-        const indexOptions = {
-            traceId: options.traceId,
-            severity: options.severity,
-        };
-        // Helper to apply non-indexable filters to a log
-        const applyFilters = (log) => {
-            if (options.search && !log.body.toLowerCase().includes(options.search.toLowerCase()))
-                return false;
-            // Apply boolean search with multiple terms
-            if (options.searchTerms && options.searchTerms.length > 0) {
-                const bodyLower = log.body.toLowerCase();
-                const operator = options.searchOperator || 'AND';
-                if (operator === 'AND') {
-                    const allMatch = options.searchTerms.every(term => bodyLower.includes(term.toLowerCase()));
-                    if (!allMatch)
-                        return false;
-                }
-                else {
-                    const anyMatch = options.searchTerms.some(term => bodyLower.includes(term.toLowerCase()));
-                    if (!anyMatch)
-                        return false;
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.logs', () => ({
+            'obs_toolkit.query.type': 'logs',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? 100,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('logs', durationMs, 'local');
+                this.queryHistograms.logs.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    console.warn(`[obs-toolkit] Slow query: queryLogs took ${durationMs.toFixed(1)}ms`);
                 }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-            if (options.excludeSearch && log.body.toLowerCase().includes(options.excludeSearch.toLowerCase()))
-                return false;
-            // Apply attributeExists filter
-            if (options.attributeExists) {
-                for (const key of options.attributeExists) {
-                    if (log.attributes?.[key] === undefined)
-                        return false;
+            // Check cache first
+            const cacheKey = makeCacheKey('logs', options);
+            const cached = this.logCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('logs');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
                 }
+                return cached;
             }
-            // Apply attributeNotExists filter
-            if (options.attributeNotExists) {
-                for (const key of options.attributeNotExists) {
-                    if (log.attributes?.[key] !== undefined)
-                        return false;
+            const files = getFilesInRange(this.telemetryDir, /logs-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
+            const results = [];
+            const limit = options.limit || 100;
+            const offset = options.offset || 0;
+            // Build index query options for indexable filters
+            const indexOptions = {
+                traceId: options.traceId,
+                severity: options.severity,
+            };
+            // Helper to apply non-indexable filters to a log
+            const applyFilters = (logRecord) => {
+                if (options.search && !logRecord.body.toLowerCase().includes(options.search.toLowerCase()))
+                    return false;
+                // Apply boolean search with multiple terms
+                if (options.searchTerms && options.searchTerms.length > 0) {
+                    const bodyLower = logRecord.body.toLowerCase();
+                    const operator = options.searchOperator || 'AND';
+                    if (operator === 'AND') {
+                        const allMatch = options.searchTerms.every(term => bodyLower.includes(term.toLowerCase()));
+                        if (!allMatch)
+                            return false;
+                    }
+                    else {
+                        const anyMatch = options.searchTerms.some(term => bodyLower.includes(term.toLowerCase()));
+                        if (!anyMatch)
+                            return false;
+                    }
                 }
-            }
-            // Apply numeric filter conditions
-            if (options.numericFilter && options.numericFilter.length > 0) {
-                if (!applyNumericFilters(log.attributes, options.numericFilter))
+                if (options.excludeSearch && logRecord.body.toLowerCase().includes(options.excludeSearch.toLowerCase()))
                     return false;
-            }
-            return true;
-        };
-        try {
-            for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'logs', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const raw of rawRecords) {
-                        const log = normalizeLog(raw, options.extractFields);
-                        if (!log)
-                            continue;
-                        if (!applyFilters(log))
-                            continue;
-                        results.push(log);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.logCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
-                        }
+                // Apply attributeExists filter
+                if (options.attributeExists) {
+                    for (const key of options.attributeExists) {
+                        if (logRecord.attributes?.[key] === undefined)
+                            return false;
+                    }
+                }
+                // Apply attributeNotExists filter
+                if (options.attributeNotExists) {
+                    for (const key of options.attributeNotExists) {
+                        if (logRecord.attributes?.[key] !== undefined)
+                            return false;
                     }
                 }
-                else {
-                    // Fall back to full file scan
-                    for await (const raw of streamJsonl(file)) {
-                        const log = normalizeLog(raw, options.extractFields);
-                        if (!log)
-                            continue;
-                        // Apply indexable filters (since no index was used)
-                        if (options.severity && log.severity.toUpperCase() !== options.severity.toUpperCase())
-                            continue;
-                        if (options.traceId && log.traceId !== options.traceId)
-                            continue;
-                        if (!applyFilters(log))
-                            continue;
-                        results.push(log);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.logCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+                // Apply numeric filter conditions
+                if (options.numericFilter && options.numericFilter.length > 0) {
+                    if (!applyNumericFilters(logRecord.attributes, options.numericFilter))
+                        return false;
+                }
+                return true;
+            };
+            try {
+                for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'logs', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const raw of rawRecords) {
+                            const logRecord = normalizeLog(raw, options.extractFields);
+                            if (!logRecord)
+                                continue;
+                            if (!applyFilters(logRecord))
+                                continue;
+                            results.push(logRecord);
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.logCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
+                        }
+                    }
+                    else {
+                        // Fall back to full file scan
+                        for await (const raw of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            const logRecord = normalizeLog(raw, options.extractFields);
+                            if (!logRecord)
+                                continue;
+                            // Apply indexable filters (since no index was used)
+                            if (options.severity && logRecord.severity.toUpperCase() !== options.severity.toUpperCase())
+                                continue;
+                            if (options.traceId && logRecord.traceId !== options.traceId)
+                                continue;
+                            if (!applyFilters(logRecord))
+                                continue;
+                            results.push(logRecord);
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.logCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
                         }
                     }
                 }
+                const paginated = paginateResults(results, offset, limit);
+                this.logCache.set(cacheKey, paginated);
+                this.circuitBreaker.recordSuccess();
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.logCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
-        }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
-        }
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
+            }
+        });
     }
     async queryMetrics(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryMetrics took ${durationMs.toFixed(1)}ms`);
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.metrics', () => ({
+            'obs_toolkit.query.type': 'metrics',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? 100,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('metrics', durationMs, 'local');
+                this.queryHistograms.metrics.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    console.warn(`[obs-toolkit] Slow query: queryMetrics took ${durationMs.toFixed(1)}ms`);
+                }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
-            return [];
-        }
-        // Check cache first
-        const cacheKey = makeCacheKey('metrics', options);
-        const cached = this.metricCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
-        }
-        const files = getFilesInRange(this.telemetryDir, /metrics-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Build index query options for indexable filters
-        const indexOptions = {
-            metricName: options.metricName,
-        };
-        try {
-            outer: for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'metrics', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const raw of rawRecords) {
-                        const point = normalizeMetric(raw);
-                        if (!point)
-                            continue;
-                        results.push(point);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            break outer;
+            // Check cache first
+            const cacheKey = makeCacheKey('metrics', options);
+            const cached = this.metricCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('metrics');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
+                }
+                return cached;
+            }
+            const files = getFilesInRange(this.telemetryDir, /metrics-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
+            const results = [];
+            const limit = options.limit || 100;
+            const offset = options.offset || 0;
+            // Build index query options for indexable filters
+            const indexOptions = {
+                metricName: options.metricName,
+            };
+            try {
+                outer: for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'metrics', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const raw of rawRecords) {
+                            const point = normalizeMetric(raw);
+                            if (!point)
+                                continue;
+                            results.push(point);
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                break outer;
+                            }
                         }
                     }
-                }
-                else {
-                    // Fall back to full file scan
-                    for await (const raw of streamJsonl(file)) {
-                        const point = normalizeMetric(raw);
-                        if (!point)
-                            continue;
-                        // Apply filters (since no index was used)
-                        if (options.metricName && !point.name.includes(options.metricName))
-                            continue;
-                        results.push(point);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            break outer;
+                    else {
+                        // Fall back to full file scan
+                        for await (const raw of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            const point = normalizeMetric(raw);
+                            if (!point)
+                                continue;
+                            // Apply filters (since no index was used)
+                            if (options.metricName && !point.name.includes(options.metricName))
+                                continue;
+                            results.push(point);
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                break outer;
+                            }
                         }
                     }
                 }
-            }
-            // Apply aggregation if requested
-            if (options.aggregation && results.length > 0) {
-                const aggregated = this.aggregate(results, options.aggregation, options.groupBy, options.timeBucket);
-                this.metricCache.set(cacheKey, aggregated);
+                // Apply aggregation if requested
+                if (options.aggregation && results.length > 0) {
+                    const aggregated = this.aggregate(results, options.aggregation, options.groupBy, options.timeBucket);
+                    this.metricCache.set(cacheKey, aggregated);
+                    this.circuitBreaker.recordSuccess();
+                    logTiming();
+                    if (span)
+                        span.setAttribute('obs_toolkit.query.result_count', aggregated.length);
+                    return aggregated;
+                }
+                const paginated = paginateResults(results, offset, limit);
+                this.metricCache.set(cacheKey, paginated);
                 this.circuitBreaker.recordSuccess();
                 logTiming();
-                return aggregated;
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.metricCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
-        }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
-        }
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
+            }
+        });
     }
     aggregate(points, aggregation, groupBy, timeBucket) {
         // Parse time bucket if provided
@@ -987,247 +1193,304 @@ export class LocalJsonlBackend {
         return (lastPoint.value - firstPoint.value) / durationSeconds;
     }
     async queryLLMEvents(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryLLMEvents took ${durationMs.toFixed(1)}ms`);
-            }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
-            return [];
-        }
-        // Check cache first
-        const cacheKey = makeCacheKey('llm-events', options);
-        const cached = this.llmEventCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
-        }
-        const files = getFilesInRange(this.telemetryDir, /llm-events-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Build index query options - eventName maps to spanName in index
-        const indexOptions = {
-            spanName: options.eventName,
-        };
-        // Helper to apply non-indexable filters to an event
-        const applyFilters = (event) => {
-            if (options.model) {
-                const model = event.attributes?.['gen_ai.request.model'] || event.attributes?.['model'];
-                if (model !== options.model)
-                    return false;
-            }
-            if (options.provider) {
-                // OTel GenAI compliant: gen_ai.provider.name -> gen_ai.system -> provider
-                const provider = event.attributes?.['gen_ai.provider.name'] ||
-                    event.attributes?.['gen_ai.system'] ||
-                    event.attributes?.['provider'];
-                if (provider !== options.provider)
-                    return false;
-            }
-            if (options.operationName) {
-                const opName = event.attributes?.['gen_ai.operation.name'];
-                if (opName !== options.operationName)
-                    return false;
-            }
-            if (options.conversationId) {
-                const convId = event.attributes?.['gen_ai.conversation.id'];
-                if (convId !== options.conversationId)
-                    return false;
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.llm_events', () => ({
+            'obs_toolkit.query.type': 'llm_events',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? 100,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('llm_events', durationMs, 'local');
+                this.queryHistograms.llmEvents.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    console.warn(`[obs-toolkit] Slow query: queryLLMEvents took ${durationMs.toFixed(1)}ms`);
+                }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-            if (options.search) {
-                const searchLower = options.search.toLowerCase();
-                const attrStr = JSON.stringify(event.attributes).toLowerCase();
-                if (!attrStr.includes(searchLower) && !event.name.toLowerCase().includes(searchLower))
-                    return false;
+            // Check cache first
+            const cacheKey = makeCacheKey('llm-events', options);
+            const cached = this.llmEventCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('llmEvents');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
+                }
+                return cached;
             }
-            return true;
-        };
-        try {
-            for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'llm-events', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const event of rawRecords) {
-                        if (!event.timestamp || !event.name)
-                            continue;
-                        if (!applyFilters(event))
-                            continue;
-                        results.push({
-                            timestamp: event.timestamp,
-                            name: event.name,
-                            attributes: event.attributes,
-                        });
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.llmEventCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+            const files = getFilesInRange(this.telemetryDir, /llm-events-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
+            const results = [];
+            const limit = options.limit || 100;
+            const offset = options.offset || 0;
+            // Build index query options - eventName maps to spanName in index
+            const indexOptions = {
+                spanName: options.eventName,
+            };
+            // Helper to apply non-indexable filters to an event
+            const applyFilters = (event) => {
+                if (options.model) {
+                    const model = event.attributes?.['gen_ai.request.model'] || event.attributes?.['model'];
+                    if (model !== options.model)
+                        return false;
+                }
+                if (options.provider) {
+                    // OTel GenAI compliant: gen_ai.provider.name -> gen_ai.system -> provider
+                    const provider = event.attributes?.['gen_ai.provider.name'] ||
+                        event.attributes?.['gen_ai.system'] ||
+                        event.attributes?.['provider'];
+                    if (provider !== options.provider)
+                        return false;
+                }
+                if (options.operationName) {
+                    const opName = event.attributes?.['gen_ai.operation.name'];
+                    if (opName !== options.operationName)
+                        return false;
+                }
+                if (options.conversationId) {
+                    const convId = event.attributes?.['gen_ai.conversation.id'];
+                    if (convId !== options.conversationId)
+                        return false;
+                }
+                if (options.search) {
+                    const searchLower = options.search.toLowerCase();
+                    const attrStr = JSON.stringify(event.attributes).toLowerCase();
+                    if (!attrStr.includes(searchLower) && !event.name.toLowerCase().includes(searchLower))
+                        return false;
+                }
+                return true;
+            };
+            try {
+                for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'llm-events', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const event of rawRecords) {
+                            if (!event.timestamp || !event.name)
+                                continue;
+                            if (!applyFilters(event))
+                                continue;
+                            results.push({
+                                timestamp: event.timestamp,
+                                name: event.name,
+                                attributes: event.attributes,
+                            });
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.llmEventCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
                         }
                     }
-                }
-                else {
-                    // Fall back to full file scan
-                    for await (const event of streamJsonl(file)) {
-                        if (!event.timestamp || !event.name)
-                            continue;
-                        // Apply indexable filters (since no index was used)
-                        if (options.eventName && !event.name.includes(options.eventName))
-                            continue;
-                        if (!applyFilters(event))
-                            continue;
-                        results.push({
-                            timestamp: event.timestamp,
-                            name: event.name,
-                            attributes: event.attributes,
-                        });
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.llmEventCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+                    else {
+                        // Fall back to full file scan
+                        for await (const event of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            if (!event.timestamp || !event.name)
+                                continue;
+                            // Apply indexable filters (since no index was used)
+                            if (options.eventName && !event.name.includes(options.eventName))
+                                continue;
+                            if (!applyFilters(event))
+                                continue;
+                            results.push({
+                                timestamp: event.timestamp,
+                                name: event.name,
+                                attributes: event.attributes,
+                            });
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.llmEventCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
                         }
                     }
                 }
+                const paginated = paginateResults(results, offset, limit);
+                this.llmEventCache.set(cacheKey, paginated);
+                this.circuitBreaker.recordSuccess();
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.llmEventCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
-        }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
-        }
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
+            }
+        });
     }
     async queryEvaluations(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryEvaluations took ${durationMs.toFixed(1)}ms`);
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.evaluations', () => ({
+            'obs_toolkit.query.type': 'evaluations',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? 100,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('evaluations', durationMs, 'local');
+                this.queryHistograms.evaluations.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    console.warn(`[obs-toolkit] Slow query: queryEvaluations took ${durationMs.toFixed(1)}ms`);
+                }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
-            return [];
-        }
-        // Check cache first
-        const cacheKey = makeCacheKey('evaluations', options);
-        const cached = this.evaluationCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
-        }
-        const files = getFilesInRange(this.telemetryDir, /evaluations-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Build index query options for indexable filters
-        const indexOptions = {
-            traceId: options.traceId,
-            evaluationName: options.evaluationName,
-            scoreLabel: options.scoreLabel,
-            responseId: options.responseId,
-            evaluator: options.evaluator,
-        };
-        // Helper to apply non-indexable filters to an evaluation
-        const applyFilters = (evaluation) => {
-            // Score range filters - only apply when evaluation HAS a scoreValue (P1-1 fix)
-            // Evaluations with only scoreLabel (qualitative) should pass through
-            if (options.scoreMin !== undefined && evaluation.scoreValue !== undefined) {
-                if (evaluation.scoreValue < options.scoreMin)
-                    return false;
+            // Check cache first
+            const cacheKey = makeCacheKey('evaluations', options);
+            const cached = this.evaluationCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('evaluations');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
+                }
+                return cached;
             }
-            if (options.scoreMax !== undefined && evaluation.scoreValue !== undefined) {
-                if (evaluation.scoreValue > options.scoreMax)
+            const files = getFilesInRange(this.telemetryDir, /evaluations-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
+            const results = [];
+            const limit = options.limit || 100;
+            const offset = options.offset || 0;
+            // Build index query options for indexable filters
+            const indexOptions = {
+                traceId: options.traceId,
+                evaluationName: options.evaluationName,
+                scoreLabel: options.scoreLabel,
+                responseId: options.responseId,
+                evaluator: options.evaluator,
+            };
+            // Helper to apply non-indexable filters to an evaluation
+            const applyFilters = (evaluation) => {
+                // Score range filters - only apply when evaluation HAS a scoreValue (P1-1 fix)
+                // Evaluations with only scoreLabel (qualitative) should pass through
+                if (options.scoreMin !== undefined && evaluation.scoreValue !== undefined) {
+                    if (evaluation.scoreValue < options.scoreMin)
+                        return false;
+                }
+                if (options.scoreMax !== undefined && evaluation.scoreValue !== undefined) {
+                    if (evaluation.scoreValue > options.scoreMax)
+                        return false;
+                }
+                // Session filter
+                if (options.sessionId && evaluation.sessionId !== options.sessionId) {
                     return false;
-            }
-            // Session filter
-            if (options.sessionId && evaluation.sessionId !== options.sessionId) {
-                return false;
-            }
-            // Evaluator type filter (not indexed, so always applied here)
-            if (options.evaluatorType && evaluation.evaluatorType !== options.evaluatorType) {
-                return false;
-            }
-            return true;
-        };
-        try {
-            for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'evaluations', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const raw of rawRecords) {
-                        const evaluation = normalizeEvaluation(raw);
-                        if (!evaluation)
-                            continue;
-                        if (!applyFilters(evaluation))
-                            continue;
-                        results.push(evaluation);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.evaluationCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+                }
+                // Evaluator type filter (not indexed, so always applied here)
+                if (options.evaluatorType && evaluation.evaluatorType !== options.evaluatorType) {
+                    return false;
+                }
+                // Agent-as-Judge filters (Section 10.7)
+                if (options.agentId && evaluation.agentId !== options.agentId) {
+                    return false;
+                }
+                if (options.agentName && evaluation.agentName !== options.agentName) {
+                    return false;
+                }
+                return true;
+            };
+            try {
+                for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'evaluations', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const raw of rawRecords) {
+                            const evaluation = normalizeEvaluation(raw);
+                            if (!evaluation)
+                                continue;
+                            if (!applyFilters(evaluation))
+                                continue;
+                            results.push(evaluation);
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.evaluationCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
                         }
                     }
-                }
-                else {
-                    // Fall back to full file scan
-                    for await (const raw of streamJsonl(file)) {
-                        const evaluation = normalizeEvaluation(raw);
-                        if (!evaluation)
-                            continue;
-                        // Apply indexable filters (since no index was used)
-                        if (options.traceId && evaluation.traceId !== options.traceId)
-                            continue;
-                        if (options.evaluationName && !evaluation.evaluationName.includes(options.evaluationName))
-                            continue;
-                        if (options.scoreLabel && evaluation.scoreLabel !== options.scoreLabel)
-                            continue;
-                        if (options.responseId && evaluation.responseId !== options.responseId)
-                            continue;
-                        if (options.evaluator && evaluation.evaluator !== options.evaluator)
-                            continue;
-                        if (!applyFilters(evaluation))
-                            continue;
-                        results.push(evaluation);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.evaluationCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+                    else {
+                        // Fall back to full file scan
+                        for await (const raw of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            const evaluation = normalizeEvaluation(raw);
+                            if (!evaluation)
+                                continue;
+                            // Apply indexable filters (since no index was used)
+                            if (options.traceId && evaluation.traceId !== options.traceId)
+                                continue;
+                            if (options.evaluationName && !evaluation.evaluationName.includes(options.evaluationName))
+                                continue;
+                            if (options.scoreLabel && evaluation.scoreLabel !== options.scoreLabel)
+                                continue;
+                            if (options.responseId && evaluation.responseId !== options.responseId)
+                                continue;
+                            if (options.evaluator && evaluation.evaluator !== options.evaluator)
+                                continue;
+                            if (!applyFilters(evaluation))
+                                continue;
+                            results.push(evaluation);
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.evaluationCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
                         }
                     }
                 }
+                const paginated = paginateResults(results, offset, limit);
+                this.evaluationCache.set(cacheKey, paginated);
+                this.circuitBreaker.recordSuccess();
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.evaluationCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
-        }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
-        }
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
+            }
+        });
     }
     async healthCheck() {
         // Check circuit breaker state
@@ -1410,6 +1673,105 @@ export class MultiDirectoryBackend {
         }
         return result;
     }
+    /**
+     * Get aggregated parse statistics from all backends
+     */
+    getParseStats() {
+        // Aggregate stats from all backends
+        let totalFiles = 0;
+        let totalLines = 0;
+        let totalParsed = 0;
+        let totalSkipped = 0;
+        const allWorstFiles = [];
+        for (const backend of this.backends) {
+            const stats = backend.getParseStats();
+            totalFiles += stats.totalFiles;
+            totalLines += stats.totalLines;
+            totalParsed += stats.totalParsed;
+            totalSkipped += stats.totalSkipped;
+            // Collect worst files with skip rates
+            for (const file of stats.worstFiles) {
+                const totalProcessed = file.parsedLines + file.skippedLines;
+                if (totalProcessed > 0) {
+                    allWorstFiles.push({
+                        ...file,
+                        skipRate: file.skippedLines / totalProcessed,
+                    });
+                }
+            }
+        }
+        // Calculate overall success rate
+        const totalProcessed = totalParsed + totalSkipped;
+        const parseSuccessRate = totalProcessed > 0 ? totalParsed / totalProcessed : 1;
+        // Get top worst files across all backends
+        const worstFiles = allWorstFiles
+            .sort((a, b) => b.skipRate - a.skipRate)
+            .slice(0, WORST_FILES_LIMIT)
+            .map(({ skipRate: _, ...f }) => f);
+        return {
+            totalFiles,
+            totalLines,
+            totalParsed,
+            totalSkipped,
+            parseSuccessRate,
+            worstFiles,
+        };
+    }
+    /**
+     * Get aggregated query latency statistics from all backends
+     */
+    getQueryStats() {
+        // Aggregate histogram data from all backends
+        // For percentiles, we use the first backend's data since aggregating
+        // percentiles across backends requires full data merge
+        // Count and sum can be aggregated directly
+        const types = ['traces', 'logs', 'metrics', 'llmEvents', 'evaluations'];
+        const result = {};
+        for (const type of types) {
+            let totalCount = 0;
+            let weightedP50 = 0;
+            let weightedP95 = 0;
+            let weightedP99 = 0;
+            let minValue = Infinity;
+            let maxValue = -Infinity;
+            let totalSum = 0;
+            for (const backend of this.backends) {
+                const stats = backend.getQueryStats()[type];
+                if (stats.count > 0) {
+                    totalCount += stats.count;
+                    totalSum += stats.mean * stats.count;
+                    weightedP50 += stats.p50 * stats.count;
+                    weightedP95 += stats.p95 * stats.count;
+                    weightedP99 += stats.p99 * stats.count;
+                    minValue = Math.min(minValue, stats.min);
+                    maxValue = Math.max(maxValue, stats.max);
+                }
+            }
+            if (totalCount > 0) {
+                result[type] = {
+                    min: minValue,
+                    max: maxValue,
+                    mean: totalSum / totalCount,
+                    p50: weightedP50 / totalCount,
+                    p95: weightedP95 / totalCount,
+                    p99: weightedP99 / totalCount,
+                    count: totalCount,
+                };
+            }
+            else {
+                result[type] = {
+                    min: 0,
+                    max: 0,
+                    mean: 0,
+                    p50: 0,
+                    p95: 0,
+                    p99: 0,
+                    count: 0,
+                };
+            }
+        }
+        return result;
+    }
     /**
      * Export traces in OTLP JSON format
      */