observability-toolkit 1.8.5 → 2.1.0

package/dist/backends/local-jsonl.js

@@ -1,24 +1,38 @@
 /**
  * Local JSONL backend for reading telemetry data from ~/.claude/telemetry/
  *
- * The local telemetry files use a flat JSONL format where each line is a complete
- * span or log record, not the batched OpenTelemetry export format.
+ * Supports two JSONL line formats:
+ * - **Flat**: one span/log/metric per line with embedded resource (original hooks format)
+ * - **OTLP envelope**: one ResourceSpans/ResourceLogs/ResourceMetrics envelope per line
+ *   (as produced by the L5 batch migration script or future OTLP writers)
+ *
+ * Format detection is automatic via the unwrap adapters (A4 items 1-6).
  */
 import { join } from 'path';
-import { GENAI_EVALUATION_ATTRIBUTES, } from './index.js';
-import { convertToOTLPTraces, convertToOTLPLogs, convertToOTLPMetrics, } from '../lib/otlp-export.js';
-import { TELEMETRY_DIR, getTelemetryDirectories, getSpanKind, getStatusCodeName } from '../lib/constants.js';
-import { listFiles, streamJsonl, parseDateFromFilename, getDateString, paginateResults, hasReachedLimit, } from '../lib/file-utils.js';
-import { QueryCache, makeCacheKey } from '../lib/cache.js';
-import { getIndexPath, readIndex, isIndexStale, queryIndex, readLinesByNumber, } from '../lib/indexer.js';
-import { sanitizePath } from '../lib/error-sanitizer.js';
-import { CircuitBreaker } from '../lib/circuit-breaker.js';
+import { datasetRecordSchema, datasetTraceRecordSchema, datasetRunRecordSchema, rawEvaluatorTypeSchema, } from '../lib/validation/api-schemas.js';
+import { homedir } from 'os';
+import { GENAI_EVALUATION_ATTRIBUTES, GENAI_RESPONSE_ATTRIBUTES, GENAI_AGENT_ATTRIBUTES, AGENT_JUDGE_ATTRIBUTES, AGGREGATION_TEMPORALITY, } from './index.js';
+import { convertToOTLPTraces, convertToOTLPLogs, convertToOTLPMetrics, } from '../lib/exports/otlp-export.js';
+import { DEFAULT_QUERY_RESULT_LIMIT, DATASET_TRACE_TRUNCATION_LIMIT, MAX_STEP_SCORES, MAX_TOOL_VERIFICATIONS, HOOKS_TELEMETRY_DIR, WORST_FILES_LIMIT, getOtelLogSeverityNumber, getSpanKind, getStatusCodeName, getTelemetryDirectories, normalizeOtelLogSeverity } from '../lib/core/constants.js';
+import { ONE_MILLION, NANOSECONDS_PER_SECOND, NANOSECONDS_PER_SECOND_BIGINT, TIME_MS, PERCENTILE, PERCENT_MULTIPLIER, } from '../lib/core/units.js';
+import { listFiles, streamJsonl, parseDateFromFilename, getDateString, paginateResults, hasReachedLimit, } from '../lib/core/file-utils.js';
+import { QueryCache, makeCacheKey, CACHE_TTL_MS, DEFAULT_CACHE_SIZE } from '../lib/resilience/cache.js';
+import { recordQueryDuration, recordCacheHit } from '../lib/observability/metrics.js';
+import { Histogram } from '../lib/observability/histogram.js';
+import { getIndexPath, readIndex, isIndexStale, queryIndex, readLinesByNumber, } from '../lib/observability/indexer.js';
+import { sanitizePath } from '../lib/errors/error-sanitizer.js';
+import { CircuitBreaker } from '../lib/resilience/circuit-breaker.js';
+import { withSpan } from '../lib/observability/instrumentation.js';
 import { existsSync } from 'fs';
+import { appendFile, mkdir, readFile, writeFile, rename } from 'fs/promises';
+import { randomUUID } from 'crypto';
+import { ParseStatsTracker } from '../lib/observability/parse-stats.js';
+import { createLogger } from '../lib/core/logger.js';
+const queryLog = createLogger('query');
+const configLog = createLogger('config');
 function startTiming() {
     const start = performance.now();
-    return {
-        end: () => performance.now() - start
-    };
+    return { end: () => performance.now() - start };
 }
 const SLOW_QUERY_THRESHOLD_MS = 500;
 /**
@@ -29,29 +43,39 @@ const regexCache = new Map();
 const MAX_CACHED_REGEX_PATTERNS = 100;
 /**
  * Get a cached regex or compile and cache a new one.
+ * Accepts an optional pre-compiled RegExp to seed the cache on first miss (CR-L3).
  * Returns null for invalid patterns (logs warning).
  */
-function getCachedRegex(pattern) {
+function getCachedRegex(pattern, precompiled) {
     const cached = regexCache.get(pattern);
     if (cached) {
+        // Move to end of Map for proper LRU ordering
+        regexCache.delete(pattern);
+        regexCache.set(pattern, cached);
         return cached;
     }
-    try {
-        const regex = new RegExp(pattern);
-        // LRU eviction: remove oldest entry if at capacity
-        if (regexCache.size >= MAX_CACHED_REGEX_PATTERNS) {
-            const firstKey = regexCache.keys().next().value;
-            if (firstKey !== undefined) {
-                regexCache.delete(firstKey);
-            }
+    // Use pre-compiled regex if provided (avoids double-compilation when validator returns RegExp)
+    const regex = precompiled ?? (() => {
+        try {
+            return new RegExp(pattern);
         }
-        regexCache.set(pattern, regex);
-        return regex;
-    }
-    catch {
-        console.warn(`Invalid regex pattern: ${pattern}`);
+        catch {
+            const truncatedPattern = pattern.length > PERCENTILE.P50 ? pattern.slice(0, PERCENTILE.P50) + '...' : pattern;
+            configLog.warn(`Invalid regex pattern: ${truncatedPattern.replace(/[\n\r]/g, ' ')}`);
+            return null;
+        }
+    })();
+    if (!regex)
         return null;
+    // LRU eviction: remove oldest entry if at capacity
+    if (regexCache.size >= MAX_CACHED_REGEX_PATTERNS) {
+        const firstKey = regexCache.keys().next().value;
+        if (firstKey !== undefined) {
+            regexCache.delete(firstKey);
+        }
     }
+    regexCache.set(pattern, regex);
+    return regex;
 }
 /**
  * Insert item into a sorted array, maintaining sort order and max size.
@@ -132,11 +156,11 @@ function parseTimeBucket(bucket) {
     const unit = match[2];
     switch (unit) {
         case 'm':
-            return value * 60 * 1000; // minutes to ms
+            return value * TIME_MS.MINUTE; // minutes to ms
         case 'h':
-            return value * 60 * 60 * 1000; // hours to ms
+            return value * TIME_MS.HOUR; // hours to ms
         case 'd':
-            return value * 24 * 60 * 60 * 1000; // days to ms
+            return value * TIME_MS.DAY; // days to ms
         default:
             return null;
     }
@@ -148,6 +172,325 @@ function floorToBucket(timestamp, bucketMs) {
     const ts = new Date(timestamp).getTime();
     return Math.floor(ts / bucketMs) * bucketMs;
 }
+// ---------------------------------------------------------------------------
+// OTLP envelope unwrap adapters (A4 items 1-6)
+//
+// When the L5 batch migration script converts flat JSONL to OTLP format,
+// each JSONL line becomes an OTLP envelope wrapping one or more records.
+// These adapters detect envelope lines and unwrap them to Flat* shapes
+// so the existing normalize* functions can process them unchanged.
+// ---------------------------------------------------------------------------
+/** Reserved keys that must not be set on plain objects to prevent prototype pollution */
+const RESERVED_KEYS = new Set(['__proto__', 'constructor', 'prototype']);
+/** Convert OTLP KeyValue array to flat attributes record (null-prototype to prevent pollution) */
+function fromOTLPAttributes(kvs) {
+    if (!kvs)
+        return Object.create(null);
+    const out = Object.create(null);
+    for (const kv of kvs) {
+        if (RESERVED_KEYS.has(kv.key))
+            continue;
+        out[kv.key] = fromOTLPValue(kv.value);
+    }
+    return out;
+}
+/** Convert a single OTLP AnyValue to a JS primitive/object */
+function fromOTLPValue(v) {
+    if (!v)
+        return undefined;
+    if (v.stringValue !== undefined)
+        return v.stringValue;
+    if (v.intValue !== undefined) {
+        const n = Number(v.intValue);
+        return Number.isFinite(n) ? n : undefined;
+    }
+    if (v.doubleValue !== undefined)
+        return v.doubleValue;
+    if (v.boolValue !== undefined)
+        return v.boolValue;
+    if (v.arrayValue)
+        return v.arrayValue.values.map(fromOTLPValue);
+    if (v.kvlistValue)
+        return fromOTLPAttributes(v.kvlistValue.values);
+    return undefined;
+}
+/** Convert OTLP nanosecond string to [seconds, nanoseconds] tuple. Returns undefined on invalid input. */
+function nanoStrToTuple(ns) {
+    if (!ns)
+        return undefined;
+    try {
+        const n = BigInt(ns);
+        return [
+            Number(n / NANOSECONDS_PER_SECOND_BIGINT),
+            Number(n % NANOSECONDS_PER_SECOND_BIGINT),
+        ];
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Convert OTLP nanosecond string to epoch milliseconds. Returns undefined on invalid input. */
+function nanoStrToMs(ns) {
+    if (!ns)
+        return undefined;
+    try {
+        return Number(BigInt(ns) / BigInt(ONE_MILLION));
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Convert OTLP nanosecond string to ISO-8601 timestamp. Returns undefined on invalid input. */
+function nanoStrToISO(ns) {
+    const ms = nanoStrToMs(ns);
+    return ms !== undefined ? new Date(ms).toISOString() : undefined;
+}
+/** Extract numeric value from OTLP data point (asDouble preferred, asInt fallback with precision guard). */
+function dataPointValue(dp) {
+    if (dp.asDouble !== undefined) {
+        const d = Number(dp.asDouble);
+        return Number.isFinite(d) ? d : 0;
+    }
+    const n = Number(dp.asInt ?? '0');
+    return Number.isFinite(n) ? n : 0;
+}
+/**
+ * Extract service.name and service.version from OTLP resource attributes.
+ * Only these two fields are mapped; all other resource attributes (host.name,
+ * process.pid, telemetry.sdk.*, etc.) are intentionally dropped because flat
+ * records carry them in span-level attributes and the normalize* functions
+ * already handle that shape.
+ */
+function extractResource(attrs) {
+    if (!attrs)
+        return undefined;
+    let serviceName;
+    let serviceVersion;
+    for (const kv of attrs) {
+        if (kv.key === 'service.name' && kv.value?.stringValue)
+            serviceName = kv.value.stringValue;
+        if (kv.key === 'service.version' && kv.value?.stringValue)
+            serviceVersion = kv.value.stringValue;
+        if (serviceName && serviceVersion)
+            break;
+    }
+    return (serviceName || serviceVersion) ? { serviceName, serviceVersion } : undefined;
+}
+/**
+ * Detect and unwrap an OTLP trace envelope line to FlatSpan[].
+ * Returns null if the record is not an OTLP envelope (i.e., it's a flat span).
+ */
+function unwrapOTLPSpans(record) {
+    if (!record || typeof record !== 'object' || !('resourceSpans' in record))
+        return null;
+    const envelope = record;
+    const results = [];
+    for (const rs of envelope.resourceSpans) {
+        const resource = extractResource(rs.resource?.attributes);
+        for (const ss of rs.scopeSpans ?? []) {
+            const instrumentationScope = ss.scope
+                ? { name: ss.scope.name, version: ss.scope.version }
+                : undefined;
+            for (const span of ss.spans ?? []) {
+                const attrs = fromOTLPAttributes(span.attributes);
+                // Merge resource attributes into span attributes (service.name, etc.)
+                if (resource?.serviceName)
+                    attrs['service.name'] = resource.serviceName;
+                if (resource?.serviceVersion)
+                    attrs['service.version'] = resource.serviceVersion;
+                results.push({
+                    traceId: span.traceId,
+                    spanId: span.spanId,
+                    parentSpanId: span.parentSpanId,
+                    name: span.name,
+                    kind: span.kind,
+                    startTime: nanoStrToTuple(span.startTimeUnixNano),
+                    endTime: nanoStrToTuple(span.endTimeUnixNano),
+                    status: span.status,
+                    attributes: attrs,
+                    events: span.events?.map(e => ({
+                        name: e.name,
+                        timestamp: nanoStrToMs(e.timeUnixNano),
+                        attributes: fromOTLPAttributes(e.attributes),
+                    })),
+                    links: span.links?.map(l => ({
+                        context: { traceId: l.traceId, spanId: l.spanId },
+                        attributes: fromOTLPAttributes(l.attributes),
+                    })),
+                    resource,
+                    instrumentationScope,
+                });
+            }
+        }
+    }
+    return results;
+}
+/**
+ * Detect and unwrap an OTLP log envelope line to FlatLog[].
+ * Returns null if the record is not an OTLP envelope.
+ */
+function unwrapOTLPLogs(record) {
+    if (!record || typeof record !== 'object' || !('resourceLogs' in record))
+        return null;
+    const envelope = record;
+    const results = [];
+    for (const rl of envelope.resourceLogs) {
+        const resource = extractResource(rl.resource?.attributes);
+        for (const sl of rl.scopeLogs ?? []) {
+            const instrumentationScope = sl.scope
+                ? { name: sl.scope.name, version: sl.scope.version }
+                : undefined;
+            for (const log of sl.logRecords ?? []) {
+                const attrs = fromOTLPAttributes(log.attributes);
+                const bodyValue = log.body;
+                results.push({
+                    timestamp: nanoStrToISO(log.timeUnixNano) ?? '',
+                    severityText: log.severityText,
+                    body: bodyValue?.stringValue ?? '',
+                    traceId: log.traceId,
+                    spanId: log.spanId,
+                    attributes: attrs,
+                    resource: resource ? { serviceName: resource.serviceName } : undefined,
+                    instrumentationScope,
+                });
+            }
+        }
+    }
+    return results;
+}
+/**
+ * Detect and unwrap an OTLP metric envelope line to FlatMetric[].
+ * Returns null if the record is not an OTLP envelope.
+ */
+function unwrapOTLPMetrics(record) {
+    if (!record || typeof record !== 'object' || !('resourceMetrics' in record))
+        return null;
+    const envelope = record;
+    const results = [];
+    for (const rm of envelope.resourceMetrics) {
+        const resource = extractResource(rm.resource?.attributes);
+        for (const sm of rm.scopeMetrics ?? []) {
+            for (const metric of sm.metrics ?? []) {
+                const name = metric.name;
+                const unit = metric.unit;
+                // Determine metric type and extract data points
+                const gauge = metric.gauge;
+                const sum = metric.sum;
+                const histogram = metric.histogram;
+                if (histogram?.dataPoints) {
+                    for (const dp of histogram.dataPoints) {
+                        const attrs = fromOTLPAttributes(dp.attributes);
+                        const bucketCounts = dp.bucketCounts ?? [];
+                        const explicitBounds = dp.explicitBounds ?? [];
+                        // Convert OTLP per-bucket counts + explicitBounds to cumulative {le, count}
+                        let cumulative = 0;
+                        const buckets = [];
+                        for (let i = 0; i < bucketCounts.length; i++) {
+                            cumulative += Number(bucketCounts[i]);
+                            buckets.push({ le: i < explicitBounds.length ? explicitBounds[i] : Infinity, count: cumulative });
+                        }
+                        const histSum = dp.sum ?? 0;
+                        results.push({
+                            timestamp: nanoStrToISO(dp.timeUnixNano) ?? '',
+                            name,
+                            value: histSum,
+                            type: 'histogram',
+                            unit,
+                            attributes: attrs,
+                            resource: resource ? { serviceName: resource.serviceName } : undefined,
+                            histogram: { buckets, sum: histSum, count: Number(dp.count ?? 0) },
+                            aggregationTemporality: histogram.aggregationTemporality,
+                        });
+                    }
+                }
+                else if (sum?.dataPoints) {
+                    for (const dp of sum.dataPoints) {
+                        const attrs = fromOTLPAttributes(dp.attributes);
+                        results.push({
+                            timestamp: nanoStrToISO(dp.timeUnixNano) ?? '',
+                            name,
+                            value: dataPointValue(dp),
+                            type: 'counter',
+                            unit,
+                            attributes: attrs,
+                            resource: resource ? { serviceName: resource.serviceName } : undefined,
+                            aggregationTemporality: sum.aggregationTemporality,
+                        });
+                    }
+                }
+                else if (gauge?.dataPoints) {
+                    for (const dp of gauge.dataPoints) {
+                        const attrs = fromOTLPAttributes(dp.attributes);
+                        results.push({
+                            timestamp: nanoStrToISO(dp.timeUnixNano) ?? '',
+                            name,
+                            value: dataPointValue(dp),
+                            type: 'gauge',
+                            unit,
+                            attributes: attrs,
+                            resource: resource ? { serviceName: resource.serviceName } : undefined,
+                        });
+                    }
+                }
+            }
+        }
+    }
+    return results;
+}
+/**
+ * Detect and unwrap an OTLP trace envelope containing evaluation events.
+ * Evaluations in OTLP are span events with name "gen_ai.evaluation.result".
+ * Returns null if the record is not an OTLP envelope.
+ */
+function unwrapOTLPEvaluations(record) {
+    if (!record || typeof record !== 'object' || !('resourceSpans' in record))
+        return null;
+    const envelope = record;
+    const results = [];
+    for (const rs of envelope.resourceSpans) {
+        for (const ss of rs.scopeSpans ?? []) {
+            for (const span of ss.spans ?? []) {
+                for (const event of span.events ?? []) {
+                    if (event.name !== 'gen_ai.evaluation.result')
+                        continue;
+                    const attrs = fromOTLPAttributes(event.attributes);
+                    results.push({
+                        timestamp: nanoStrToISO(event.timeUnixNano) ?? '',
+                        name: event.name,
+                        attributes: attrs,
+                        traceId: span.traceId,
+                        spanId: span.spanId,
+                    });
+                }
+            }
+        }
+    }
+    // Return null (not []) when envelope contains no evaluation events,
+    // so flattenEvaluationRecord falls through to the passthrough path.
+    // This prevents a flat record that coincidentally has a 'resourceSpans'
+    // key from being silently swallowed.
+    return results.length > 0 ? results : null;
+}
+/** Unwrap-or-passthrough for trace records. OTLP envelopes yield multiple spans; flat records yield one. */
+function flattenSpanRecord(raw) {
+    const unwrapped = unwrapOTLPSpans(raw);
+    return unwrapped ?? [raw];
+}
+/** Unwrap-or-passthrough for log records */
+function flattenLogRecord(raw) {
+    const unwrapped = unwrapOTLPLogs(raw);
+    return unwrapped ?? [raw];
+}
+/** Unwrap-or-passthrough for metric records */
+function flattenMetricRecord(raw) {
+    const unwrapped = unwrapOTLPMetrics(raw);
+    return unwrapped ?? [raw];
+}
+/** Unwrap-or-passthrough for evaluation records */
+function flattenEvaluationRecord(raw) {
+    const unwrapped = unwrapOTLPEvaluations(raw);
+    return unwrapped ?? [raw];
+}
 /**
  * Extract a field from an object using dot notation path.
  * Returns undefined if the path doesn't exist or the object is not traversable.
@@ -156,7 +499,7 @@ function extractField(obj, path) {
     const parts = path.split('.');
     let current = obj;
     for (const part of parts) {
-        if (current == null || typeof current !== 'object')
+        if (current === null || typeof current !== 'object')
             return undefined;
         current = current[part];
     }
@@ -201,21 +544,95 @@ function extractValidNumber(value) {
         return undefined;
     return Number.isFinite(value) ? value : undefined;
 }
-/** Valid evaluator types per our schema */
-const VALID_EVALUATOR_TYPES = ['llm', 'human', 'rule', 'classifier'];
 /**
- * Extract and validate evaluatorType
+ * Extract and validate evaluatorType using schema-based normalization (NARROW-11).
  */
 function extractValidEvaluatorType(value) {
     if (typeof value !== 'string')
         return undefined;
-    const trimmed = value.trim().toLowerCase();
-    return VALID_EVALUATOR_TYPES.includes(trimmed) ? trimmed : undefined;
+    const result = rawEvaluatorTypeSchema.safeParse(value);
+    return result.success ? result.data : undefined;
 }
 /**
  * Convert flat evaluation to normalized EvaluationResult
  * Includes runtime validation for data integrity (P0-2 fix)
  */
+/**
+ * Safely extract StepScore array from raw attribute value.
+ * Returns undefined if not a valid array or if parsing fails.
+ */
+function extractStepScores(value) {
+    if (!value)
+        return undefined;
+    // If it's a string, try to parse as JSON
+    let arr;
+    if (typeof value === 'string') {
+        try {
+            arr = JSON.parse(value);
+            if (!Array.isArray(arr))
+                return undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    else if (Array.isArray(value)) {
+        arr = value;
+    }
+    else {
+        return undefined;
+    }
+    // Validate each element and filter invalid entries
+    return arr
+        .filter((item) => {
+        if (!item || typeof item !== 'object')
+            return false;
+        const obj = item;
+        return ((typeof obj.step === 'string' || typeof obj.step === 'number') &&
+            typeof obj.score === 'number' &&
+            Number.isFinite(obj.score));
+    })
+        .slice(0, MAX_STEP_SCORES); // Respect MAX_STEP_SCORES limit
+}
+/**
+ * Safely extract ToolVerification array from raw attribute value.
+ * Returns undefined if not a valid array or if parsing fails.
+ */
+function extractToolVerifications(value) {
+    if (!value)
+        return undefined;
+    // If it's a string, try to parse as JSON
+    let arr;
+    if (typeof value === 'string') {
+        try {
+            arr = JSON.parse(value);
+            if (!Array.isArray(arr))
+                return undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    else if (Array.isArray(value)) {
+        arr = value;
+    }
+    else {
+        return undefined;
+    }
+    // Validate each element and filter invalid entries
+    return arr
+        .filter((item) => {
+        if (!item || typeof item !== 'object')
+            return false;
+        const obj = item;
+        return (typeof obj.toolName === 'string' &&
+            typeof obj.toolCorrect === 'boolean' &&
+            typeof obj.argsCorrect === 'boolean' &&
+            typeof obj.score === 'number' &&
+            Number.isFinite(obj.score));
+    })
+        .slice(0, MAX_TOOL_VERIFICATIONS);
+}
 function normalizeEvaluation(raw) {
     const attrs = raw.attributes;
     // evaluationName is required per OTel spec - must be non-empty string
@@ -235,24 +652,18 @@ function normalizeEvaluation(raw) {
         // Evaluator fields (custom extension)
         evaluator: extractValidString(attrs?.[GENAI_EVALUATION_ATTRIBUTES.EVALUATOR]),
         evaluatorType: extractValidEvaluatorType(attrs?.[GENAI_EVALUATION_ATTRIBUTES.EVALUATOR_TYPE]),
-        responseId: extractValidString(attrs?.['gen_ai.response.id']),
+        responseId: extractValidString(attrs?.[GENAI_RESPONSE_ATTRIBUTES.ID]),
         traceId: raw.traceId,
         spanId: raw.spanId,
         sessionId: extractValidString(attrs?.['session.id']),
+        // Agent-as-Judge fields (Section 10.7)
+        agentId: extractValidString(attrs?.[GENAI_AGENT_ATTRIBUTES.AGENT_ID]),
+        agentName: extractValidString(attrs?.[GENAI_AGENT_ATTRIBUTES.AGENT_NAME]),
+        stepScores: extractStepScores(attrs?.[AGENT_JUDGE_ATTRIBUTES.STEP_SCORES]),
+        toolVerifications: extractToolVerifications(attrs?.[AGENT_JUDGE_ATTRIBUTES.TOOL_VERIFICATIONS]),
+        trajectoryLength: extractValidNumber(attrs?.[AGENT_JUDGE_ATTRIBUTES.TRAJECTORY_LENGTH]),
     };
 }
-/**
- * OTel-compliant severity number mapping
- * https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
- */
-const SEVERITY_MAP = {
-    TRACE: 1,
-    DEBUG: 5,
-    INFO: 9,
-    WARN: 13,
-    ERROR: 17,
-    FATAL: 21,
-};
 /**
  * Convert flat span to normalized TraceSpan
  */
@@ -261,18 +672,18 @@ function normalizeSpan(raw) {
         return null;
     // Convert [seconds, nanoseconds] to nanoseconds
     const startTimeUnixNano = raw.startTime
-        ? raw.startTime[0] * 1_000_000_000 + raw.startTime[1]
+        ? raw.startTime[0] * NANOSECONDS_PER_SECOND + raw.startTime[1]
         : 0;
     const endTimeUnixNano = raw.endTime
-        ? raw.endTime[0] * 1_000_000_000 + raw.endTime[1]
+        ? raw.endTime[0] * NANOSECONDS_PER_SECOND + raw.endTime[1]
         : undefined;
     // Calculate duration in ms
     let durationMs;
     if (raw.duration) {
-        durationMs = (raw.duration[0] * 1000) + (raw.duration[1] / 1_000_000);
+        durationMs = (raw.duration[0] * TIME_MS.SECOND) + (raw.duration[1] / ONE_MILLION);
     }
     else if (endTimeUnixNano) {
-        durationMs = (endTimeUnixNano - startTimeUnixNano) / 1_000_000;
+        durationMs = (endTimeUnixNano - startTimeUnixNano) / ONE_MILLION;
     }
     // Add service name to attributes if present in resource
     const attributes = { ...raw.attributes };
@@ -324,14 +735,15 @@ function normalizeLog(raw, extractFields) {
     // Handle timestamp: either ISO string or [seconds, nanoseconds] array
     let timestamp;
     if (Array.isArray(raw.timestamp)) {
-        const ms = raw.timestamp[0] * 1000 + Math.floor(raw.timestamp[1] / 1_000_000);
+        const ms = raw.timestamp[0] * TIME_MS.SECOND + Math.floor(raw.timestamp[1] / ONE_MILLION);
         timestamp = new Date(ms).toISOString();
     }
     else {
         timestamp = raw.timestamp;
     }
-    const severity = raw.severityText || raw.severity || 'INFO';
-    const severityNumber = SEVERITY_MAP[severity.toUpperCase()];
+    const rawSeverity = raw.severityText || raw.severity || 'INFO';
+    const severity = normalizeOtelLogSeverity(rawSeverity) ?? rawSeverity;
+    const severityNumber = getOtelLogSeverityNumber(severity);
     const body = raw.body || '';
     // Extract fields from JSON body if requested
     let extractedFields;
@@ -368,11 +780,11 @@ function normalizeAggregationTemporality(value) {
     }
     // Handle numeric values (OTel spec)
     switch (value) {
-        case 1:
+        case AGGREGATION_TEMPORALITY.DELTA:
             return 'DELTA';
-        case 2:
+        case AGGREGATION_TEMPORALITY.CUMULATIVE:
             return 'CUMULATIVE';
-        case 0:
+        case AGGREGATION_TEMPORALITY.UNSPECIFIED:
         default:
             return 'UNSPECIFIED';
     }
@@ -386,7 +798,7 @@ function normalizeExemplarTimestamp(ts, metricTimestamp) {
     if (typeof ts === 'string')
         return ts;
     // Convert [seconds, nanoseconds] to ISO string
-    const ms = ts[0] * 1000 + Math.floor(ts[1] / 1_000_000);
+    const ms = ts[0] * TIME_MS.SECOND + Math.floor(ts[1] / ONE_MILLION);
     return new Date(ms).toISOString();
 }
 /**
@@ -434,10 +846,12 @@ function normalizeMetric(raw) {
     };
 }
 /**
- * Get files within date range
+ * Get files within date range.
+ * allowedBases constrains which directories listFiles may access (CR-H2).
+ * Callers should pass the telemetryDir as a base so test-provided directories are allowed.
  */
-function getFilesInRange(dir, pattern, startDate, endDate) {
-    const files = listFiles(dir, pattern);
+function getFilesInRange(dir, pattern, startDate, endDate, allowedBases) {
+    const files = listFiles(dir, pattern, allowedBases);
     if (!startDate && !endDate) {
         // Default to today only
         const today = getDateString();
@@ -457,20 +871,47 @@ function getFilesInRange(dir, pattern, startDate, endDate) {
         return true;
     });
 }
+function encodeCursor(payload) {
+    return Buffer.from(JSON.stringify(payload)).toString('base64url');
+}
+function decodeCursor(cursor) {
+    try {
+        return JSON.parse(Buffer.from(cursor, 'base64url').toString());
+    }
+    catch {
+        return null;
+    }
+}
 export class LocalJsonlBackend {
     name = 'local-jsonl';
     telemetryDir;
-    traceCache = new QueryCache();
-    logCache = new QueryCache();
-    metricCache = new QueryCache();
-    llmEventCache = new QueryCache();
-    evaluationCache = new QueryCache();
+    /** Security: directories listFiles is allowed to access (CR-H2) */
+    allowedBases;
+    traceCache = new QueryCache(DEFAULT_CACHE_SIZE, CACHE_TTL_MS, 'traces');
+    logCache = new QueryCache(DEFAULT_CACHE_SIZE, CACHE_TTL_MS, 'logs');
+    metricCache = new QueryCache(DEFAULT_CACHE_SIZE, CACHE_TTL_MS, 'metrics');
+    llmEventCache = new QueryCache(DEFAULT_CACHE_SIZE, CACHE_TTL_MS, 'llmEvents');
+    evaluationCache = new QueryCache(DEFAULT_CACHE_SIZE, CACHE_TTL_MS, 'evaluations');
     useIndexes;
     circuitBreaker;
+    parseStatsTracker;
+    /** Count of trace lines that expanded to multiple spans (OTLP envelopes), across both indexed and full-scan paths */
+    otlpTraceExpansions = 0;
+    /** Histograms for tracking query latency per type */
+    queryHistograms = {
+        traces: new Histogram(),
+        logs: new Histogram(),
+        metrics: new Histogram(),
+        llmEvents: new Histogram(),
+        evaluations: new Histogram(),
+    };
     constructor(telemetryDir, useIndexes = true) {
-        this.telemetryDir = telemetryDir || TELEMETRY_DIR;
+        this.telemetryDir = telemetryDir || HOOKS_TELEMETRY_DIR;
         this.useIndexes = useIndexes;
         this.circuitBreaker = new CircuitBreaker({ name: 'local-file-io' });
+        this.parseStatsTracker = new ParseStatsTracker();
+        // Allow telemetryDir itself so custom dirs (including test temp dirs) are permitted (CR-H2)
+        this.allowedBases = [homedir(), process.cwd(), this.telemetryDir];
     }
     /**
      * Get circuit breaker state (for health check and testing)
@@ -506,6 +947,45 @@ export class LocalJsonlBackend {
             evaluations: this.evaluationCache.getStats(),
         };
     }
+    /**
+     * Get parse statistics for JSONL file processing.
+     * Returns aggregate stats including success rate and worst files.
+     */
+    getParseStats() {
+        return {
+            ...this.parseStatsTracker.getAggregateStats(),
+            otlpTraceExpansions: this.otlpTraceExpansions,
+        };
+    }
+    /**
+     * Reset parse statistics (useful for testing)
+     */
+    resetParseStats() {
+        this.parseStatsTracker.reset();
+        this.otlpTraceExpansions = 0;
+    }
+    /**
+     * Get query latency statistics per query type
+     */
+    getQueryStats() {
+        return {
+            traces: this.queryHistograms.traces.getStats(),
+            logs: this.queryHistograms.logs.getStats(),
+            metrics: this.queryHistograms.metrics.getStats(),
+            llmEvents: this.queryHistograms.llmEvents.getStats(),
+            evaluations: this.queryHistograms.evaluations.getStats(),
+        };
+    }
+    /**
+     * Reset query latency histograms (useful for testing)
+     */
+    resetQueryStats() {
+        this.queryHistograms.traces.reset();
+        this.queryHistograms.logs.reset();
+        this.queryHistograms.metrics.reset();
+        this.queryHistograms.llmEvents.reset();
+        this.queryHistograms.evaluations.reset();
+    }
     /**
      * Try to use an index for a file, returning matching line numbers or null if full scan needed
      */
@@ -521,365 +1001,474 @@ export class LocalJsonlBackend {
         return queryIndex(index, indexOptions);
     }
     async queryTraces(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryTraces took ${durationMs.toFixed(1)}ms`);
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.traces', () => ({
+            'obs_toolkit.query.type': 'traces',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? DEFAULT_QUERY_RESULT_LIMIT,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('traces', durationMs, 'local');
+                this.queryHistograms.traces.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    queryLog.warn(`Slow query: queryTraces took ${durationMs.toFixed(1)}ms`, { durationMs });
+                }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
-            return [];
-        }
-        // Check cache first
-        const cacheKey = makeCacheKey('traces', options);
-        const cached = this.traceCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
-        }
-        const files = getFilesInRange(this.telemetryDir, /traces-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Use cached regex to avoid recompilation for frequently-used patterns
-        const spanNameRegex = options.spanNameRegex
-            ? getCachedRegex(options.spanNameRegex)
-            : null;
-        // Build index query options for indexable filters
-        const indexOptions = {
-            traceId: options.traceId,
-            spanName: options.spanName,
-            serviceName: options.serviceName,
-        };
-        // Helper to apply non-indexable filters to a span
-        const applyFilters = (span) => {
-            // Regex filter (not indexable)
-            if (spanNameRegex && !spanNameRegex.test(span.name))
-                return false;
-            if (options.excludeSpanName && span.name.includes(options.excludeSpanName))
-                return false;
-            if (options.minDurationMs && (span.durationMs || 0) < options.minDurationMs)
-                return false;
-            if (options.maxDurationMs && (span.durationMs || Infinity) > options.maxDurationMs)
-                return false;
-            // Apply attribute filter
-            if (options.attributeFilter) {
-                for (const [key, value] of Object.entries(options.attributeFilter)) {
-                    if (span.attributes?.[key] !== value)
-                        return false;
+            // Check cache first
+            const cacheKey = makeCacheKey('traces', options);
+            const cached = this.traceCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('traces');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
                 }
+                return cached;
             }
-            // Apply attributeExists filter
-            if (options.attributeExists) {
-                for (const key of options.attributeExists) {
-                    if (span.attributes?.[key] === undefined)
+            const files = getFilesInRange(this.telemetryDir, /traces-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate, this.allowedBases);
+            const results = [];
+            const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
+            const offset = options.offset || 0;
+            // Use cached regex to avoid recompilation for frequently-used patterns
+            const spanNameRegex = options.spanNameRegex
+                ? getCachedRegex(options.spanNameRegex)
+                : null;
+            // Build index query options for indexable filters
+            const indexOptions = {
+                traceId: options.traceId,
+                spanName: options.spanName,
+                serviceName: options.serviceName,
+            };
+            // Helper to apply non-indexable filters to a span
+            const applyFilters = (s) => {
+                // Regex filter (not indexable)
+                if (spanNameRegex && !spanNameRegex.test(s.name))
+                    return false;
+                if (options.excludeSpanName && s.name.includes(options.excludeSpanName))
+                    return false;
+                if (options.minDurationMs && (s.durationMs || 0) < options.minDurationMs)
+                    return false;
+                if (options.maxDurationMs && (s.durationMs || Infinity) > options.maxDurationMs)
+                    return false;
+                // Apply attribute filter
+                if (options.attributeFilter) {
+                    for (const [key, value] of Object.entries(options.attributeFilter)) {
+                        if (s.attributes?.[key] !== value)
+                            return false;
+                    }
+                }
+                // Apply attributeExists filter
+                if (options.attributeExists) {
+                    for (const key of options.attributeExists) {
+                        if (s.attributes?.[key] === undefined)
+                            return false;
+                    }
+                }
+                // Apply attributeNotExists filter
+                if (options.attributeNotExists) {
+                    for (const key of options.attributeNotExists) {
+                        if (s.attributes?.[key] !== undefined)
+                            return false;
+                    }
+                }
+                // Apply numeric filter conditions
+                if (options.numericFilter && options.numericFilter.length > 0) {
+                    if (!applyNumericFilters(s.attributes, options.numericFilter))
                         return false;
                 }
-            }
-            // Apply attributeNotExists filter
-            if (options.attributeNotExists) {
-                for (const key of options.attributeNotExists) {
-                    if (span.attributes?.[key] !== undefined)
+                // OTel GenAI agent/tool filters
+                if (options.agentId && s.attributes?.['gen_ai.agent.id'] !== options.agentId)
+                    return false;
+                if (options.agentName && s.attributes?.['gen_ai.agent.name'] !== options.agentName)
+                    return false;
+                if (options.agentDescription) {
+                    const desc = s.attributes?.['gen_ai.agent.description'];
+                    if (typeof desc !== 'string' || !desc.toLowerCase().includes(options.agentDescription.toLowerCase()))
                         return false;
                 }
-            }
-            // Apply numeric filter conditions
-            if (options.numericFilter && options.numericFilter.length > 0) {
-                if (!applyNumericFilters(span.attributes, options.numericFilter))
+                if (options.agentVersion) {
+                    const ver = s.attributes?.['gen_ai.agent.version'];
+                    if (typeof ver !== 'string' || !ver.toLowerCase().includes(options.agentVersion.toLowerCase()))
+                        return false;
+                }
+                if (options.toolName && s.attributes?.['gen_ai.tool.name'] !== options.toolName)
                     return false;
-            }
-            // OTel GenAI agent/tool filters
-            if (options.agentId && span.attributes?.['gen_ai.agent.id'] !== options.agentId)
-                return false;
-            if (options.agentName && span.attributes?.['gen_ai.agent.name'] !== options.agentName)
-                return false;
-            if (options.toolName && span.attributes?.['gen_ai.tool.name'] !== options.toolName)
-                return false;
-            if (options.toolCallId && span.attributes?.['gen_ai.tool.call.id'] !== options.toolCallId)
-                return false;
-            if (options.toolType && span.attributes?.['gen_ai.tool.type'] !== options.toolType)
-                return false;
-            if (options.operationName && span.attributes?.['gen_ai.operation.name'] !== options.operationName)
-                return false;
-            return true;
-        };
-        try {
-            for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'traces', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const raw of rawRecords) {
-                        const span = normalizeSpan(raw);
-                        if (!span)
-                            continue;
-                        if (!applyFilters(span))
-                            continue;
-                        results.push(span);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.traceCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
-                        }
-                    }
+                if (options.toolCallId && s.attributes?.['gen_ai.tool.call.id'] !== options.toolCallId)
+                    return false;
+                if (options.toolType && s.attributes?.['gen_ai.tool.type'] !== options.toolType)
+                    return false;
+                if (options.operationName && s.attributes?.['gen_ai.operation.name'] !== options.operationName)
+                    return false;
+                if (options.outputType) {
+                    const ot = s.attributes?.['gen_ai.output.type'];
+                    if (typeof ot !== 'string' || ot.toLowerCase() !== options.outputType.toLowerCase())
+                        return false;
                 }
-                else {
-                    // Fall back to full file scan
-                    for await (const raw of streamJsonl(file)) {
-                        const span = normalizeSpan(raw);
-                        if (!span)
-                            continue;
-                        // Apply indexable filters (since no index was used)
-                        if (options.traceId && span.traceId !== options.traceId)
-                            continue;
-                        if (options.spanName && !span.name.includes(options.spanName))
-                            continue;
-                        if (options.serviceName) {
-                            const svc = span.attributes?.['service.name'];
-                            if (svc !== options.serviceName)
-                                continue;
+                return true;
+            };
+            try {
+                for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'traces', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        // Note: index was built against flat records. For OTLP envelope lines,
+                        // multi-span envelopes may have only one traceId indexed per line.
+                        // The full-scan fallback path handles these correctly.
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const rawLine of rawRecords) {
+                            const expanded = flattenSpanRecord(rawLine);
+                            if (expanded.length > 1)
+                                this.otlpTraceExpansions++;
+                            for (const raw of expanded) {
+                                const traceSpan = normalizeSpan(raw);
+                                if (!traceSpan)
+                                    continue;
+                                if (!applyFilters(traceSpan))
+                                    continue;
+                                results.push(traceSpan);
+                                if (hasReachedLimit(results.length, offset, limit)) {
+                                    const paginated = paginateResults(results, offset, limit);
+                                    this.traceCache.set(cacheKey, paginated);
+                                    this.circuitBreaker.recordSuccess();
+                                    logTiming();
+                                    if (span)
+                                        span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                    return paginated;
+                                }
+                            }
                         }
-                        if (!applyFilters(span))
-                            continue;
-                        results.push(span);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.traceCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+                    }
+                    else {
+                        // Fall back to full file scan
+                        for await (const rawLine of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            const expanded = flattenSpanRecord(rawLine);
+                            if (expanded.length > 1)
+                                this.otlpTraceExpansions++;
+                            for (const raw of expanded) {
+                                const traceSpan = normalizeSpan(raw);
+                                if (!traceSpan)
+                                    continue;
+                                // Apply indexable filters (since no index was used)
+                                if (options.traceId && traceSpan.traceId !== options.traceId)
+                                    continue;
+                                if (options.spanName && !traceSpan.name.includes(options.spanName))
+                                    continue;
+                                if (options.serviceName) {
+                                    const svc = traceSpan.attributes?.['service.name'];
+                                    if (svc !== options.serviceName)
+                                        continue;
+                                }
+                                if (!applyFilters(traceSpan))
+                                    continue;
+                                results.push(traceSpan);
+                                if (hasReachedLimit(results.length, offset, limit)) {
+                                    const paginated = paginateResults(results, offset, limit);
+                                    this.traceCache.set(cacheKey, paginated);
+                                    this.circuitBreaker.recordSuccess();
+                                    logTiming();
+                                    if (span)
+                                        span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                    return paginated;
+                                }
+                            }
                         }
                     }
                 }
+                const paginated = paginateResults(results, offset, limit);
+                this.traceCache.set(cacheKey, paginated);
+                this.circuitBreaker.recordSuccess();
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.traceCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
-        }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
-        }
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
+            }
+        });
     }
     async queryLogs(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryLogs took ${durationMs.toFixed(1)}ms`);
-            }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
-            return [];
-        }
-        // Check cache first
-        const cacheKey = makeCacheKey('logs', options);
-        const cached = this.logCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
-        }
-        const files = getFilesInRange(this.telemetryDir, /logs-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Build index query options for indexable filters
-        const indexOptions = {
-            traceId: options.traceId,
-            severity: options.severity,
-        };
-        // Helper to apply non-indexable filters to a log
-        const applyFilters = (log) => {
-            if (options.search && !log.body.toLowerCase().includes(options.search.toLowerCase()))
-                return false;
-            // Apply boolean search with multiple terms
-            if (options.searchTerms && options.searchTerms.length > 0) {
-                const bodyLower = log.body.toLowerCase();
-                const operator = options.searchOperator || 'AND';
-                if (operator === 'AND') {
-                    const allMatch = options.searchTerms.every(term => bodyLower.includes(term.toLowerCase()));
-                    if (!allMatch)
-                        return false;
-                }
-                else {
-                    const anyMatch = options.searchTerms.some(term => bodyLower.includes(term.toLowerCase()));
-                    if (!anyMatch)
-                        return false;
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.logs', () => ({
+            'obs_toolkit.query.type': 'logs',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? DEFAULT_QUERY_RESULT_LIMIT,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('logs', durationMs, 'local');
+                this.queryHistograms.logs.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    queryLog.warn(`Slow query: queryLogs took ${durationMs.toFixed(1)}ms`, { durationMs });
                 }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-            if (options.excludeSearch && log.body.toLowerCase().includes(options.excludeSearch.toLowerCase()))
-                return false;
-            // Apply attributeExists filter
-            if (options.attributeExists) {
-                for (const key of options.attributeExists) {
-                    if (log.attributes?.[key] === undefined)
-                        return false;
+            // Check cache first
+            const cacheKey = makeCacheKey('logs', options);
+            const cached = this.logCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('logs');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
                 }
+                return cached;
             }
-            // Apply attributeNotExists filter
-            if (options.attributeNotExists) {
-                for (const key of options.attributeNotExists) {
-                    if (log.attributes?.[key] !== undefined)
-                        return false;
+            const files = getFilesInRange(this.telemetryDir, /logs-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate, this.allowedBases);
+            const results = [];
+            const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
+            const offset = options.offset || 0;
+            // Build index query options for indexable filters
+            const indexOptions = {
+                traceId: options.traceId,
+                severity: options.severity,
+            };
+            // Helper to apply non-indexable filters to a log
+            const applyFilters = (logRecord) => {
+                if (options.search && !logRecord.body.toLowerCase().includes(options.search.toLowerCase()))
+                    return false;
+                // Apply boolean search with multiple terms
+                if (options.searchTerms && options.searchTerms.length > 0) {
+                    const bodyLower = logRecord.body.toLowerCase();
+                    const operator = options.searchOperator || 'AND';
+                    if (operator === 'AND') {
+                        const allMatch = options.searchTerms.every(term => bodyLower.includes(term.toLowerCase()));
+                        if (!allMatch)
+                            return false;
+                    }
+                    else {
+                        const anyMatch = options.searchTerms.some(term => bodyLower.includes(term.toLowerCase()));
+                        if (!anyMatch)
+                            return false;
+                    }
                 }
-            }
-            // Apply numeric filter conditions
-            if (options.numericFilter && options.numericFilter.length > 0) {
-                if (!applyNumericFilters(log.attributes, options.numericFilter))
+                if (options.excludeSearch && logRecord.body.toLowerCase().includes(options.excludeSearch.toLowerCase()))
                     return false;
-            }
-            return true;
-        };
-        try {
-            for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'logs', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const raw of rawRecords) {
-                        const log = normalizeLog(raw, options.extractFields);
-                        if (!log)
-                            continue;
-                        if (!applyFilters(log))
-                            continue;
-                        results.push(log);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.logCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
-                        }
+                // Apply attributeExists filter
+                if (options.attributeExists) {
+                    for (const key of options.attributeExists) {
+                        if (logRecord.attributes?.[key] === undefined)
+                            return false;
+                    }
+                }
+                // Apply attributeNotExists filter
+                if (options.attributeNotExists) {
+                    for (const key of options.attributeNotExists) {
+                        if (logRecord.attributes?.[key] !== undefined)
+                            return false;
                     }
                 }
-                else {
-                    // Fall back to full file scan
-                    for await (const raw of streamJsonl(file)) {
-                        const log = normalizeLog(raw, options.extractFields);
-                        if (!log)
-                            continue;
-                        // Apply indexable filters (since no index was used)
-                        if (options.severity && log.severity.toUpperCase() !== options.severity.toUpperCase())
-                            continue;
-                        if (options.traceId && log.traceId !== options.traceId)
-                            continue;
-                        if (!applyFilters(log))
-                            continue;
-                        results.push(log);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.logCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+                // Apply numeric filter conditions
+                if (options.numericFilter && options.numericFilter.length > 0) {
+                    if (!applyNumericFilters(logRecord.attributes, options.numericFilter))
+                        return false;
+                }
+                return true;
+            };
+            try {
+                for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'logs', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const rawLine of rawRecords) {
+                            for (const raw of flattenLogRecord(rawLine)) {
+                                const logRecord = normalizeLog(raw, options.extractFields);
+                                if (!logRecord)
+                                    continue;
+                                if (!applyFilters(logRecord))
+                                    continue;
+                                results.push(logRecord);
+                                if (hasReachedLimit(results.length, offset, limit)) {
+                                    const paginated = paginateResults(results, offset, limit);
+                                    this.logCache.set(cacheKey, paginated);
+                                    this.circuitBreaker.recordSuccess();
+                                    logTiming();
+                                    if (span)
+                                        span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                    return paginated;
+                                }
+                            }
+                        }
+                    }
+                    else {
+                        // Fall back to full file scan
+                        for await (const rawLine of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            for (const raw of flattenLogRecord(rawLine)) {
+                                const logRecord = normalizeLog(raw, options.extractFields);
+                                if (!logRecord)
+                                    continue;
+                                // Apply indexable filters (since no index was used)
+                                if (options.severity && logRecord.severity !== options.severity)
+                                    continue;
+                                if (options.traceId && logRecord.traceId !== options.traceId)
+                                    continue;
+                                if (!applyFilters(logRecord))
+                                    continue;
+                                results.push(logRecord);
+                                if (hasReachedLimit(results.length, offset, limit)) {
+                                    const paginated = paginateResults(results, offset, limit);
+                                    this.logCache.set(cacheKey, paginated);
+                                    this.circuitBreaker.recordSuccess();
+                                    logTiming();
+                                    if (span)
+                                        span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                    return paginated;
+                                }
+                            }
                         }
                     }
                 }
+                const paginated = paginateResults(results, offset, limit);
+                this.logCache.set(cacheKey, paginated);
+                this.circuitBreaker.recordSuccess();
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.logCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
-        }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
-        }
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
+            }
+        });
     }
     async queryMetrics(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryMetrics took ${durationMs.toFixed(1)}ms`);
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.metrics', () => ({
+            'obs_toolkit.query.type': 'metrics',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? DEFAULT_QUERY_RESULT_LIMIT,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('metrics', durationMs, 'local');
+                this.queryHistograms.metrics.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    queryLog.warn(`Slow query: queryMetrics took ${durationMs.toFixed(1)}ms`, { durationMs });
+                }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
-            return [];
-        }
-        // Check cache first
-        const cacheKey = makeCacheKey('metrics', options);
-        const cached = this.metricCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
-        }
-        const files = getFilesInRange(this.telemetryDir, /metrics-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Build index query options for indexable filters
-        const indexOptions = {
-            metricName: options.metricName,
-        };
-        try {
-            outer: for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'metrics', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const raw of rawRecords) {
-                        const point = normalizeMetric(raw);
-                        if (!point)
-                            continue;
-                        results.push(point);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            break outer;
+            // Check cache first
+            const cacheKey = makeCacheKey('metrics', options);
+            const cached = this.metricCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('metrics');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
+                }
+                return cached;
+            }
+            const files = getFilesInRange(this.telemetryDir, /metrics-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate, this.allowedBases);
+            const results = [];
+            const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
+            const offset = options.offset || 0;
+            // Build index query options for indexable filters
+            const indexOptions = {
+                metricName: options.metricName,
+            };
+            try {
+                outer: for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'metrics', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const rawLine of rawRecords) {
+                            for (const raw of flattenMetricRecord(rawLine)) {
+                                const point = normalizeMetric(raw);
+                                if (!point)
+                                    continue;
+                                results.push(point);
+                                if (hasReachedLimit(results.length, offset, limit)) {
+                                    break outer;
+                                }
+                            }
                         }
                     }
-                }
-                else {
-                    // Fall back to full file scan
-                    for await (const raw of streamJsonl(file)) {
-                        const point = normalizeMetric(raw);
-                        if (!point)
-                            continue;
-                        // Apply filters (since no index was used)
-                        if (options.metricName && !point.name.includes(options.metricName))
-                            continue;
-                        results.push(point);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            break outer;
+                    else {
+                        // Fall back to full file scan
+                        for await (const rawLine of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            for (const raw of flattenMetricRecord(rawLine)) {
+                                const point = normalizeMetric(raw);
+                                if (!point)
+                                    continue;
+                                // Apply filters (since no index was used)
+                                if (options.metricName && !point.name.includes(options.metricName))
+                                    continue;
+                                results.push(point);
+                                if (hasReachedLimit(results.length, offset, limit)) {
+                                    break outer;
+                                }
+                            }
                         }
                     }
                 }
-            }
-            // Apply aggregation if requested
-            if (options.aggregation && results.length > 0) {
-                const aggregated = this.aggregate(results, options.aggregation, options.groupBy, options.timeBucket);
-                this.metricCache.set(cacheKey, aggregated);
+                // Apply aggregation if requested
+                if (options.aggregation && results.length > 0) {
+                    const aggregated = this.aggregate(results, options.aggregation, options.groupBy, options.timeBucket);
+                    this.metricCache.set(cacheKey, aggregated);
+                    this.circuitBreaker.recordSuccess();
+                    logTiming();
+                    if (span)
+                        span.setAttribute('obs_toolkit.query.result_count', aggregated.length);
+                    return aggregated;
+                }
+                const paginated = paginateResults(results, offset, limit);
+                this.metricCache.set(cacheKey, paginated);
                 this.circuitBreaker.recordSuccess();
                 logTiming();
-                return aggregated;
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.metricCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
-        }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
-        }
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
+            }
+        });
     }
     aggregate(points, aggregation, groupBy, timeBucket) {
         // Parse time bucket if provided
@@ -926,13 +1515,13 @@ export class LocalJsonlBackend {
                     value = values.length;
                     break;
                 case 'p50':
-                    value = this.calculatePercentile(values, 50);
+                    value = this.calculatePercentile(values, PERCENTILE.P50);
                     break;
                 case 'p95':
-                    value = this.calculatePercentile(values, 95);
+                    value = this.calculatePercentile(values, PERCENTILE.P95);
                     break;
                 case 'p99':
-                    value = this.calculatePercentile(values, 99);
+                    value = this.calculatePercentile(values, PERCENTILE.P99);
                     break;
                 case 'rate':
                     value = this.calculateRate(group);
@@ -962,7 +1551,7 @@ export class LocalJsonlBackend {
         if (values.length === 0)
             return 0;
         const sorted = [...values].sort((a, b) => a - b);
-        const index = Math.ceil((percentile / 100) * sorted.length) - 1;
+        const index = Math.ceil((percentile / PERCENT_MULTIPLIER) * sorted.length) - 1;
         return sorted[Math.max(0, index)];
     }
     /**
@@ -971,7 +1560,8 @@ export class LocalJsonlBackend {
      * Edge cases: single value returns 0, same timestamp returns 0
      */
     calculateRate(group) {
-        if (group.length < 2)
+        const MIN_POINTS_FOR_RATE = 2;
+        if (group.length < MIN_POINTS_FOR_RATE)
             return 0;
         // Sort by timestamp
         const sorted = [...group].sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
@@ -983,250 +1573,665 @@ export class LocalJsonlBackend {
         // Avoid division by zero when timestamps are the same
         if (durationMs === 0)
             return 0;
-        const durationSeconds = durationMs / 1000;
+        const durationSeconds = durationMs / TIME_MS.SECOND;
         return (lastPoint.value - firstPoint.value) / durationSeconds;
     }
     async queryLLMEvents(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryLLMEvents took ${durationMs.toFixed(1)}ms`);
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.llm_events', () => ({
+            'obs_toolkit.query.type': 'llm_events',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? DEFAULT_QUERY_RESULT_LIMIT,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('llm_events', durationMs, 'local');
+                this.queryHistograms.llmEvents.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    queryLog.warn(`Slow query: queryLLMEvents took ${durationMs.toFixed(1)}ms`, { durationMs });
+                }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
-            return [];
-        }
-        // Check cache first
-        const cacheKey = makeCacheKey('llm-events', options);
-        const cached = this.llmEventCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
-        }
-        const files = getFilesInRange(this.telemetryDir, /llm-events-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Build index query options - eventName maps to spanName in index
-        const indexOptions = {
-            spanName: options.eventName,
-        };
-        // Helper to apply non-indexable filters to an event
-        const applyFilters = (event) => {
-            if (options.model) {
-                const model = event.attributes?.['gen_ai.request.model'] || event.attributes?.['model'];
-                if (model !== options.model)
-                    return false;
+            // Check cache first
+            const cacheKey = makeCacheKey('llm-events', options);
+            const cached = this.llmEventCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('llmEvents');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
+                }
+                return cached;
             }
-            if (options.provider) {
-                // OTel GenAI compliant: gen_ai.provider.name -> gen_ai.system -> provider
-                const provider = event.attributes?.['gen_ai.provider.name'] ||
-                    event.attributes?.['gen_ai.system'] ||
-                    event.attributes?.['provider'];
-                if (provider !== options.provider)
-                    return false;
+            const files = getFilesInRange(this.telemetryDir, /llm-events-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate, this.allowedBases);
+            const results = [];
+            const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
+            const offset = options.offset || 0;
+            // Build index query options - eventName maps to spanName in index
+            const indexOptions = {
+                spanName: options.eventName,
+            };
+            // Helper to apply non-indexable filters to an event
+            const applyFilters = (event) => {
+                if (options.model) {
+                    const model = event.attributes?.['gen_ai.request.model'] || event.attributes?.['model'];
+                    if (model !== options.model)
+                        return false;
+                }
+                if (options.provider) {
+                    // OTel GenAI compliant: gen_ai.provider.name -> gen_ai.system -> provider
+                    const provider = event.attributes?.['gen_ai.provider.name'] ||
+                        event.attributes?.['gen_ai.system'] ||
+                        event.attributes?.['provider'];
+                    if (provider !== options.provider)
+                        return false;
+                }
+                if (options.operationName) {
+                    const opName = event.attributes?.['gen_ai.operation.name'];
+                    if (opName !== options.operationName)
+                        return false;
+                }
+                if (options.conversationId) {
+                    const convId = event.attributes?.['gen_ai.conversation.id'];
+                    if (convId !== options.conversationId)
+                        return false;
+                }
+                if (options.search) {
+                    const searchLower = options.search.toLowerCase();
+                    const attrStr = JSON.stringify(event.attributes).toLowerCase();
+                    if (!attrStr.includes(searchLower) && !event.name.toLowerCase().includes(searchLower))
+                        return false;
+                }
+                return true;
+            };
+            try {
+                for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'llm-events', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const event of rawRecords) {
+                            if (!event.timestamp || !event.name)
+                                continue;
+                            if (!applyFilters(event))
+                                continue;
+                            results.push({
+                                timestamp: event.timestamp,
+                                name: event.name,
+                                attributes: event.attributes,
+                            });
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.llmEventCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
+                        }
+                    }
+                    else {
+                        // Fall back to full file scan
+                        for await (const event of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            if (!event.timestamp || !event.name)
+                                continue;
+                            // Apply indexable filters (since no index was used)
+                            if (options.eventName && !event.name.includes(options.eventName))
+                                continue;
+                            if (!applyFilters(event))
+                                continue;
+                            results.push({
+                                timestamp: event.timestamp,
+                                name: event.name,
+                                attributes: event.attributes,
+                            });
+                            if (hasReachedLimit(results.length, offset, limit)) {
+                                const paginated = paginateResults(results, offset, limit);
+                                this.llmEventCache.set(cacheKey, paginated);
+                                this.circuitBreaker.recordSuccess();
+                                logTiming();
+                                if (span)
+                                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                return paginated;
+                            }
+                        }
+                    }
+                }
+                const paginated = paginateResults(results, offset, limit);
+                this.llmEventCache.set(cacheKey, paginated);
+                this.circuitBreaker.recordSuccess();
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
             }
-            if (options.operationName) {
-                const opName = event.attributes?.['gen_ai.operation.name'];
-                if (opName !== options.operationName)
-                    return false;
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
             }
-            if (options.conversationId) {
-                const convId = event.attributes?.['gen_ai.conversation.id'];
-                if (convId !== options.conversationId)
-                    return false;
+        });
+    }
+    async queryEvaluations(options) {
+        // P2 fix: Defer attribute allocation until span is actually created
+        return withSpan('obs_toolkit.query.evaluations', () => ({
+            'obs_toolkit.query.type': 'evaluations',
+            'obs_toolkit.query.backend': 'local',
+            'obs_toolkit.query.limit': options.limit ?? DEFAULT_QUERY_RESULT_LIMIT,
+            'obs_toolkit.query.start_date': options.startDate,
+            'obs_toolkit.query.end_date': options.endDate,
+        }), async (span) => {
+            const timer = startTiming();
+            const logTiming = () => {
+                const durationMs = timer.end();
+                recordQueryDuration('evaluations', durationMs, 'local');
+                this.queryHistograms.evaluations.observe(durationMs);
+                if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
+                    queryLog.warn(`Slow query: queryEvaluations took ${durationMs.toFixed(1)}ms`, { durationMs });
+                }
+            };
+            // Check circuit breaker - fail fast if open
+            if (!this.circuitBreaker.canRequest()) {
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', 0);
+                return [];
             }
-            if (options.search) {
-                const searchLower = options.search.toLowerCase();
-                const attrStr = JSON.stringify(event.attributes).toLowerCase();
-                if (!attrStr.includes(searchLower) && !event.name.toLowerCase().includes(searchLower))
-                    return false;
+            // Check cache first
+            const cacheKey = makeCacheKey('evaluations', options);
+            const cached = this.evaluationCache.get(cacheKey);
+            if (cached) {
+                logTiming();
+                // M3 fix: Record cache hit for metrics and span event
+                recordCacheHit('evaluations');
+                if (span) {
+                    span.addEvent('cache_hit');
+                    span.setAttribute('obs_toolkit.query.result_count', cached.length);
+                }
+                return cached;
             }
-            return true;
-        };
-        try {
-            for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'llm-events', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const event of rawRecords) {
-                        if (!event.timestamp || !event.name)
-                            continue;
-                        if (!applyFilters(event))
-                            continue;
-                        results.push({
-                            timestamp: event.timestamp,
-                            name: event.name,
-                            attributes: event.attributes,
-                        });
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.llmEventCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+            const files = getFilesInRange(this.telemetryDir, /evaluations-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate, this.allowedBases);
+            const results = [];
+            const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
+            const offset = options.offset || 0;
+            // Build index query options for indexable filters
+            const indexOptions = {
+                traceId: options.traceId,
+                evaluationName: options.evaluationName,
+                scoreLabel: options.scoreLabel,
+                responseId: options.responseId,
+                evaluator: options.evaluator,
+            };
+            // Pre-load dataset trace IDs when filtering by datasetId
+            let datasetTraceIds;
+            if (options.datasetId) {
+                const allTraces = await this.readDatasetTraces();
+                datasetTraceIds = new Set(allTraces.filter(t => t.datasetId === options.datasetId).map(t => t.traceId));
+            }
+            // Helper to apply non-indexable filters to an evaluation
+            const applyFilters = (evaluation) => {
+                // Dataset membership filter
+                if (datasetTraceIds && (!evaluation.traceId || !datasetTraceIds.has(evaluation.traceId))) {
+                    return false;
+                }
+                // Score range filters - only apply when evaluation HAS a scoreValue (P1-1 fix)
+                // Evaluations with only scoreLabel (qualitative) should pass through
+                if (options.scoreMin !== undefined && evaluation.scoreValue !== undefined) {
+                    if (evaluation.scoreValue < options.scoreMin)
+                        return false;
+                }
+                if (options.scoreMax !== undefined && evaluation.scoreValue !== undefined) {
+                    if (evaluation.scoreValue > options.scoreMax)
+                        return false;
+                }
+                // Session filter
+                if (options.sessionId && evaluation.sessionId !== options.sessionId) {
+                    return false;
+                }
+                // Evaluator type filter (not indexed, so always applied here)
+                if (options.evaluatorType && evaluation.evaluatorType !== options.evaluatorType) {
+                    return false;
+                }
+                // Agent-as-Judge filters (Section 10.7)
+                if (options.agentId && evaluation.agentId !== options.agentId) {
+                    return false;
+                }
+                if (options.agentName && evaluation.agentName !== options.agentName) {
+                    return false;
+                }
+                return true;
+            };
+            try {
+                for (const file of files) {
+                    // Try to use index for pre-filtering
+                    const matchingLines = this.tryUseIndex(file, 'evaluations', indexOptions);
+                    if (matchingLines !== null) {
+                        // Use indexed query - read only matching lines
+                        const rawRecords = await readLinesByNumber(file, matchingLines);
+                        for (const rawLine of rawRecords) {
+                            for (const raw of flattenEvaluationRecord(rawLine)) {
+                                const evaluation = normalizeEvaluation(raw);
+                                if (!evaluation)
+                                    continue;
+                                if (!applyFilters(evaluation))
+                                    continue;
+                                results.push(evaluation);
+                                if (hasReachedLimit(results.length, offset, limit)) {
+                                    const paginated = paginateResults(results, offset, limit);
+                                    this.evaluationCache.set(cacheKey, paginated);
+                                    this.circuitBreaker.recordSuccess();
+                                    logTiming();
+                                    if (span)
+                                        span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                    return paginated;
+                                }
+                            }
                         }
                     }
-                }
-                else {
-                    // Fall back to full file scan
-                    for await (const event of streamJsonl(file)) {
-                        if (!event.timestamp || !event.name)
-                            continue;
-                        // Apply indexable filters (since no index was used)
-                        if (options.eventName && !event.name.includes(options.eventName))
-                            continue;
-                        if (!applyFilters(event))
-                            continue;
-                        results.push({
-                            timestamp: event.timestamp,
-                            name: event.name,
-                            attributes: event.attributes,
-                        });
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.llmEventCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
+                    else {
+                        // Fall back to full file scan
+                        for await (const rawLine of streamJsonl(file, { statsTracker: this.parseStatsTracker })) {
+                            for (const raw of flattenEvaluationRecord(rawLine)) {
+                                const evaluation = normalizeEvaluation(raw);
+                                if (!evaluation)
+                                    continue;
+                                // Apply indexable filters (since no index was used)
+                                if (options.traceId && evaluation.traceId !== options.traceId)
+                                    continue;
+                                if (options.evaluationName && !evaluation.evaluationName.includes(options.evaluationName))
+                                    continue;
+                                if (options.scoreLabel && evaluation.scoreLabel !== options.scoreLabel)
+                                    continue;
+                                if (options.responseId && evaluation.responseId !== options.responseId)
+                                    continue;
+                                if (options.evaluator && evaluation.evaluator !== options.evaluator)
+                                    continue;
+                                if (!applyFilters(evaluation))
+                                    continue;
+                                results.push(evaluation);
+                                if (hasReachedLimit(results.length, offset, limit)) {
+                                    const paginated = paginateResults(results, offset, limit);
+                                    this.evaluationCache.set(cacheKey, paginated);
+                                    this.circuitBreaker.recordSuccess();
+                                    logTiming();
+                                    if (span)
+                                        span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                                    return paginated;
+                                }
+                            }
                         }
                     }
                 }
+                const paginated = paginateResults(results, offset, limit);
+                this.evaluationCache.set(cacheKey, paginated);
+                this.circuitBreaker.recordSuccess();
+                logTiming();
+                if (span)
+                    span.setAttribute('obs_toolkit.query.result_count', paginated.length);
+                return paginated;
+            }
+            catch (error) {
+                this.circuitBreaker.recordFailure();
+                logTiming();
+                throw error;
+            }
+        });
+    }
+    // --- Dataset Management ---
+    get datasetsFile() { return join(this.telemetryDir, 'datasets.jsonl'); }
+    get datasetTracesFile() { return join(this.telemetryDir, 'dataset-traces.jsonl'); }
+    get datasetRunsFile() { return join(this.telemetryDir, 'dataset-runs.jsonl'); }
+    /** Promise-chain mutex — serializes dataset mutations to prevent race conditions */
+    _datasetMutex = Promise.resolve();
+    _withDatasetLock(fn) {
+        const prev = this._datasetMutex;
+        let resolve;
+        this._datasetMutex = new Promise(r => { resolve = r; });
+        return prev.then(fn).finally(() => resolve());
+    }
+    async readDatasets() {
+        try {
+            const content = await readFile(this.datasetsFile, 'utf-8');
+            const results = [];
+            for (const line of content.trim().split('\n').filter(Boolean)) {
+                const parsed = JSON.parse(line);
+                const result = datasetRecordSchema.safeParse(parsed);
+                if (result.success) {
+                    results.push(result.data);
+                }
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.llmEventCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
+            return results;
         }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
+        catch {
+            return [];
         }
     }
-    async queryEvaluations(options) {
-        const timer = startTiming();
-        const logTiming = () => {
-            const durationMs = timer.end();
-            if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
-                console.warn(`[obs-toolkit] Slow query: queryEvaluations took ${durationMs.toFixed(1)}ms`);
+    async readDatasetTraces() {
+        try {
+            const content = await readFile(this.datasetTracesFile, 'utf-8');
+            const results = [];
+            for (const line of content.trim().split('\n').filter(Boolean)) {
+                const parsed = JSON.parse(line);
+                const result = datasetTraceRecordSchema.safeParse(parsed);
+                if (result.success) {
+                    results.push(result.data);
+                }
             }
-        };
-        // Check circuit breaker - fail fast if open
-        if (!this.circuitBreaker.canRequest()) {
-            logTiming();
+            return results;
+        }
+        catch {
             return [];
         }
-        // Check cache first
-        const cacheKey = makeCacheKey('evaluations', options);
-        const cached = this.evaluationCache.get(cacheKey);
-        if (cached) {
-            logTiming();
-            return cached;
+    }
+    async appendDataset(record) {
+        await mkdir(this.telemetryDir, { recursive: true });
+        await appendFile(this.datasetsFile, JSON.stringify(record) + '\n');
+    }
+    async appendDatasetTraces(records) {
+        if (records.length === 0)
+            return;
+        await mkdir(this.telemetryDir, { recursive: true });
+        const lines = records.map(r => JSON.stringify(r)).join('\n') + '\n';
+        await appendFile(this.datasetTracesFile, lines);
+    }
+    /** Atomic rewrite: write to temp file then rename (POSIX atomic on same volume) */
+    async atomicWriteFile(filePath, content) {
+        await mkdir(this.telemetryDir, { recursive: true });
+        const tmp = `${filePath}.${Date.now()}.tmp`;
+        await writeFile(tmp, content);
+        await rename(tmp, filePath);
+    }
+    async rewriteDatasetTraces(records) {
+        const lines = records.map(r => JSON.stringify(r)).join('\n') + (records.length ? '\n' : '');
+        await this.atomicWriteFile(this.datasetTracesFile, lines);
+    }
+    async readDatasetRuns() {
+        try {
+            const content = await readFile(this.datasetRunsFile, 'utf-8');
+            const results = [];
+            for (const line of content.trim().split('\n').filter(Boolean)) {
+                const parsed = JSON.parse(line);
+                const result = datasetRunRecordSchema.safeParse(parsed);
+                if (result.success) {
+                    results.push(result.data);
+                }
+            }
+            return results;
         }
-        const files = getFilesInRange(this.telemetryDir, /evaluations-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
-        const results = [];
-        const limit = options.limit || 100;
-        const offset = options.offset || 0;
-        // Build index query options for indexable filters
-        const indexOptions = {
-            traceId: options.traceId,
-            evaluationName: options.evaluationName,
-            scoreLabel: options.scoreLabel,
-            responseId: options.responseId,
-            evaluator: options.evaluator,
-        };
-        // Helper to apply non-indexable filters to an evaluation
-        const applyFilters = (evaluation) => {
-            // Score range filters - only apply when evaluation HAS a scoreValue (P1-1 fix)
-            // Evaluations with only scoreLabel (qualitative) should pass through
-            if (options.scoreMin !== undefined && evaluation.scoreValue !== undefined) {
-                if (evaluation.scoreValue < options.scoreMin)
-                    return false;
+        catch {
+            return [];
+        }
+    }
+    async appendDatasetRun(record) {
+        await mkdir(this.telemetryDir, { recursive: true });
+        await appendFile(this.datasetRunsFile, JSON.stringify(record) + '\n');
+    }
+    latestVersion(allVersions, datasetId) {
+        return allVersions
+            .filter(d => d.id === datasetId)
+            .sort((a, b) => (b.version ?? 0) - (a.version ?? 0))[0];
+    }
+    async manageDatasets(options) {
+        const MUTATING = new Set(['create', 'delete', 'add_traces', 'remove_traces', 'promote']);
+        if (MUTATING.has(options.action)) {
+            return this._withDatasetLock(() => this._manageDatasetsInner(options));
+        }
+        return this._manageDatasetsInner(options);
+    }
+    async _manageDatasetsInner(options) {
+        const now = new Date().toISOString();
+        switch (options.action) {
+            case 'create': {
+                const id = randomUUID();
+                const traceIds = [...new Set(options.traceIds ?? [])];
+                const record = {
+                    id,
+                    name: options.name,
+                    description: options.description ?? null,
+                    createdAt: now,
+                    frozen: options.frozen ?? 0,
+                    traceCount: traceIds.length,
+                    evaluationNames: options.evaluationNames ?? [],
+                    dateRangeStart: null,
+                    dateRangeEnd: null,
+                    selectionCriteria: options.selectionCriteria ?? null,
+                    version: 1,
+                    parentVersion: null,
+                    versionCreatedAt: now,
+                    versionReason: 'created',
+                    fieldMapping: options.fieldMapping,
+                    traceSnapshot: traceIds,
+                };
+                await this.appendDataset(record);
+                const traceRecords = traceIds.map(traceId => ({ datasetId: id, traceId, addedAt: now }));
+                await this.appendDatasetTraces(traceRecords);
+                return { action: 'create', dataset: record };
             }
-            if (options.scoreMax !== undefined && evaluation.scoreValue !== undefined) {
-                if (evaluation.scoreValue > options.scoreMax)
-                    return false;
+            case 'list': {
+                const all = await this.readDatasets();
+                const byId = new Map();
+                for (const d of all) {
+                    const existing = byId.get(d.id);
+                    if (!existing || (d.version ?? 0) > (existing.version ?? 0)) {
+                        byId.set(d.id, d);
+                    }
+                }
+                let datasets = [...byId.values()];
+                if (options.filterFrozen !== undefined) {
+                    datasets = datasets.filter(d => d.frozen === options.filterFrozen);
+                }
+                // Sort by createdAt descending, then id for stable ordering
+                datasets.sort((a, b) => b.createdAt.localeCompare(a.createdAt) || a.id.localeCompare(b.id));
+                // Apply cursor
+                if (options.cursor) {
+                    const decoded = decodeCursor(options.cursor);
+                    if (decoded?.createdAt && decoded.id) {
+                        const cursorAt = decoded.createdAt;
+                        const cursorId = decoded.id;
+                        const idx = datasets.findIndex(d => d.createdAt < cursorAt ||
+                            (d.createdAt === cursorAt && d.id > cursorId));
+                        datasets = idx >= 0 ? datasets.slice(idx) : [];
+                    }
+                }
+                const limit = options.limit ?? DEFAULT_QUERY_RESULT_LIMIT;
+                const hasMore = datasets.length > limit;
+                const page = datasets.slice(0, limit);
+                const last = page[page.length - 1];
+                return {
+                    action: 'list',
+                    datasets: page,
+                    count: page.length,
+                    hasMore,
+                    ...(hasMore && last && { nextCursor: encodeCursor({ createdAt: last.createdAt, id: last.id }) }),
+                };
             }
-            // Session filter
-            if (options.sessionId && evaluation.sessionId !== options.sessionId) {
-                return false;
+            case 'get': {
+                const all = await this.readDatasets();
+                const dataset = this.latestVersion(all, options.datasetId);
+                if (!dataset)
+                    throw new Error(`Dataset not found: ${options.datasetId}`);
+                const allTraces = await this.readDatasetTraces();
+                const traces = allTraces.filter(t => t.datasetId === options.datasetId);
+                const truncated = traces.length > DATASET_TRACE_TRUNCATION_LIMIT;
+                return {
+                    action: 'get',
+                    dataset,
+                    traces: truncated ? traces.slice(0, DATASET_TRACE_TRUNCATION_LIMIT) : traces,
+                    traceCount: traces.length,
+                    truncated,
+                };
             }
-            // Evaluator type filter (not indexed, so always applied here)
-            if (options.evaluatorType && evaluation.evaluatorType !== options.evaluatorType) {
-                return false;
+            case 'delete': {
+                const all = await this.readDatasets();
+                const remaining = all.filter(d => d.id !== options.datasetId);
+                const deleted = remaining.length < all.length;
+                const lines = remaining.map(r => JSON.stringify(r)).join('\n') + (remaining.length ? '\n' : '');
+                await this.atomicWriteFile(this.datasetsFile, lines);
+                const allTraces = await this.readDatasetTraces();
+                const remainingTraces = allTraces.filter(t => t.datasetId !== options.datasetId);
+                await this.rewriteDatasetTraces(remainingTraces);
+                return { action: 'delete', deleted, id: options.datasetId };
             }
-            return true;
-        };
-        try {
-            for (const file of files) {
-                // Try to use index for pre-filtering
-                const matchingLines = this.tryUseIndex(file, 'evaluations', indexOptions);
-                if (matchingLines !== null) {
-                    // Use indexed query - read only matching lines
-                    const rawRecords = await readLinesByNumber(file, matchingLines);
-                    for (const raw of rawRecords) {
-                        const evaluation = normalizeEvaluation(raw);
-                        if (!evaluation)
-                            continue;
-                        if (!applyFilters(evaluation))
-                            continue;
-                        results.push(evaluation);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.evaluationCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
-                        }
+            case 'add_traces': {
+                const all = await this.readDatasets();
+                const current = this.latestVersion(all, options.datasetId);
+                if (!current)
+                    throw new Error(`Dataset not found: ${options.datasetId}`);
+                if (current.frozen === 1)
+                    throw new Error('Cannot modify a frozen dataset');
+                const existingTraces = (await this.readDatasetTraces())
+                    .filter(t => t.datasetId === options.datasetId);
+                const existingIds = new Set(existingTraces.map(t => t.traceId));
+                const newTraceIds = options.traceIds.filter(id => !existingIds.has(id));
+                const allTraceIds = [...existingIds, ...newTraceIds];
+                const newVersion = {
+                    ...current,
+                    version: (current.version ?? 1) + 1,
+                    parentVersion: current.version ?? 1,
+                    versionCreatedAt: now,
+                    versionReason: 'traces_added',
+                    traceCount: allTraceIds.length,
+                    traceSnapshot: allTraceIds,
+                };
+                await this.appendDataset(newVersion);
+                const traceRecords = newTraceIds.map(traceId => ({ datasetId: options.datasetId, traceId, addedAt: now }));
+                await this.appendDatasetTraces(traceRecords);
+                return { action: 'add_traces', dataset: newVersion, tracesAdded: newTraceIds.length };
+            }
+            case 'remove_traces': {
+                const all = await this.readDatasets();
+                const current = this.latestVersion(all, options.datasetId);
+                if (!current)
+                    throw new Error(`Dataset not found: ${options.datasetId}`);
+                if (current.frozen === 1)
+                    throw new Error('Cannot modify a frozen dataset');
+                const allTraces = await this.readDatasetTraces();
+                const removeSet = new Set(options.traceIds);
+                const before = allTraces.filter(t => t.datasetId === options.datasetId).length;
+                const remaining = allTraces.filter(t => !(t.datasetId === options.datasetId && removeSet.has(t.traceId)));
+                const after = remaining.filter(t => t.datasetId === options.datasetId).length;
+                const removed = before - after;
+                const remainingTraceIds = remaining
+                    .filter(t => t.datasetId === options.datasetId)
+                    .map(t => t.traceId);
+                const newVersion = {
+                    ...current,
+                    version: (current.version ?? 1) + 1,
+                    parentVersion: current.version ?? 1,
+                    versionCreatedAt: now,
+                    versionReason: 'traces_removed',
+                    traceCount: after,
+                    traceSnapshot: remainingTraceIds,
+                };
+                await this.appendDataset(newVersion);
+                await this.rewriteDatasetTraces(remaining);
+                return { action: 'remove_traces', dataset: newVersion, tracesRemoved: removed };
+            }
+            case 'get_version': {
+                const all = await this.readDatasets();
+                const version = all.find(d => d.id === options.datasetId && d.version === options.version);
+                if (!version)
+                    throw new Error(`Version ${options.version} not found for dataset ${options.datasetId}`);
+                // Use traceSnapshot if available (populated since L8), otherwise fall back to current traces
+                if (version.traceSnapshot) {
+                    const snapshotSet = new Set(version.traceSnapshot);
+                    const allTraces = await this.readDatasetTraces();
+                    const traces = allTraces.filter(t => t.datasetId === options.datasetId && snapshotSet.has(t.traceId));
+                    return { action: 'get_version', dataset: version, traces };
+                }
+                const allTraces = await this.readDatasetTraces();
+                const traces = allTraces.filter(t => t.datasetId === options.datasetId);
+                return { action: 'get_version', dataset: version, traces };
+            }
+            case 'list_versions': {
+                const all = await this.readDatasets();
+                let versions = all
+                    .filter(d => d.id === options.datasetId)
+                    .sort((a, b) => (b.version ?? 0) - (a.version ?? 0));
+                // Apply cursor (version number, descending)
+                if (options.cursor) {
+                    const decoded = decodeCursor(options.cursor);
+                    const cursorVersion = decoded?.version;
+                    if (cursorVersion != null) {
+                        versions = versions.filter(d => (d.version ?? 0) < cursorVersion);
                     }
                 }
-                else {
-                    // Fall back to full file scan
-                    for await (const raw of streamJsonl(file)) {
-                        const evaluation = normalizeEvaluation(raw);
-                        if (!evaluation)
-                            continue;
-                        // Apply indexable filters (since no index was used)
-                        if (options.traceId && evaluation.traceId !== options.traceId)
-                            continue;
-                        if (options.evaluationName && !evaluation.evaluationName.includes(options.evaluationName))
-                            continue;
-                        if (options.scoreLabel && evaluation.scoreLabel !== options.scoreLabel)
-                            continue;
-                        if (options.responseId && evaluation.responseId !== options.responseId)
-                            continue;
-                        if (options.evaluator && evaluation.evaluator !== options.evaluator)
-                            continue;
-                        if (!applyFilters(evaluation))
-                            continue;
-                        results.push(evaluation);
-                        if (hasReachedLimit(results.length, offset, limit)) {
-                            const paginated = paginateResults(results, offset, limit);
-                            this.evaluationCache.set(cacheKey, paginated);
-                            this.circuitBreaker.recordSuccess();
-                            logTiming();
-                            return paginated;
-                        }
+                const limit = options.limit ?? DEFAULT_QUERY_RESULT_LIMIT;
+                const hasMore = versions.length > limit;
+                const page = versions.slice(0, limit);
+                const last = page[page.length - 1];
+                return {
+                    action: 'list_versions',
+                    versions: page,
+                    count: page.length,
+                    hasMore,
+                    ...(hasMore && last?.version != null && { nextCursor: encodeCursor({ version: last.version }) }),
+                };
+            }
+            case 'promote': {
+                const id = randomUUID();
+                const traceIds = [...new Set(options.traceIds)];
+                const record = {
+                    id,
+                    name: options.name,
+                    description: options.description ?? null,
+                    createdAt: now,
+                    frozen: options.frozen ?? 1,
+                    traceCount: traceIds.length,
+                    evaluationNames: options.evaluationNames ?? [],
+                    dateRangeStart: null,
+                    dateRangeEnd: null,
+                    selectionCriteria: null,
+                    version: 1,
+                    parentVersion: null,
+                    versionCreatedAt: now,
+                    versionReason: 'created',
+                    traceSnapshot: traceIds,
+                };
+                await this.appendDataset(record);
+                const traceRecords = traceIds.map(traceId => ({ datasetId: id, traceId, addedAt: now }));
+                await this.appendDatasetTraces(traceRecords);
+                return { action: 'promote', dataset: record };
+            }
+            case 'list_runs': {
+                const allRuns = await this.readDatasetRuns();
+                let runs = allRuns
+                    .filter(r => r.datasetId === options.datasetId)
+                    .sort((a, b) => b.runAt.localeCompare(a.runAt));
+                if (options.cursor) {
+                    const decoded = decodeCursor(options.cursor);
+                    if (decoded?.createdAt != null) {
+                        const cursorAt = decoded.createdAt;
+                        runs = runs.filter(r => r.runAt < cursorAt);
                     }
                 }
+                const limit = options.limit ?? DEFAULT_QUERY_RESULT_LIMIT;
+                const hasMore = runs.length > limit;
+                const page = runs.slice(0, limit);
+                const last = page[page.length - 1];
+                return {
+                    action: 'list_runs',
+                    runs: page,
+                    count: page.length,
+                    hasMore,
+                    ...(hasMore && last && { nextCursor: encodeCursor({ createdAt: last.runAt }) }),
+                };
             }
-            const paginated = paginateResults(results, offset, limit);
-            this.evaluationCache.set(cacheKey, paginated);
-            this.circuitBreaker.recordSuccess();
-            logTiming();
-            return paginated;
-        }
-        catch (error) {
-            this.circuitBreaker.recordFailure();
-            logTiming();
-            throw error;
+            default:
+                throw new Error(`Unknown dataset action: ${options.action}`);
         }
     }
     async healthCheck() {
@@ -1299,7 +2304,7 @@ export class MultiDirectoryBackend {
         return this.directories;
     }
     async queryTraces(options) {
-        const limit = options.limit || 100;
+        const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
         // Query all backends in parallel
         const allBackendResults = await Promise.all(this.backends.map(b => b.queryTraces({ ...options, limit })));
         // Merge results using bounded insertion for efficient top-K selection
@@ -1312,7 +2317,7 @@ export class MultiDirectoryBackend {
         return topResults;
     }
     async queryLogs(options) {
-        const limit = options.limit || 100;
+        const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
         // Query all backends in parallel
         const allBackendResults = await Promise.all(this.backends.map(b => b.queryLogs({ ...options, limit })));
         // Merge results using bounded insertion for efficient top-K selection
@@ -1325,7 +2330,7 @@ export class MultiDirectoryBackend {
         return topResults;
     }
     async queryMetrics(options) {
-        const limit = options.limit || 100;
+        const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
         // Query all backends in parallel
         const allBackendResults = await Promise.all(this.backends.map(b => b.queryMetrics({ ...options, limit })));
         // Flatten and limit results
@@ -1333,7 +2338,7 @@ export class MultiDirectoryBackend {
         return allResults.slice(0, limit);
     }
     async queryLLMEvents(options) {
-        const limit = options.limit || 100;
+        const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
         // Query all backends in parallel
         const allBackendResults = await Promise.all(this.backends.map(b => b.queryLLMEvents({ ...options, limit })));
         // Merge results using bounded insertion for efficient top-K selection
@@ -1346,7 +2351,7 @@ export class MultiDirectoryBackend {
         return topResults;
     }
     async queryEvaluations(options) {
-        const limit = options.limit || 100;
+        const limit = options.limit || DEFAULT_QUERY_RESULT_LIMIT;
         // Query all backends in parallel
         const allBackendResults = await Promise.all(this.backends.map(b => b.queryEvaluations({ ...options, limit })));
         // Merge results using bounded insertion for efficient top-K selection
@@ -1410,6 +2415,105 @@ export class MultiDirectoryBackend {
         }
         return result;
     }
+    /**
+     * Get aggregated parse statistics from all backends
+     */
+    getParseStats() {
+        // Aggregate stats from all backends
+        let totalFiles = 0;
+        let totalLines = 0;
+        let totalParsed = 0;
+        let totalSkipped = 0;
+        const allWorstFiles = [];
+        for (const backend of this.backends) {
+            const stats = backend.getParseStats();
+            totalFiles += stats.totalFiles;
+            totalLines += stats.totalLines;
+            totalParsed += stats.totalParsed;
+            totalSkipped += stats.totalSkipped;
+            // Collect worst files with skip rates
+            for (const file of stats.worstFiles) {
+                const totalProcessed = file.parsedLines + file.skippedLines;
+                if (totalProcessed > 0) {
+                    allWorstFiles.push({
+                        ...file,
+                        skipRate: file.skippedLines / totalProcessed,
+                    });
+                }
+            }
+        }
+        // Calculate overall success rate
+        const totalProcessed = totalParsed + totalSkipped;
+        const parseSuccessRate = totalProcessed > 0 ? totalParsed / totalProcessed : 1;
+        // Get top worst files across all backends
+        const worstFiles = allWorstFiles
+            .sort((a, b) => b.skipRate - a.skipRate)
+            .slice(0, WORST_FILES_LIMIT)
+            .map(({ skipRate: _, ...f }) => f);
+        return {
+            totalFiles,
+            totalLines,
+            totalParsed,
+            totalSkipped,
+            parseSuccessRate,
+            worstFiles,
+        };
+    }
+    /**
+     * Get aggregated query latency statistics from all backends
+     */
+    getQueryStats() {
+        // Aggregate histogram data from all backends
+        // For percentiles, we use the first backend's data since aggregating
+        // percentiles across backends requires full data merge
+        // Count and sum can be aggregated directly
+        const types = ['traces', 'logs', 'metrics', 'llmEvents', 'evaluations'];
+        const result = {};
+        for (const type of types) {
+            let totalCount = 0;
+            let weightedP50 = 0;
+            let weightedP95 = 0;
+            let weightedP99 = 0;
+            let minValue = Infinity;
+            let maxValue = -Infinity;
+            let totalSum = 0;
+            for (const backend of this.backends) {
+                const stats = backend.getQueryStats()[type];
+                if (stats.count > 0) {
+                    totalCount += stats.count;
+                    totalSum += stats.mean * stats.count;
+                    weightedP50 += stats.p50 * stats.count;
+                    weightedP95 += stats.p95 * stats.count;
+                    weightedP99 += stats.p99 * stats.count;
+                    minValue = Math.min(minValue, stats.min);
+                    maxValue = Math.max(maxValue, stats.max);
+                }
+            }
+            if (totalCount > 0) {
+                result[type] = {
+                    min: minValue,
+                    max: maxValue,
+                    mean: totalSum / totalCount,
+                    p50: weightedP50 / totalCount,
+                    p95: weightedP95 / totalCount,
+                    p99: weightedP99 / totalCount,
+                    count: totalCount,
+                };
+            }
+            else {
+                result[type] = {
+                    min: 0,
+                    max: 0,
+                    mean: 0,
+                    p50: 0,
+                    p95: 0,
+                    p99: 0,
+                    count: 0,
+                };
+            }
+        }
+        return result;
+    }
     /**
      * Export traces in OTLP JSON format
      */
@@ -1431,5 +2535,110 @@ export class MultiDirectoryBackend {
         const metrics = await this.queryMetrics(options);
         return convertToOTLPMetrics(metrics);
     }
+    async manageDatasets(options) {
+        if (this.backends.length === 0)
+            throw new Error('No telemetry directories available');
+        const READ_ACTIONS = new Set(['list', 'get', 'get_version', 'list_versions', 'list_runs']);
+        // Write actions delegate to first backend
+        if (!READ_ACTIONS.has(options.action)) {
+            return this.backends[0].manageDatasets(options);
+        }
+        // Single backend — no merge needed
+        if (this.backends.length === 1) {
+            return this.backends[0].manageDatasets(options);
+        }
+        // Fan out read actions across all backends and merge
+        switch (options.action) {
+            case 'list': {
+                const results = await Promise.all(this.backends.map(b => b.manageDatasets(options).catch(() => null)));
+                const byId = new Map();
+                for (const r of results) {
+                    if (!r || r.action !== 'list')
+                        continue;
+                    for (const d of r.datasets) {
+                        const existing = byId.get(d.id);
+                        if (!existing || (d.version ?? 0) > (existing.version ?? 0)) {
+                            byId.set(d.id, d);
+                        }
+                    }
+                }
+                const datasets = [...byId.values()];
+                const limit = options.limit ?? DEFAULT_QUERY_RESULT_LIMIT;
+                return {
+                    action: 'list',
+                    datasets: datasets.slice(0, limit),
+                    count: datasets.length,
+                    hasMore: datasets.length > limit,
+                };
+            }
+            case 'get': {
+                let best = null;
+                for (const b of this.backends) {
+                    try {
+                        const r = await b.manageDatasets(options);
+                        if (r.action === 'get') {
+                            if (!best || best.action !== 'get' ||
+                                (r.dataset.version ?? 0) > (best.dataset.version ?? 0)) {
+                                best = r;
+                            }
+                        }
+                    }
+                    catch { /* not found in this backend */ }
+                }
+                if (!best)
+                    throw new Error(`Dataset not found: ${options.datasetId}`);
+                return best;
+            }
+            case 'get_version': {
+                for (const b of this.backends) {
+                    try {
+                        return await b.manageDatasets(options);
+                    }
+                    catch { /* not in this backend */ }
+                }
+                throw new Error(`Version ${options.version} not found for dataset ${options.datasetId}`);
+            }
+            case 'list_versions': {
+                const results = await Promise.all(this.backends.map(b => b.manageDatasets(options).catch(() => null)));
+                const byVersion = new Map();
+                for (const r of results) {
+                    if (!r || r.action !== 'list_versions')
+                        continue;
+                    for (const v of r.versions) {
+                        if (!byVersion.has(v.version ?? 0)) {
+                            byVersion.set(v.version ?? 0, v);
+                        }
+                    }
+                }
+                const versions = [...byVersion.values()].sort((a, b) => (b.version ?? 0) - (a.version ?? 0));
+                const limit = options.limit ?? DEFAULT_QUERY_RESULT_LIMIT;
+                return {
+                    action: 'list_versions',
+                    versions: versions.slice(0, limit),
+                    count: versions.length,
+                    hasMore: versions.length > limit,
+                };
+            }
+            case 'list_runs': {
+                const results = await Promise.all(this.backends.map(b => b.manageDatasets(options).catch(() => null)));
+                const allRuns = [];
+                for (const r of results) {
+                    if (!r || r.action !== 'list_runs')
+                        continue;
+                    allRuns.push(...r.runs);
+                }
+                allRuns.sort((a, b) => b.runAt.localeCompare(a.runAt));
+                const limit = options.limit ?? DEFAULT_QUERY_RESULT_LIMIT;
+                return {
+                    action: 'list_runs',
+                    runs: allRuns.slice(0, limit),
+                    count: allRuns.length,
+                    hasMore: allRuns.length > limit,
+                };
+            }
+            default:
+                return this.backends[0].manageDatasets(options);
+        }
+    }
 }
 //# sourceMappingURL=local-jsonl.js.map