observability-toolkit 2.0.0 → 2.1.0

package/dist/tools/routing-telemetry.test.js

@@ -0,0 +1,747 @@
+/**
+ * Unit tests for routing-telemetry tool (R8)
+ */
+import { describe, it } from 'vitest';
+import assert from 'node:assert';
+import { ROUTING_ATTRIBUTES } from '../backends/index.js';
+import { createTraceBackend } from '../test-helpers/mock-backends.js';
+import { routingTelemetry, routingTelemetrySchema, ROUTING_TELEMETRY_DEFAULT_LIMIT } from './routing-telemetry.js';
+import { ONE_MILLION, ROUND_4DP_FACTOR } from '../lib/core/units.js';
+import { MODEL_PRICING } from '../lib/core/constants-models.js';
+import { SHORT_ID_LENGTH } from '../lib/core/constants.js';
+import { RANDOM_STRING_RADIX, RANDOM_SUFFIX_START } from '../test-helpers/test-constants.js';
+// ---------- test constants ----------
+const ID_SLICE_START = RANDOM_SUFFIX_START;
+const ID_SLICE_END = SHORT_ID_LENGTH;
+const SPAN_DURATION_MS = 100;
+const CLASSIFICATION_TIME_MS = 42;
+// model fixtures
+const MODEL_AUTO = 'auto';
+const MODEL_GPT4O = 'gpt-4o';
+const MODEL_GPT4O_MINI = 'gpt-4o-mini';
+const MODEL_CLAUDE_OPUS = 'claude-opus-4-6';
+const MODEL_CLAUDE_HAIKU = 'claude-haiku-4-5-20251001';
+// token fixtures
+const INPUT_TOKENS_SMALL = 1_000;
+const OUTPUT_TOKENS_SMALL = 200;
+const INPUT_TOKENS_LARGE = 5_000;
+const OUTPUT_TOKENS_LARGE = 1_000;
+// routing attribute fixtures
+const STRATEGY_COST = 'cost_optimized';
+const STRATEGY_PERF = 'performance';
+// expected counts
+const EXPECTED_ONE = 1;
+const EXPECTED_TWO = 2;
+const EXPECTED_THREE = 3;
+// expected fallback rate (1 out of 2 = 0.5)
+const EXPECTED_FALLBACK_RATE_HALF = 0.5;
+const EXPECTED_FULL_FALLBACK_RATE = 1;
+// ---------- helpers ----------
+function makeSpan(overrides) {
+    const duration = overrides.durationMs ?? SPAN_DURATION_MS;
+    return {
+        traceId: overrides.traceId ?? 'trace-1',
+        spanId: `span-${Math.random().toString(RANDOM_STRING_RADIX).slice(ID_SLICE_START, ID_SLICE_END)}`,
+        name: 'chat',
+        startTimeUnixNano: Date.now() * ONE_MILLION,
+        endTimeUnixNano: (Date.now() + duration) * ONE_MILLION,
+        durationMs: duration,
+        attributes: {
+            'gen_ai.request.model': overrides.requestModel,
+            ...(overrides.responseModel !== undefined && {
+                'gen_ai.response.model': overrides.responseModel,
+            }),
+            ...(overrides.inputTokens !== undefined && {
+                'gen_ai.usage.input_tokens': overrides.inputTokens,
+            }),
+            ...(overrides.outputTokens !== undefined && {
+                'gen_ai.usage.output_tokens': overrides.outputTokens,
+            }),
+            ...(overrides.strategy !== undefined && {
+                'routing.strategy': overrides.strategy,
+            }),
+            ...(overrides.fallbackTriggered !== undefined && {
+                'routing.fallback_triggered': overrides.fallbackTriggered,
+            }),
+            ...(overrides.classificationTimeMs !== undefined && {
+                'routing.classification.time_ms': overrides.classificationTimeMs,
+            }),
+            ...(overrides.finishReasons !== undefined && {
+                'gen_ai.response.finish_reasons': overrides.finishReasons,
+            }),
+            ...(overrides.provider !== undefined && {
+                'gen_ai.provider.name': overrides.provider,
+            }),
+            ...(overrides.system !== undefined && {
+                'gen_ai.system': overrides.system,
+            }),
+        },
+    };
+}
+function makeBackend(spans) {
+    const store = { local: spans };
+    return createTraceBackend(store, 'local');
+}
+function isNoData(r) {
+    return r.kind === 'no-data';
+}
+function isResult(r) {
+    return r.kind === 'result';
+}
+// ---------- schema ----------
+void describe('routingTelemetrySchema', () => {
+    void it('should accept empty input with defaults', () => {
+        const result = routingTelemetrySchema.parse({});
+        assert.strictEqual(result.backend, 'cloud');
+        assert.strictEqual(result.groupBy, 'model_pair');
+        assert.strictEqual(result.limit, ROUTING_TELEMETRY_DEFAULT_LIMIT);
+    });
+    void it('should accept groupBy strategy', () => {
+        const result = routingTelemetrySchema.parse({ groupBy: 'strategy' });
+        assert.strictEqual(result.groupBy, 'strategy');
+    });
+    void it('should reject invalid groupBy value', () => {
+        assert.throws(() => routingTelemetrySchema.parse({ groupBy: 'invalid' }));
+    });
+    void it('should accept optional model filter', () => {
+        const result = routingTelemetrySchema.parse({ model: MODEL_GPT4O });
+        assert.strictEqual(result.model, MODEL_GPT4O);
+    });
+    void it('should accept startDate and endDate filters', () => {
+        const result = routingTelemetrySchema.parse({
+            startDate: '2026-01-01',
+            endDate: '2026-03-01',
+        });
+        assert.strictEqual(result.startDate, '2026-01-01');
+        assert.strictEqual(result.endDate, '2026-03-01');
+    });
+});
+// ---------- handler ----------
+void describe('routingTelemetry', () => {
+    void it('should return no-routing message when all spans have matching request/response models', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_GPT4O, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_CLAUDE_OPUS, responseModel: MODEL_CLAUDE_OPUS }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isNoData(result), 'should return a no-data result indicating no routing');
+        assert.ok(result.message.toLowerCase().includes('no routing') || result.message.toLowerCase().includes('routing decisions'), `unexpected message: ${result.message}`);
+    });
+    void it('should return no-routing message when backend has no spans', async () => {
+        const backend = makeBackend([]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isNoData(result), 'should return a no-data result for empty backend');
+    });
+    void it('should detect routing decisions where request model differs from response model', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.summary.routedSpans, EXPECTED_ONE);
+    });
+    void it('should include routing pair in results', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        const group = result.groups[0];
+        assert.ok(group, 'should have at least one group');
+        assert.ok(('requestedModel' in group && group.requestedModel === MODEL_AUTO) || ('pair' in group && group.pair.includes(MODEL_AUTO)) || JSON.stringify(group).includes(MODEL_AUTO), 'routing pair should reference the requested model');
+        assert.ok(('actualModel' in group && group.actualModel === MODEL_GPT4O) || ('pair' in group && group.pair.includes(MODEL_GPT4O)) || JSON.stringify(group).includes(MODEL_GPT4O), 'routing pair should reference the actual model');
+    });
+    void it('should skip spans missing response model', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.summary.routedSpans, EXPECTED_ONE);
+    });
+    void it('should compute model distribution counting actual models selected', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O_MINI }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.ok(result.modelDistribution, 'should include model distribution');
+        const gpt4oCount = result.modelDistribution[MODEL_GPT4O];
+        assert.ok(gpt4oCount !== undefined, 'gpt-4o should appear in distribution');
+        assert.strictEqual(gpt4oCount, EXPECTED_TWO);
+    });
+    void it('should compute fallback rate from routing.fallback_triggered spans', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, fallbackTriggered: true }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, fallbackTriggered: false }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.summary.fallbackRate, EXPECTED_FALLBACK_RATE_HALF);
+    });
+    void it('should compute P50 and P99 routing latency when classification time is present', async () => {
+        const spans = Array.from({ length: 10 }, (_, i) => makeSpan({
+            requestModel: MODEL_AUTO,
+            responseModel: MODEL_GPT4O,
+            classificationTimeMs: (i + 1) * 10,
+        }));
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.ok(result.routingLatency, 'should include routing latency stats');
+        // HdrHistogram: input [10,20,...,100]
+        // p50: 50% of 10 = 5th value = 50
+        // p99: 99% of 10 → 10th value = 100
+        const EXPECTED_P50 = 50;
+        const EXPECTED_P99 = 100;
+        assert.strictEqual(result.routingLatency.p50, EXPECTED_P50, 'p50 should be 50');
+        assert.strictEqual(result.routingLatency.p99, EXPECTED_P99, `p99 should be 100, got ${result.routingLatency.p99}`);
+        assert.ok(result.routingLatency.p99 >= result.routingLatency.p50, 'p99 should be >= p50');
+    });
+    void it('should fallback to span durationMs when no spans have classification.time_ms (R8-F6)', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.ok(result.routingLatency, 'should have routing latency from span duration fallback');
+        assert.strictEqual(result.routingLatency.p50, SPAN_DURATION_MS, 'p50 should equal span durationMs');
+        assert.strictEqual(result.routingLatency.p99, SPAN_DURATION_MS, 'p99 should equal span durationMs');
+        assert.strictEqual(result.routingLatency.source, 'span_duration', 'source should indicate span_duration fallback');
+    });
+    void it('should compute routing latency from varied span durations (R8-F6-L2)', async () => {
+        // 10 spans with durationMs = 10,20,...,100 — no classificationTimeMs → fallback path
+        const spans = Array.from({ length: 10 }, (_, i) => makeSpan({
+            requestModel: MODEL_AUTO,
+            responseModel: MODEL_GPT4O,
+            durationMs: (i + 1) * 10,
+        }));
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.ok(result.routingLatency, 'should have routing latency from span duration fallback');
+        assert.strictEqual(result.routingLatency.source, 'span_duration', 'source should indicate span_duration fallback');
+        // HdrHistogram: input [10,20,...,100]
+        // p50: 50% of 10 = 5th value = 50
+        const EXPECTED_P50 = 50;
+        assert.strictEqual(result.routingLatency.p50, EXPECTED_P50, 'p50 should be 50');
+        // p99: 99% of 10 → 10th value = 100
+        const EXPECTED_P99 = 100;
+        assert.strictEqual(result.routingLatency.p99, EXPECTED_P99, `p99 should be 100, got ${result.routingLatency.p99}`);
+    });
+    void it('should prefer classification.time_ms over durationMs when both present (R8-F6)', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, classificationTimeMs: CLASSIFICATION_TIME_MS }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.ok(result.routingLatency, 'should have routing latency');
+        assert.strictEqual(result.routingLatency.p50, CLASSIFICATION_TIME_MS, 'should use classification time, not durationMs');
+        assert.strictEqual(result.routingLatency.source, 'classification_time', 'source should indicate classification_time');
+    });
+    void it('should group by model pair when groupBy is model_pair', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O_MINI }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local', groupBy: 'model_pair' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.groups.length, EXPECTED_TWO);
+        const gpt4oGroup = result.groups.find((g) => ('actualModel' in g && g.actualModel === MODEL_GPT4O) || ('pair' in g && g.pair.includes(MODEL_GPT4O)));
+        assert.ok(gpt4oGroup, 'gpt-4o group should exist');
+        assert.strictEqual(gpt4oGroup.count, EXPECTED_TWO);
+    });
+    void it('should group by strategy when groupBy is strategy', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, strategy: STRATEGY_COST }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O_MINI, strategy: STRATEGY_COST }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_CLAUDE_OPUS, strategy: STRATEGY_PERF }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local', groupBy: 'strategy' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.groups.length, EXPECTED_TWO);
+        const costGroup = result.groups.find((g) => 'strategy' in g && g.strategy === STRATEGY_COST);
+        assert.ok(costGroup, 'cost_optimized group should exist');
+        assert.strictEqual(costGroup.count, EXPECTED_TWO);
+    });
+    void it('should include per-strategy fallbackCount and fallbackRate (R8-F7)', async () => {
+        const backend = makeBackend([
+            // cost_optimized: 1/2 fallback → rate 0.5
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, strategy: STRATEGY_COST, fallbackTriggered: true }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O_MINI, strategy: STRATEGY_COST, fallbackTriggered: false }),
+            // performance: 0/1 fallback → rate 0
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_CLAUDE_OPUS, strategy: STRATEGY_PERF, fallbackTriggered: false }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local', groupBy: 'strategy' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.groups.length, EXPECTED_TWO);
+        const costGroup = result.groups.find((g) => 'strategy' in g && g.strategy === STRATEGY_COST);
+        assert.ok(costGroup && 'fallbackCount' in costGroup, 'cost group should have fallbackCount');
+        assert.strictEqual(costGroup.fallbackCount, EXPECTED_ONE, 'cost group should have 1 fallback');
+        assert.strictEqual(costGroup.fallbackRate, EXPECTED_FALLBACK_RATE_HALF, 'cost group fallback rate should be 0.5');
+        const perfGroup = result.groups.find((g) => 'strategy' in g && g.strategy === STRATEGY_PERF);
+        assert.ok(perfGroup && 'fallbackRate' in perfGroup, 'perf group should have fallbackRate');
+        assert.strictEqual(perfGroup.fallbackCount, 0, 'perf group should have 0 fallbacks');
+        assert.strictEqual(perfGroup.fallbackRate, 0, 'perf group fallback rate should be 0');
+    });
+    void it('should include providerDistribution and per-group provider from gen_ai.provider.name (R8-F4)', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, provider: 'openai' }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O_MINI, provider: 'openai' }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_CLAUDE_OPUS, provider: 'anthropic' }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local', groupBy: 'model_pair' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.ok(result.providerDistribution, 'should include providerDistribution');
+        assert.strictEqual(result.providerDistribution['openai'], EXPECTED_TWO, 'openai should have 2 spans');
+        assert.strictEqual(result.providerDistribution['anthropic'], EXPECTED_ONE, 'anthropic should have 1 span');
+        // Model pair groups should carry provider
+        const opusGroup = result.groups.find((g) => 'actualModel' in g && g.actualModel === MODEL_CLAUDE_OPUS);
+        assert.ok(opusGroup && 'provider' in opusGroup, 'opus group should have provider');
+        assert.strictEqual(opusGroup.provider, 'anthropic');
+    });
+    void it('should fallback to gen_ai.system when gen_ai.provider.name is absent (R8-F4)', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, system: 'openai' }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.providerDistribution['openai'], EXPECTED_ONE, 'should resolve provider from gen_ai.system');
+    });
+    void it('should use (unknown) provider when neither gen_ai.provider.name nor gen_ai.system present (R8-F4)', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.providerDistribution['(unknown)'], EXPECTED_ONE, 'should count as (unknown)');
+        // Model pair group should have null provider
+        const group = result.groups[0];
+        assert.ok(group && 'provider' in group, 'group should have provider field');
+        assert.strictEqual(group.provider, null, 'provider should be null when absent');
+    });
+    void it('should filter by model name matching either request or response model', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_CLAUDE_HAIKU }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local', model: MODEL_GPT4O }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.summary.routedSpans, EXPECTED_TWO);
+    });
+    void it('should compute cost savings comparing actual vs hypothetical expensive-model cost', async () => {
+        const backend = makeBackend([
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: MODEL_GPT4O_MINI,
+                inputTokens: INPUT_TOKENS_LARGE,
+                outputTokens: OUTPUT_TOKENS_LARGE,
+            }),
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: MODEL_GPT4O,
+                inputTokens: INPUT_TOKENS_SMALL,
+                outputTokens: OUTPUT_TOKENS_SMALL,
+            }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.ok(typeof result.costSavings === 'number', 'cost savings should be a number');
+        assert.ok(result.costSavings >= 0, 'cost savings should be non-negative');
+    });
+    void it('should include total routed span count in summary', async () => {
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_CLAUDE_HAIKU }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return a full result');
+        assert.strictEqual(result.summary.routedSpans, EXPECTED_THREE);
+    });
+    void it('should include tool name obs_routing_telemetry in routingTelemetryTool export', async () => {
+        const { routingTelemetryTool } = await import('./routing-telemetry.js');
+        assert.strictEqual(routingTelemetryTool.name, 'obs_routing_telemetry');
+    });
+});
+// ---------- regression tests (review findings) ----------
+void describe('H1/F1 — phantom pricing baseline', () => {
+    void it('should use single most expensive model by total cost, not mixed input/output', async () => {
+        // Model A: high input ($10), low output ($5) → total $15
+        // Model B: low input ($2), high output ($12) → total $14
+        // Phantom bug would pick input=$10 + output=$12 = $22 (wrong)
+        // Fix picks Model A (total $15) → hypothetical uses {input: 10, output: 5}
+        MODEL_PRICING['phantom-model-a'] = { input: 10, output: 5, provider: 'test' };
+        MODEL_PRICING['phantom-model-b'] = { input: 2, output: 12, provider: 'test' };
+        const spans = [
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: 'phantom-model-a',
+                inputTokens: 1_000_000,
+                outputTokens: 1_000_000,
+            }),
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: 'phantom-model-b',
+                inputTokens: 1_000_000,
+                outputTokens: 1_000_000,
+            }),
+        ];
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        // Actual: model-a (10+5=15) + model-b (2+12=14) = 29
+        // Hypothetical (model-a pair): 2 * (10+5) = 30
+        // Savings = 30 - 29 = 1
+        assert.strictEqual(result.costSavings, 1);
+        delete MODEL_PRICING['phantom-model-a'];
+        delete MODEL_PRICING['phantom-model-b'];
+    });
+    void it('should round costSavings to 4 decimal places (F5)', async () => {
+        MODEL_PRICING['round-cheap'] = { input: 0.001, output: 0.001, provider: 'test' };
+        MODEL_PRICING['round-pricey'] = { input: 0.003, output: 0.003, provider: 'test' };
+        const spans = [
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: 'round-cheap', inputTokens: 333, outputTokens: 333 }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: 'round-pricey', inputTokens: 333, outputTokens: 333 }),
+        ];
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        // costSavings * ROUND_4DP_FACTOR should be integer (no more than 4dp)
+        assert.strictEqual(result.costSavings * ROUND_4DP_FACTOR, Math.round(result.costSavings * ROUND_4DP_FACTOR));
+        delete MODEL_PRICING['round-cheap'];
+        delete MODEL_PRICING['round-pricey'];
+    });
+});
+void describe('M2/M3 — numeric coercion for OTLP string attributes', () => {
+    void it('should handle string token counts from OTLP JSON (M2)', async () => {
+        const spans = [
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+        ];
+        // Simulate OTLP string values by overwriting attributes
+        const span0 = spans[0];
+        assert(span0 !== undefined && span0.attributes !== undefined, 'span0 and its attributes must exist');
+        span0.attributes['gen_ai.usage.input_tokens'] = '1000';
+        span0.attributes['gen_ai.usage.output_tokens'] = '500';
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        assert.ok(Number.isFinite(result.costSavings), 'costSavings should not be NaN');
+        assert.ok(result.costSavings >= 0, 'costSavings should be non-negative');
+    });
+    void it('should handle string classificationTimeMs (M3)', async () => {
+        const spans = [
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+        ];
+        const m3span0 = spans[0];
+        const m3span1 = spans[1];
+        assert(m3span0?.attributes !== undefined && m3span1?.attributes !== undefined, 'spans and attributes must exist');
+        m3span0.attributes['routing.classification.time_ms'] = '42.5';
+        m3span1.attributes['routing.classification.time_ms'] = '100';
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        assert.ok(result.routingLatency, 'should have routing latency');
+        assert.ok(result.routingLatency.p50 > 0, 'p50 should be positive');
+        assert.ok(result.routingLatency.p99 > 0, 'p99 should be positive');
+    });
+    void it('should treat non-numeric token values as 0', async () => {
+        const spans = [
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+        ];
+        const m4span0 = spans[0];
+        assert(m4span0?.attributes !== undefined, 'span0 and its attributes must exist');
+        m4span0.attributes['gen_ai.usage.input_tokens'] = 'not-a-number';
+        m4span0.attributes['gen_ai.usage.output_tokens'] = undefined;
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        assert.ok(Number.isFinite(result.costSavings), 'should not produce NaN');
+    });
+});
+void describe('F3 — ROUTING_ATTRIBUTES exported constants', () => {
+    void it('should export routing attribute keys from backends/index', () => {
+        assert.strictEqual(ROUTING_ATTRIBUTES.STRATEGY, 'routing.strategy');
+        assert.strictEqual(ROUTING_ATTRIBUTES.FALLBACK_TRIGGERED, 'routing.fallback_triggered');
+        assert.strictEqual(ROUTING_ATTRIBUTES.CLASSIFICATION_TIME_MS, 'routing.classification.time_ms');
+    });
+});
+void describe('F8 — unknown models in MODEL_PRICING', () => {
+    void it('should produce costSavings=0 when all models are unknown (no pricing data)', async () => {
+        const backend = makeBackend([
+            makeSpan({
+                requestModel: 'unknown-model-x',
+                responseModel: 'unknown-model-y',
+                inputTokens: INPUT_TOKENS_LARGE,
+                outputTokens: OUTPUT_TOKENS_LARGE,
+            }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        assert.strictEqual(result.costSavings, 0, 'unknown models should produce zero cost savings');
+    });
+    void it('should inflate savings when only some models are unknown (actualCost=0 for unknown)', async () => {
+        const backend = makeBackend([
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: 'unknown-model-z',
+                inputTokens: INPUT_TOKENS_LARGE,
+                outputTokens: OUTPUT_TOKENS_LARGE,
+            }),
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: MODEL_GPT4O,
+                inputTokens: INPUT_TOKENS_SMALL,
+                outputTokens: OUTPUT_TOKENS_SMALL,
+            }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        // Unknown model contributes 0 to actualCost but hypothetical uses gpt-4o pricing → savings inflated
+        assert.ok(result.costSavings > 0, 'savings inflated when unknown model produces zero actual cost');
+    });
+});
+void describe('F10 — finish_reasons fallback signal', () => {
+    void it('should detect fallback from content_filter finish reason', async () => {
+        const backend = makeBackend([
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: MODEL_GPT4O,
+                finishReasons: ['content_filter'],
+            }),
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: MODEL_GPT4O,
+                finishReasons: ['stop'],
+            }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        assert.strictEqual(result.summary.fallbackRate, EXPECTED_FALLBACK_RATE_HALF, 'content_filter should count as fallback');
+    });
+    void it('should detect fallback from length finish reason', async () => {
+        const backend = makeBackend([
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: MODEL_GPT4O,
+                finishReasons: ['length'],
+            }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        assert.strictEqual(result.summary.fallbackRate, EXPECTED_FULL_FALLBACK_RATE, 'length should count as fallback');
+    });
+    void it('should not double-count when both fallbackTriggered and finish_reasons indicate fallback', async () => {
+        const backend = makeBackend([
+            makeSpan({
+                requestModel: MODEL_AUTO,
+                responseModel: MODEL_GPT4O,
+                fallbackTriggered: true,
+                finishReasons: ['content_filter'],
+            }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        assert.strictEqual(result.summary.fallbackRate, EXPECTED_FULL_FALLBACK_RATE, 'should count as single fallback');
+    });
+});
+void describe('F12 — totalSpansScanned field', () => {
+    void it('should include totalSpansScanned in no-data result', async () => {
+        const TOTAL_SPANS = 3;
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_GPT4O, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_GPT4O, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_GPT4O, responseModel: MODEL_GPT4O }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isNoData(result));
+        assert.strictEqual(result.totalSpansScanned, TOTAL_SPANS);
+    });
+    void it('should include totalSpansScanned in result', async () => {
+        const TOTAL_SPANS = 3;
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_GPT4O, responseModel: MODEL_GPT4O }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_CLAUDE_HAIKU }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        assert.strictEqual(result.totalSpansScanned, TOTAL_SPANS);
+    });
+});
+// ============================================================================
+// Stress Tests (ST-R8)
+// ============================================================================
+// MAX_QUERY_LIMIT = 1000; use max to stress the routing at full capacity
+const ST_R8_SPAN_COUNT = 1_000;
+const ST_R8_UNIQUE_MODELS = 8;
+const ST_R8_REQUEST_MODELS = ['auto', 'router', 'gpt-4-turbo', 'claude-3'];
+const ST_R8_RESPONSE_MODELS = [
+    MODEL_GPT4O,
+    MODEL_GPT4O_MINI,
+    MODEL_CLAUDE_OPUS,
+    MODEL_CLAUDE_HAIKU,
+    'claude-3-5-sonnet-20241022',
+    'gpt-4-turbo',
+    'claude-3-haiku-20240307',
+    'claude-3-opus-20240229',
+];
+function makeStressSpan(i) {
+    const requestModel = ST_R8_REQUEST_MODELS[i % ST_R8_REQUEST_MODELS.length];
+    const responseModel = ST_R8_RESPONSE_MODELS[i % ST_R8_UNIQUE_MODELS];
+    return makeSpan({
+        requestModel,
+        responseModel,
+        inputTokens: 500 + (i % 2000),
+        outputTokens: 100 + (i % 500),
+        strategy: i % 3 === 0 ? STRATEGY_COST : (i % 3 === 1 ? STRATEGY_PERF : undefined),
+        fallbackTriggered: i % 7 === 0,
+        classificationTimeMs: i % 5 === 0 ? 10 + (i % 50) : undefined,
+    });
+}
+void describe('routingTelemetry stress (ST-R8)', () => {
+    void it('high-volume (1K spans at max limit): produces a result and all numeric fields are finite', async () => {
+        const spans = Array.from({ length: ST_R8_SPAN_COUNT }, (_, i) => makeStressSpan(i));
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local', limit: ST_R8_SPAN_COUNT }, { localBackend: backend });
+        assert.ok(isResult(result), `expected result, got no-data for ${ST_R8_SPAN_COUNT} spans`);
+        if (!isResult(result))
+            return;
+        assert.ok(isFinite(result.costSavings), `costSavings should be finite, got ${result.costSavings}`);
+        assert.ok(result.costSavings >= 0, `costSavings should be non-negative, got ${result.costSavings}`);
+        assert.ok(isFinite(result.summary.fallbackRate), `fallbackRate should be finite`);
+        assert.ok(result.summary.fallbackRate >= 0 && result.summary.fallbackRate <= 1, `fallbackRate ${result.summary.fallbackRate} out of [0,1]`);
+    });
+    void it('high-volume: modelDistribution has exactly the expected unique models', async () => {
+        const spans = Array.from({ length: ST_R8_SPAN_COUNT }, (_, i) => makeStressSpan(i));
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local', limit: ST_R8_SPAN_COUNT }, { localBackend: backend });
+        assert.ok(isResult(result));
+        if (!isResult(result))
+            return;
+        // All 8 response models should appear (ST_R8_SPAN_COUNT / ST_R8_UNIQUE_MODELS = 125 each)
+        assert.strictEqual(Object.keys(result.modelDistribution).length, ST_R8_UNIQUE_MODELS, `expected ${ST_R8_UNIQUE_MODELS} unique models in distribution`);
+    });
+    void it('unknown model in pricing table: costSavings is 0 and does not throw', async () => {
+        const spans = [
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: 'unknown-future-model-xyz', inputTokens: 1000, outputTokens: 200 }),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: 'another-unknown-model', inputTokens: 1000, outputTokens: 200 }),
+        ];
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should return result (not no-data) for unknown-model spans');
+        if (!isResult(result))
+            return;
+        // Unknown models have no pricing → computeCost returns 0 → savings = max(0, 0-0) = 0
+        assert.strictEqual(result.costSavings, 0, 'cost savings should be 0 when models have no pricing entry');
+    });
+    void it('string-typed OTLP attributes: token counts parsed from strings work correctly', async () => {
+        // OTLP attributes may arrive as strings rather than numbers in some SDK versions
+        const spanWithStringAttrs = {
+            traceId: 'trace-str',
+            spanId: 'span-str',
+            name: 'chat',
+            startTimeUnixNano: Date.now() * 1_000_000,
+            endTimeUnixNano: (Date.now() + 100) * 1_000_000,
+            durationMs: 100,
+            attributes: {
+                'gen_ai.request.model': MODEL_AUTO,
+                'gen_ai.response.model': MODEL_GPT4O,
+                'gen_ai.usage.input_tokens': '1000', // string-typed
+                'gen_ai.usage.output_tokens': '200', // string-typed
+                'routing.fallback_triggered': 'true', // string-typed bool
+            },
+        };
+        const backend = makeBackend([spanWithStringAttrs]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result), 'should handle string-typed OTLP attributes');
+        if (!isResult(result))
+            return;
+        // String 'true' for fallback_triggered should be parsed as true
+        assert.strictEqual(result.summary.fallbackRate, 1, 'string "true" fallback_triggered should count as fallback');
+        // Token costs should be computed correctly from string-parsed numbers
+        assert.ok(result.costSavings >= 0, 'costSavings should be non-negative with string tokens');
+    });
+    void it('identical values produce p50 = p99 (no NaN or divide-by-zero)', async () => {
+        // All spans have the same classificationTimeMs — identical values case
+        const SPAN_COUNT = 100;
+        const spans = Array.from({ length: SPAN_COUNT }, () => makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, classificationTimeMs: CLASSIFICATION_TIME_MS }));
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local', limit: SPAN_COUNT }, { localBackend: backend });
+        assert.ok(isResult(result));
+        if (!isResult(result))
+            return;
+        assert.ok(result.routingLatency !== undefined, 'routingLatency should be defined');
+        if (!result.routingLatency)
+            return;
+        assert.strictEqual(result.routingLatency.p50, CLASSIFICATION_TIME_MS, `p50 should be ${CLASSIFICATION_TIME_MS} for identical values`);
+        assert.strictEqual(result.routingLatency.p99, CLASSIFICATION_TIME_MS, `p99 should be ${CLASSIFICATION_TIME_MS} for identical values`);
+        assert.ok(isFinite(result.routingLatency.p50), 'p50 should be finite');
+        assert.ok(isFinite(result.routingLatency.p99), 'p99 should be finite');
+    });
+    void it('single-span array returns the span value for p50 and p99', async () => {
+        const SINGLE_MS = 77;
+        const backend = makeBackend([
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, classificationTimeMs: SINGLE_MS }),
+        ]);
+        const result = await routingTelemetry({ backend: 'local' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        if (!isResult(result))
+            return;
+        assert.ok(result.routingLatency !== undefined);
+        assert.strictEqual(result.routingLatency?.p50, SINGLE_MS, `p50 should equal the single value (${SINGLE_MS})`);
+        assert.strictEqual(result.routingLatency?.p99, SINGLE_MS, `p99 should equal the single value (${SINGLE_MS})`);
+    });
+    void it('cost savings accuracy: cheaper model saves the expected amount vs expensive baseline', async () => {
+        const PRICING_GPT4O = MODEL_PRICING[MODEL_GPT4O];
+        const PRICING_HAIKU = MODEL_PRICING[MODEL_CLAUDE_HAIKU];
+        if (!PRICING_GPT4O || !PRICING_HAIKU) {
+            // Skip if either model is not in pricing table
+            return;
+        }
+        const INPUT_TOKENS = 10_000;
+        const OUTPUT_TOKENS = 2_000;
+        const CHEAP_SPAN_COUNT = 100;
+        // Most spans route 'auto' → haiku (cheap); one span routes 'auto' → gpt-4o (expensive baseline).
+        // Cost savings are computed vs the most expensive response model per request model.
+        // Both are routed (auto ≠ haiku, auto ≠ gpt-4o), so both are included in routedSpans.
+        const spans = [
+            ...Array.from({ length: CHEAP_SPAN_COUNT }, () => makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_CLAUDE_HAIKU, inputTokens: INPUT_TOKENS, outputTokens: OUTPUT_TOKENS })),
+            makeSpan({ requestModel: MODEL_AUTO, responseModel: MODEL_GPT4O, inputTokens: INPUT_TOKENS, outputTokens: OUTPUT_TOKENS }),
+        ];
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local', limit: CHEAP_SPAN_COUNT + 1 }, { localBackend: backend });
+        assert.ok(isResult(result));
+        if (!isResult(result))
+            return;
+        // With haiku cheaper than gpt-4o, savings should be positive
+        assert.ok(result.costSavings > 0, `cost savings should be positive when routing to cheaper model, got ${result.costSavings}`);
+    });
+    void it('high cardinality: 20 unique (request, response) model pairs produce non-empty groups', async () => {
+        const UNIQUE_PAIRS = 20;
+        const spans = Array.from({ length: UNIQUE_PAIRS * 50 }, (_, i) => {
+            const pairIdx = Math.floor(i / 50);
+            return makeSpan({
+                requestModel: `request-model-${pairIdx}`,
+                responseModel: `response-model-${pairIdx}`,
+                inputTokens: 1000,
+                outputTokens: 200,
+            });
+        });
+        const backend = makeBackend(spans);
+        const result = await routingTelemetry({ backend: 'local', limit: UNIQUE_PAIRS * 50, groupBy: 'model_pair' }, { localBackend: backend });
+        assert.ok(isResult(result));
+        if (!isResult(result))
+            return;
+        assert.ok(result.groups.length > 0, 'groups should be non-empty for high cardinality inputs');
+        assert.ok(isFinite(result.costSavings), 'costSavings should be finite for high cardinality');
+    });
+});
+//# sourceMappingURL=routing-telemetry.test.js.map