@peakinfer/cli 1.0.133

package/tests/runtime-analyzer.test.ts

@@ -0,0 +1,375 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import { RuntimeAnalyzerAgent, type RuntimeAnalyzerInput } from '../src/agents/runtime-analyzer.js';
+import { setTestPricing } from '../src/costs.js';
+import type { InferenceEvent, RuntimeSummary } from '../src/types.js';
+
+// =============================================================================
+// TEST FIXTURES
+// =============================================================================
+
+beforeAll(() => {
+  setTestPricing({
+    'gpt-4o': { input: 5.0, output: 15.0 },
+    'gpt-4o-mini': { input: 0.15, output: 0.6 },
+    'claude-3-opus': { input: 15.0, output: 75.0 },
+    'claude-3-5-sonnet': { input: 3.0, output: 15.0 },
+  });
+});
+
+const makeEvent = (overrides: Partial<InferenceEvent> = {}): InferenceEvent => ({
+  id: `evt_${Math.random().toString(36).slice(2, 9)}`,
+  ts: '2024-01-01T00:00:00Z',
+  provider: 'openai',
+  model: 'gpt-4o',
+  input_tokens: 100,
+  output_tokens: 50,
+  latency_ms: 420,
+  ...overrides,
+});
+
+const makeRuntimeSummary = (overrides: Partial<RuntimeSummary> = {}): RuntimeSummary => ({
+  totalEvents: 100,
+  byProvider: {
+    openai: {
+      calls: 80,
+      tokens_in: 8000,
+      tokens_out: 4000,
+      latency_p50: 400,
+      latency_p95: 800,
+      latency_p99: 1200,
+    },
+    anthropic: {
+      calls: 20,
+      tokens_in: 2000,
+      tokens_out: 1000,
+      latency_p50: 500,
+      latency_p95: 1000,
+      latency_p99: 1500,
+    },
+  },
+  byModel: {
+    'gpt-4o': {
+      calls: 60,
+      tokens_in: 6000,
+      tokens_out: 3000,
+      latency_p50: 350,
+      latency_p95: 700,
+      latency_p99: 1100,
+    },
+    'gpt-4o-mini': {
+      calls: 20,
+      tokens_in: 2000,
+      tokens_out: 1000,
+      latency_p50: 200,
+      latency_p95: 400,
+      latency_p99: 600,
+    },
+    'claude-3-5-sonnet': {
+      calls: 20,
+      tokens_in: 2000,
+      tokens_out: 1000,
+      latency_p50: 500,
+      latency_p95: 1000,
+      latency_p99: 1500,
+    },
+  },
+  global: {
+    p50: 400,
+    p95: 800,
+    p99: 1200,
+  },
+  ...overrides,
+});
+
+const makeInput = (overrides: Partial<RuntimeAnalyzerInput> = {}): RuntimeAnalyzerInput => ({
+  events: Array.from({ length: 10 }, (_, i) => makeEvent({ id: `evt_${i}` })),
+  runtimeSummary: makeRuntimeSummary(),
+  ...overrides,
+});
+
+// =============================================================================
+// AGENT PROPERTIES TESTS
+// =============================================================================
+
+describe('RuntimeAnalyzerAgent', () => {
+  describe('agent properties', () => {
+    it('has correct name', () => {
+      expect(RuntimeAnalyzerAgent.name).toBe('runtime-analyzer');
+    });
+
+    it('has description', () => {
+      expect(RuntimeAnalyzerAgent.description).toBeDefined();
+      expect(RuntimeAnalyzerAgent.description.length).toBeGreaterThan(0);
+    });
+
+    it('has tools registry', () => {
+      expect(RuntimeAnalyzerAgent.tools).toBeDefined();
+    });
+  });
+
+  // =============================================================================
+  // EXECUTION TESTS
+  // =============================================================================
+
+  describe('execute', () => {
+    it('returns valid output structure', async () => {
+      const input = makeInput();
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      // Check insights array
+      expect(Array.isArray(result.insights)).toBe(true);
+
+      // Check detectedPatterns structure
+      expect(result.detectedPatterns).toHaveProperty('applicationType');
+      expect(result.detectedPatterns).toHaveProperty('multiModelPipeline');
+      expect(result.detectedPatterns).toHaveProperty('streamingDetected');
+      expect(result.detectedPatterns).toHaveProperty('batchingDetected');
+      expect(result.detectedPatterns).toHaveProperty('cachingDetected');
+
+      // Check summary structure
+      expect(result.summary).toHaveProperty('totalCalls');
+      expect(result.summary).toHaveProperty('totalTokens');
+      expect(result.summary).toHaveProperty('dominantProvider');
+      expect(result.summary).toHaveProperty('dominantModel');
+      expect(result.summary).toHaveProperty('estimatedDailyCostUSD');
+    });
+
+    it('application type is valid enum value', async () => {
+      const input = makeInput();
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      const validTypes = ['rag', 'agent', 'batch', 'chat', 'pipeline', 'unknown'];
+      expect(validTypes).toContain(result.detectedPatterns.applicationType);
+    });
+
+    it('correctly detects multi-model pipeline', async () => {
+      const input = makeInput({
+        runtimeSummary: makeRuntimeSummary({
+          byModel: {
+            'gpt-4o': { calls: 50, tokens_in: 5000, tokens_out: 2500, latency_p50: 400, latency_p95: 800, latency_p99: 1200 },
+            'gpt-4o-mini': { calls: 50, tokens_in: 5000, tokens_out: 2500, latency_p50: 200, latency_p95: 400, latency_p99: 600 },
+          },
+        }),
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      expect(result.detectedPatterns.multiModelPipeline).toBe(true);
+    });
+
+    it('detects streaming pattern from events', async () => {
+      const input = makeInput({
+        events: [
+          makeEvent({ streaming: true }),
+          makeEvent({ streaming: true }),
+          makeEvent({ streaming: false }),
+        ],
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      expect(result.detectedPatterns.streamingDetected).toBe(true);
+    });
+
+    it('detects batching pattern from events', async () => {
+      const input = makeInput({
+        events: [
+          makeEvent({ batch_id: 'batch_001' }),
+          makeEvent({ batch_id: 'batch_001' }),
+          makeEvent(),
+        ],
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      expect(result.detectedPatterns.batchingDetected).toBe(true);
+    });
+
+    it('detects caching pattern from events', async () => {
+      const input = makeInput({
+        events: [
+          makeEvent({ cached: true }),
+          makeEvent({ cached: false }),
+        ],
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      expect(result.detectedPatterns.cachingDetected).toBe(true);
+    });
+
+    it('calculates total tokens', async () => {
+      const runtimeSummary = makeRuntimeSummary({
+        byModel: {
+          'gpt-4o': { calls: 10, tokens_in: 1000, tokens_out: 500, latency_p50: 400, latency_p95: 800, latency_p99: 1200 },
+        },
+      });
+
+      const input = makeInput({ runtimeSummary });
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      // Total tokens should be a reasonable positive number
+      // LLM may calculate from events or summary differently
+      expect(result.summary.totalTokens).toBeGreaterThan(0);
+    });
+
+    it('identifies dominant provider', async () => {
+      const runtimeSummary = makeRuntimeSummary({
+        byProvider: {
+          openai: { calls: 100, tokens_in: 10000, tokens_out: 5000, latency_p50: 400, latency_p95: 800, latency_p99: 1200 },
+          anthropic: { calls: 20, tokens_in: 2000, tokens_out: 1000, latency_p50: 500, latency_p95: 1000, latency_p99: 1500 },
+        },
+      });
+
+      const input = makeInput({ runtimeSummary });
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      expect(result.summary.dominantProvider).toBe('openai');
+    });
+
+    it('identifies dominant model', async () => {
+      const runtimeSummary = makeRuntimeSummary({
+        byModel: {
+          'gpt-4o': { calls: 80, tokens_in: 8000, tokens_out: 4000, latency_p50: 400, latency_p95: 800, latency_p99: 1200 },
+          'gpt-4o-mini': { calls: 20, tokens_in: 2000, tokens_out: 1000, latency_p50: 200, latency_p95: 400, latency_p99: 600 },
+        },
+      });
+
+      const input = makeInput({ runtimeSummary });
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      expect(result.summary.dominantModel).toBe('gpt-4o');
+    });
+  });
+
+  // =============================================================================
+  // EDGE CASES
+  // =============================================================================
+
+  describe('edge cases', () => {
+    it('handles empty events array', async () => {
+      const input = makeInput({
+        events: [],
+        runtimeSummary: makeRuntimeSummary({ totalEvents: 0 }),
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      expect(result.detectedPatterns.streamingDetected).toBe(false);
+      expect(result.detectedPatterns.batchingDetected).toBe(false);
+      expect(result.detectedPatterns.cachingDetected).toBe(false);
+    });
+
+    it('handles single model', async () => {
+      const input = makeInput({
+        runtimeSummary: makeRuntimeSummary({
+          byModel: {
+            'gpt-4o': { calls: 100, tokens_in: 10000, tokens_out: 5000, latency_p50: 400, latency_p95: 800, latency_p99: 1200 },
+          },
+        }),
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      // LLM may still detect patterns; key assertion: returns valid result
+      expect(result.detectedPatterns).toBeDefined();
+      expect(result.summary.dominantModel).toBeDefined();
+    });
+
+    it('handles missing optional event fields', async () => {
+      const input = makeInput({
+        events: [makeEvent()], // No streaming, batch_id, cached, retry_count, fallback_used
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      expect(result.detectedPatterns.streamingDetected).toBe(false);
+      expect(result.detectedPatterns.batchingDetected).toBe(false);
+      expect(result.detectedPatterns.cachingDetected).toBe(false);
+    });
+
+    it('handles empty byProvider and byModel', async () => {
+      const input = makeInput({
+        runtimeSummary: {
+          totalEvents: 0,
+          byProvider: {},
+          byModel: {},
+          global: { p50: 0, p95: 0, p99: 0 },
+        },
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      // LLM may return 'unknown', 'none', 'n/a' etc for empty data
+      expect(['unknown', 'none', 'n/a', '']).toContain(result.summary.dominantProvider.toLowerCase());
+      expect(['unknown', 'none', 'n/a', '']).toContain(result.summary.dominantModel.toLowerCase());
+    });
+  });
+
+  // =============================================================================
+  // PATTERN DETECTION COMBINATIONS
+  // =============================================================================
+
+  describe('pattern detection combinations', () => {
+    it('detects all patterns when present', async () => {
+      const input = makeInput({
+        events: [
+          makeEvent({ streaming: true, batch_id: 'b1', cached: true }),
+        ],
+        runtimeSummary: makeRuntimeSummary({
+          byModel: {
+            'gpt-4o': { calls: 50, tokens_in: 5000, tokens_out: 2500, latency_p50: 400, latency_p95: 800, latency_p99: 1200 },
+            'claude-3-5-sonnet': { calls: 50, tokens_in: 5000, tokens_out: 2500, latency_p50: 500, latency_p95: 1000, latency_p99: 1500 },
+          },
+        }),
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      expect(result.detectedPatterns.streamingDetected).toBe(true);
+      expect(result.detectedPatterns.batchingDetected).toBe(true);
+      expect(result.detectedPatterns.cachingDetected).toBe(true);
+      expect(result.detectedPatterns.multiModelPipeline).toBe(true);
+    });
+
+    it('returns valid patterns structure for minimal input', async () => {
+      const input = makeInput({
+        events: [makeEvent()], // No optional fields
+        runtimeSummary: makeRuntimeSummary({
+          byModel: {
+            'gpt-4o': { calls: 100, tokens_in: 10000, tokens_out: 5000, latency_p50: 400, latency_p95: 800, latency_p99: 1200 },
+          },
+        }),
+      });
+
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      // LLM may infer patterns from context; key assertion: returns valid structure
+      expect(typeof result.detectedPatterns.streamingDetected).toBe('boolean');
+      expect(typeof result.detectedPatterns.batchingDetected).toBe('boolean');
+      expect(typeof result.detectedPatterns.cachingDetected).toBe('boolean');
+      expect(typeof result.detectedPatterns.multiModelPipeline).toBe('boolean');
+    });
+  });
+
+  // =============================================================================
+  // INSIGHTS VALIDATION (when LLM is available)
+  // =============================================================================
+
+  describe('insights structure', () => {
+    it('insights have required fields when generated', async () => {
+      const input = makeInput();
+      const { result } = await RuntimeAnalyzerAgent.execute(input);
+
+      // Insights may be empty in fallback mode, but if present, must have structure
+      for (const insight of result.insights) {
+        expect(insight).toHaveProperty('id');
+        expect(insight).toHaveProperty('severity');
+        expect(insight).toHaveProperty('category');
+        expect(insight).toHaveProperty('headline');
+        expect(insight).toHaveProperty('evidence');
+        expect(['critical', 'warning', 'info']).toContain(insight.severity);
+      }
+    });
+  });
+});

package/tests/runtime.test.ts

@@ -0,0 +1,205 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { parseEvents, aggregate, percentile } from '../src/runtime.js';
+import { writeFileSync, mkdirSync, rmSync } from 'fs';
+import { join } from 'path';
+
+const FIXTURES_DIR = join(__dirname, 'fixtures', 'events');
+
+// Ensure fixtures directory exists
+beforeEach(() => {
+  mkdirSync(FIXTURES_DIR, { recursive: true });
+});
+
+describe('runtime parser', () => {
+  describe('JSONL parsing', () => {
+    it('parses valid JSONL', async () => {
+      const content = [
+        '{"id":"1","ts":"2024-01-01T00:00:00Z","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420}',
+        '{"id":"2","ts":"2024-01-01T00:01:00Z","provider":"openai","model":"gpt-4o","input_tokens":200,"output_tokens":80,"latency_ms":580}',
+      ].join('\n');
+
+      const path = join(FIXTURES_DIR, 'valid.jsonl');
+      writeFileSync(path, content);
+
+      const events = await parseEvents(path);
+      expect(events.length).toBe(2);
+      expect(events[0].provider).toBe('openai');
+      expect(events[1].latency_ms).toBe(580);
+    });
+  });
+
+  describe('JSON array parsing', () => {
+    it('parses valid JSON array', async () => {
+      const data = [
+        { id: '1', ts: '2024-01-01T00:00:00Z', provider: 'anthropic', model: 'claude-3-sonnet-20240229', input_tokens: 150, output_tokens: 60, latency_ms: 350 },
+        { id: '2', ts: '2024-01-01T00:01:00Z', provider: 'anthropic', model: 'claude-3-sonnet-20240229', input_tokens: 180, output_tokens: 90, latency_ms: 400 },
+      ];
+
+      const path = join(FIXTURES_DIR, 'valid.json');
+      writeFileSync(path, JSON.stringify(data));
+
+      const events = await parseEvents(path);
+      expect(events.length).toBe(2);
+      expect(events[0].provider).toBe('anthropic');
+    });
+  });
+
+  describe('CSV parsing', () => {
+    it('parses valid CSV', async () => {
+      const content = [
+        'id,ts,provider,model,input_tokens,output_tokens,latency_ms',
+        '1,2024-01-01T00:00:00Z,openai,gpt-4o,100,50,420',
+        '2,2024-01-01T00:01:00Z,openai,gpt-4o,200,80,580',
+      ].join('\n');
+
+      const path = join(FIXTURES_DIR, 'valid.csv');
+      writeFileSync(path, content);
+
+      const events = await parseEvents(path);
+      expect(events.length).toBe(2);
+      expect(events[0].input_tokens).toBe(100);
+    });
+  });
+
+  describe('validation', () => {
+    it('rejects missing id field', async () => {
+      const content = '{"ts":"2024-01-01","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420}';
+      const path = join(FIXTURES_DIR, 'missing-id.jsonl');
+      writeFileSync(path, content);
+
+      await expect(parseEvents(path)).rejects.toThrow();
+    });
+
+    it('rejects missing ts field', async () => {
+      const content = '{"id":"1","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420}';
+      const path = join(FIXTURES_DIR, 'missing-ts.jsonl');
+      writeFileSync(path, content);
+
+      await expect(parseEvents(path)).rejects.toThrow();
+    });
+
+    it('rejects missing provider field', async () => {
+      const content = '{"id":"1","ts":"2024-01-01","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420}';
+      const path = join(FIXTURES_DIR, 'missing-provider.jsonl');
+      writeFileSync(path, content);
+
+      await expect(parseEvents(path)).rejects.toThrow();
+    });
+
+    it('rejects missing model field', async () => {
+      const content = '{"id":"1","ts":"2024-01-01","provider":"openai","input_tokens":100,"output_tokens":50,"latency_ms":420}';
+      const path = join(FIXTURES_DIR, 'missing-model.jsonl');
+      writeFileSync(path, content);
+
+      await expect(parseEvents(path)).rejects.toThrow();
+    });
+
+    it('rejects missing input_tokens field', async () => {
+      const content = '{"id":"1","ts":"2024-01-01","provider":"openai","model":"gpt-4o","output_tokens":50,"latency_ms":420}';
+      const path = join(FIXTURES_DIR, 'missing-input.jsonl');
+      writeFileSync(path, content);
+
+      await expect(parseEvents(path)).rejects.toThrow();
+    });
+
+    it('rejects missing output_tokens field', async () => {
+      const content = '{"id":"1","ts":"2024-01-01","provider":"openai","model":"gpt-4o","input_tokens":100,"latency_ms":420}';
+      const path = join(FIXTURES_DIR, 'missing-output.jsonl');
+      writeFileSync(path, content);
+
+      await expect(parseEvents(path)).rejects.toThrow();
+    });
+
+    it('rejects missing latency_ms field', async () => {
+      const content = '{"id":"1","ts":"2024-01-01","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50}';
+      const path = join(FIXTURES_DIR, 'missing-latency.jsonl');
+      writeFileSync(path, content);
+
+      await expect(parseEvents(path)).rejects.toThrow();
+    });
+
+    it('rejects wrong type for latency_ms', async () => {
+      const content = '{"id":"1","ts":"2024-01-01","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":"fast"}';
+      const path = join(FIXTURES_DIR, 'wrong-type.jsonl');
+      writeFileSync(path, content);
+
+      await expect(parseEvents(path)).rejects.toThrow();
+    });
+
+    it('accepts optional intent field', async () => {
+      const content = '{"id":"1","ts":"2024-01-01T00:00:00Z","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420,"intent":"chat"}';
+      const path = join(FIXTURES_DIR, 'with-intent.jsonl');
+      writeFileSync(path, content);
+
+      const events = await parseEvents(path);
+      expect(events[0].intent).toBe('chat');
+    });
+
+    it('accepts optional callsite_id field', async () => {
+      const content = '{"id":"1","ts":"2024-01-01T00:00:00Z","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420,"callsite_id":"cs_001"}';
+      const path = join(FIXTURES_DIR, 'with-callsite.jsonl');
+      writeFileSync(path, content);
+
+      const events = await parseEvents(path);
+      expect(events[0].callsite_id).toBe('cs_001');
+    });
+  });
+
+  describe('percentile calculation', () => {
+    it('calculates p50 correctly', () => {
+      const values = [100, 200, 300, 400, 500];
+      expect(percentile(values, 50)).toBe(300);
+    });
+
+    it('calculates p95 correctly', () => {
+      const values = Array.from({ length: 100 }, (_, i) => (i + 1) * 10);
+      expect(percentile(values, 95)).toBe(950);
+    });
+
+    it('calculates p99 correctly', () => {
+      const values = Array.from({ length: 100 }, (_, i) => (i + 1) * 10);
+      expect(percentile(values, 99)).toBe(990);
+    });
+  });
+
+  describe('aggregation', () => {
+    it('aggregates by provider', () => {
+      const events = [
+        { id: '1', ts: '2024-01-01', provider: 'openai' as const, model: 'gpt-4o', input_tokens: 100, output_tokens: 50, latency_ms: 420 },
+        { id: '2', ts: '2024-01-01', provider: 'openai' as const, model: 'gpt-4o', input_tokens: 200, output_tokens: 80, latency_ms: 580 },
+        { id: '3', ts: '2024-01-01', provider: 'anthropic' as const, model: 'claude-3', input_tokens: 150, output_tokens: 60, latency_ms: 350 },
+      ];
+
+      const summary = aggregate(events);
+      expect(summary.byProvider['openai'].calls).toBe(2);
+      expect(summary.byProvider['anthropic'].calls).toBe(1);
+    });
+
+    it('aggregates by model', () => {
+      const events = [
+        { id: '1', ts: '2024-01-01', provider: 'openai' as const, model: 'gpt-4o', input_tokens: 100, output_tokens: 50, latency_ms: 420 },
+        { id: '2', ts: '2024-01-01', provider: 'openai' as const, model: 'gpt-4o-mini', input_tokens: 200, output_tokens: 80, latency_ms: 180 },
+      ];
+
+      const summary = aggregate(events);
+      expect(summary.byModel['gpt-4o'].calls).toBe(1);
+      expect(summary.byModel['gpt-4o-mini'].calls).toBe(1);
+    });
+
+    it('handles single event', () => {
+      const events = [
+        { id: '1', ts: '2024-01-01', provider: 'openai' as const, model: 'gpt-4o', input_tokens: 100, output_tokens: 50, latency_ms: 420 },
+      ];
+
+      const summary = aggregate(events);
+      expect(summary.totalEvents).toBe(1);
+      expect(summary.global.p50).toBe(420);
+    });
+
+    it('handles empty array', () => {
+      const summary = aggregate([]);
+      expect(summary.totalEvents).toBe(0);
+      expect(summary.global.p50).toBe(0);
+    });
+  });
+});