@peakinfer/cli 1.0.133

package/tests/counterfactuals.test.ts

@@ -0,0 +1,347 @@
+import { describe, it, expect } from 'vitest';
+import {
+  generateCounterfactuals,
+  formatCounterfactualSummary,
+  hasSignificantOpportunities,
+  rankCounterfactuals,
+} from '../src/counterfactuals.js';
+import type { Callsite, InferenceMap } from '../src/types.js';
+
+// =============================================================================
+// TEST FIXTURES
+// =============================================================================
+
+function createCallsite(overrides: Partial<Callsite> = {}): Callsite {
+  return {
+    id: `cs_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`,
+    file: 'src/agent.ts',
+    line: 42,
+    col: 10,
+    provider: 'openai',
+    model: 'gpt-4o',
+    framework: null,
+    runtime: null,
+    patterns: {},
+    confidence: 0.9,
+    ...overrides,
+  };
+}
+
+function createInferenceMap(callsites: Callsite[]): InferenceMap {
+  return {
+    version: '0.1',
+    callsites,
+    frameworks: {},
+    scanResult: {
+      root: '/test',
+      files: [],
+      summary: { totalFiles: 0, totalLoc: 0, languages: [], totalCandidates: 0 },
+    },
+  };
+}
+
+// =============================================================================
+// TESTS
+// =============================================================================
+
+describe('counterfactuals', () => {
+  describe('generateCounterfactuals', () => {
+    it('should generate model swap counterfactuals for GPT-4', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4' }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      const modelSwaps = result.counterfactuals.filter(cf => cf.type === 'model_swap');
+      expect(modelSwaps.length).toBeGreaterThan(0);
+
+      // Should suggest gpt-4o and gpt-4o-mini as alternatives
+      const suggestedModels = modelSwaps.map(cf => cf.proposedState.model);
+      expect(suggestedModels).toContain('gpt-4o');
+    });
+
+    it('should generate model swap counterfactuals for Claude Opus', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'claude-3-opus' }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      const modelSwaps = result.counterfactuals.filter(cf => cf.type === 'model_swap');
+      expect(modelSwaps.length).toBeGreaterThan(0);
+
+      // Should suggest Claude Sonnet/Haiku as alternatives
+      const suggestedModels = modelSwaps.map(cf => cf.proposedState.model);
+      expect(suggestedModels.some(m => m.includes('sonnet') || m.includes('haiku'))).toBe(true);
+    });
+
+    it('should generate batching counterfactuals for multiple unbatched callsites', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4o', patterns: {} }),
+        createCallsite({ file: 'src/b.ts', line: 20, model: 'gpt-4o', patterns: {} }),
+        createCallsite({ file: 'src/c.ts', line: 30, model: 'gpt-4o', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      const batchingCfs = result.counterfactuals.filter(cf => cf.type === 'batch_optimization');
+      expect(batchingCfs.length).toBeGreaterThan(0);
+    });
+
+    it('should not generate batching counterfactuals for single callsite', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4o', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      const batchingCfs = result.counterfactuals.filter(cf => cf.type === 'batch_optimization');
+      expect(batchingCfs.length).toBe(0);
+    });
+
+    it('should generate caching counterfactuals for uncached callsites', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      const cachingCfs = result.counterfactuals.filter(cf => cf.type === 'cache_addition');
+      expect(cachingCfs.length).toBe(1);
+    });
+
+    it('should not generate caching counterfactuals for cached callsites', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, patterns: { caching: true } }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      const cachingCfs = result.counterfactuals.filter(cf => cf.type === 'cache_addition');
+      expect(cachingCfs.length).toBe(0);
+    });
+
+    it('should generate streaming counterfactuals for non-streaming callsites', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      const streamingCfs = result.counterfactuals.filter(cf => cf.type === 'streaming_enable');
+      expect(streamingCfs.length).toBe(1);
+    });
+
+    it('should not generate streaming counterfactuals for streaming callsites', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, patterns: { streaming: true } }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      const streamingCfs = result.counterfactuals.filter(cf => cf.type === 'streaming_enable');
+      expect(streamingCfs.length).toBe(0);
+    });
+
+    it('should calculate impact for each counterfactual', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      for (const cf of result.counterfactuals) {
+        expect(cf.impact).toBeDefined();
+        expect(cf.impact.latencyDeltaPercent).toBeDefined();
+        expect(cf.impact.costDeltaPercent).toBeDefined();
+      }
+    });
+
+    it('should include tradeoffs for each counterfactual', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      for (const cf of result.counterfactuals) {
+        expect(cf.impact.tradeoffs).toBeDefined();
+        expect(Array.isArray(cf.impact.tradeoffs)).toBe(true);
+      }
+    });
+
+    it('should handle empty inference map', () => {
+      const inferenceMap = createInferenceMap([]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      expect(result.counterfactuals.length).toBe(0);
+      expect(result.summary.totalOpportunities).toBe(0);
+    });
+
+    it('should calculate summary correctly', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      expect(result.summary.totalOpportunities).toBe(result.counterfactuals.length);
+      expect(result.summary.maxLatencySavingsPercent).toBeGreaterThanOrEqual(0);
+    });
+
+    it('should include affected points for each counterfactual', () => {
+      const callsite1 = createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4' });
+      const inferenceMap = createInferenceMap([callsite1]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      for (const cf of result.counterfactuals) {
+        expect(cf.affectedPoints).toBeDefined();
+        expect(Array.isArray(cf.affectedPoints)).toBe(true);
+      }
+    });
+  });
+
+  describe('formatCounterfactualSummary', () => {
+    it('should format summary with opportunity count', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+      const summary = formatCounterfactualSummary(result);
+
+      expect(summary).toContain('optimization opportunities');
+    });
+
+    it('should include latency savings in summary', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+      const summary = formatCounterfactualSummary(result);
+
+      expect(summary).toContain('latency savings');
+    });
+  });
+
+  describe('hasSignificantOpportunities', () => {
+    it('should return true when high-impact opportunities exist', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      expect(hasSignificantOpportunities(result)).toBe(true);
+    });
+
+    it('should return false when no opportunities', () => {
+      const inferenceMap = createInferenceMap([]);
+
+      const result = generateCounterfactuals(inferenceMap);
+
+      expect(hasSignificantOpportunities(result)).toBe(false);
+    });
+  });
+
+  describe('rankCounterfactuals', () => {
+    it('should rank by latency when priority is latency', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4', patterns: {} }),
+        createCallsite({ file: 'src/b.ts', line: 20, model: 'gpt-4', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+      const ranked = rankCounterfactuals(result, 'latency');
+
+      // Should be sorted by latency delta (most negative first)
+      for (let i = 1; i < ranked.length; i++) {
+        expect(ranked[i].impact.latencyDeltaPercent)
+          .toBeGreaterThanOrEqual(ranked[i - 1].impact.latencyDeltaPercent);
+      }
+    });
+
+    it('should rank by cost when priority is cost', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4', patterns: {} }),
+        createCallsite({ file: 'src/b.ts', line: 20, model: 'gpt-4', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+      const ranked = rankCounterfactuals(result, 'cost');
+
+      // Should be sorted by cost delta (most negative first)
+      for (let i = 1; i < ranked.length; i++) {
+        expect(ranked[i].impact.costDeltaPercent)
+          .toBeGreaterThanOrEqual(ranked[i - 1].impact.costDeltaPercent);
+      }
+    });
+
+    it('should rank by combined score when priority is balanced', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4', patterns: {} }),
+        createCallsite({ file: 'src/b.ts', line: 20, model: 'gpt-4', patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+      const ranked = rankCounterfactuals(result, 'balanced');
+
+      // Should be sorted by combined score (latency + cost)
+      for (let i = 1; i < ranked.length; i++) {
+        const scorePrev = ranked[i - 1].impact.latencyDeltaPercent + ranked[i - 1].impact.costDeltaPercent;
+        const scoreCurr = ranked[i].impact.latencyDeltaPercent + ranked[i].impact.costDeltaPercent;
+        expect(scoreCurr).toBeGreaterThanOrEqual(scorePrev);
+      }
+    });
+  });
+
+  describe('counterfactual types', () => {
+    it('should generate model_swap with correct structure', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4' }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+      const modelSwap = result.counterfactuals.find(cf => cf.type === 'model_swap');
+
+      expect(modelSwap).toBeDefined();
+      expect(modelSwap?.currentState.model).toBe('gpt-4');
+      expect(modelSwap?.proposedState.model).toBeDefined();
+      expect(modelSwap?.confidence).toBeDefined();
+      expect(modelSwap?.effort).toBeDefined();
+    });
+
+    it('should generate cache_addition with correct structure', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+      const caching = result.counterfactuals.find(cf => cf.type === 'cache_addition');
+
+      expect(caching).toBeDefined();
+      expect(caching?.currentState.pattern).toBe('no caching');
+      expect(caching?.proposedState.pattern).toBe('semantic cache');
+      expect(caching?.impact.costDeltaPercent).toBe(-50);
+    });
+
+    it('should generate streaming_enable with correct structure', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, patterns: {} }),
+      ]);
+
+      const result = generateCounterfactuals(inferenceMap);
+      const streaming = result.counterfactuals.find(cf => cf.type === 'streaming_enable');
+
+      expect(streaming).toBeDefined();
+      expect(streaming?.currentState.pattern).toBe('synchronous');
+      expect(streaming?.proposedState.pattern).toBe('streaming');
+      expect(streaming?.impact.latencyDeltaPercent).toBe(-80);
+      expect(streaming?.impact.costDeltaPercent).toBe(0); // Streaming doesn't affect cost
+    });
+  });
+});

package/tests/fixtures/events/missing-id.jsonl

@@ -0,0 +1 @@
+{"ts":"2024-01-01","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420}

package/tests/fixtures/events/missing-input.jsonl

@@ -0,0 +1 @@
+{"id":"1","ts":"2024-01-01","provider":"openai","model":"gpt-4o","output_tokens":50,"latency_ms":420}

package/tests/fixtures/events/missing-latency.jsonl

@@ -0,0 +1 @@
+{"id":"1","ts":"2024-01-01","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50}

package/tests/fixtures/events/missing-model.jsonl

@@ -0,0 +1 @@
+{"id":"1","ts":"2024-01-01","provider":"openai","input_tokens":100,"output_tokens":50,"latency_ms":420}

package/tests/fixtures/events/missing-output.jsonl

@@ -0,0 +1 @@
+{"id":"1","ts":"2024-01-01","provider":"openai","model":"gpt-4o","input_tokens":100,"latency_ms":420}

package/tests/fixtures/events/missing-provider.jsonl

@@ -0,0 +1 @@
+{"id":"1","ts":"2024-01-01","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420}

package/tests/fixtures/events/missing-ts.jsonl

@@ -0,0 +1 @@
+{"id":"1","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420}

package/tests/fixtures/events/valid.csv

@@ -0,0 +1,3 @@
+id,ts,provider,model,input_tokens,output_tokens,latency_ms
+1,2024-01-01T00:00:00Z,openai,gpt-4o,100,50,420
+2,2024-01-01T00:01:00Z,openai,gpt-4o,200,80,580

package/tests/fixtures/events/valid.json

@@ -0,0 +1 @@
+[{"id":"1","ts":"2024-01-01T00:00:00Z","provider":"anthropic","model":"claude-3-sonnet-20240229","input_tokens":150,"output_tokens":60,"latency_ms":350},{"id":"2","ts":"2024-01-01T00:01:00Z","provider":"anthropic","model":"claude-3-sonnet-20240229","input_tokens":180,"output_tokens":90,"latency_ms":400}]

package/tests/fixtures/events/valid.jsonl

@@ -0,0 +1,2 @@
+{"id":"1","ts":"2024-01-01T00:00:00Z","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420}
+{"id":"2","ts":"2024-01-01T00:01:00Z","provider":"openai","model":"gpt-4o","input_tokens":200,"output_tokens":80,"latency_ms":580}

package/tests/fixtures/events/with-callsite.jsonl

@@ -0,0 +1 @@
+{"id":"1","ts":"2024-01-01T00:00:00Z","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420,"callsite_id":"cs_001"}

package/tests/fixtures/events/with-intent.jsonl

@@ -0,0 +1 @@
+{"id":"1","ts":"2024-01-01T00:00:00Z","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":420,"intent":"chat"}

package/tests/fixtures/events/wrong-type.jsonl

@@ -0,0 +1 @@
+{"id":"1","ts":"2024-01-01","provider":"openai","model":"gpt-4o","input_tokens":100,"output_tokens":50,"latency_ms":"fast"}

package/tests/fixtures/repos/hybrid-router/router.py

@@ -0,0 +1,35 @@
+from openai import OpenAI
+from anthropic import Anthropic
+import os
+
+openai_client = OpenAI()
+anthropic_client = Anthropic()
+
+PROVIDER = os.getenv("LLM_PROVIDER", "openai")
+
+def route_completion(prompt: str, use_cache: bool = True) -> str:
+    if PROVIDER == "anthropic":
+        return _anthropic_completion(prompt)
+    return _openai_completion(prompt)
+
+def _openai_completion(prompt: str) -> str:
+    response = openai_client.chat.completions.create(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": prompt}],
+        stream=True
+    )
+    return response.choices[0].message.content
+
+def _anthropic_completion(prompt: str) -> str:
+    response = anthropic_client.messages.create(
+        model="claude-3-sonnet-20240229",
+        max_tokens=1024,
+        messages=[{"role": "user", "content": prompt}]
+    )
+    return response.content[0].text
+
+def with_fallback(prompt: str) -> str:
+    try:
+        return _openai_completion(prompt)
+    except Exception:
+        return _anthropic_completion(prompt)

package/tests/fixtures/repos/saas-anthropic/agent.ts

@@ -0,0 +1,27 @@
+import Anthropic from "@anthropic-ai/sdk";
+
+const client = new Anthropic();
+
+export async function runAgent(query: string): Promise<string> {
+  const response = await client.messages.create({
+    model: "claude-3-sonnet-20240229",
+    max_tokens: 1024,
+    messages: [{ role: "user", content: query }],
+  });
+
+  return response.content[0].type === "text" ? response.content[0].text : "";
+}
+
+export async function streamResponse(query: string): AsyncGenerator<string> {
+  const stream = await client.messages.stream({
+    model: "claude-3-opus-20240229",
+    max_tokens: 4096,
+    messages: [{ role: "user", content: query }],
+  });
+
+  for await (const chunk of stream) {
+    if (chunk.type === "content_block_delta") {
+      yield chunk.delta.text;
+    }
+  }
+}

package/tests/fixtures/repos/saas-openai/assistant.js

@@ -0,0 +1,33 @@
+const OpenAI = require('openai');
+
+const client = new OpenAI();
+
+async function chat(userMessage) {
+  const response = await client.chat.completions.create({
+    model: 'gpt-4o',
+    messages: [
+      { role: 'system', content: 'You are a helpful assistant.' },
+      { role: 'user', content: userMessage }
+    ],
+    stream: true
+  });
+
+  let fullResponse = '';
+  for await (const chunk of response) {
+    const delta = chunk.choices[0]?.delta?.content || '';
+    fullResponse += delta;
+    process.stdout.write(delta);
+  }
+
+  return fullResponse;
+}
+
+async function embed(text) {
+  const response = await client.embeddings.create({
+    model: 'text-embedding-ada-002',
+    input: text
+  });
+  return response.data[0].embedding;
+}
+
+module.exports = { chat, embed };

package/tests/fixtures/repos/saas-openai/client.py

@@ -0,0 +1,26 @@
+from openai import OpenAI
+
+client = OpenAI()
+
+def chat_completion(prompt: str) -> str:
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": prompt}],
+        stream=True
+    )
+    return response.choices[0].message.content
+
+def batch_embeddings(texts: list[str]) -> list[list[float]]:
+    response = client.embeddings.create(
+        model="text-embedding-3-small",
+        input=texts
+    )
+    return [item.embedding for item in response.data]
+
+async def async_completion(prompt: str) -> str:
+    response = await client.chat.completions.create(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": prompt}],
+        max_retries=3
+    )
+    return response.choices[0].message.content

package/tests/fixtures/repos/self-hosted-vllm/inference.py

@@ -0,0 +1,22 @@
+from openai import OpenAI
+
+# vLLM OpenAI-compatible server
+client = OpenAI(
+    base_url="http://localhost:8000/v1",
+    api_key="dummy"
+)
+
+def generate(prompt: str, max_tokens: int = 512) -> str:
+    response = client.completions.create(
+        model="meta-llama/Llama-3-70b-chat-hf",
+        prompt=prompt,
+        max_tokens=max_tokens,
+        temperature=0.7
+    )
+    return response.choices[0].text
+
+def batch_generate(prompts: list[str]) -> list[str]:
+    results = []
+    for prompt in prompts:
+        results.append(generate(prompt))
+    return results