@peakinfer/cli 1.0.133

package/docs/migration-v2.md

@@ -0,0 +1,293 @@
+# Migration Guide: PeakInfer v1.8 → v2.0
+
+This guide helps you migrate from PeakInfer v1.8 to v2.0.
+
+## Overview of Changes
+
+PeakInfer v2.0 is a major architectural upgrade that improves accuracy, speed, and maintainability.
+
+| Aspect | v1.8 | v2.0 |
+|--------|------|------|
+| **Analysis Engine** | TypeScript + Regex | Claude Code Agent SDK |
+| **Callsite Discovery** | Regex patterns | Semantic code understanding |
+| **Architecture** | Multi-phase agents | Unified single-call |
+| **Templates** | 27 templates | 43 templates (12 insight + 31 optimization) |
+| **Speed** | ~70s per file | ~30s per file (60% faster) |
+| **Cost** | ~$0.05 per file | ~$0.02 per file (60% cheaper) |
+| **Pricing** | 300 credits/10 days free | 50 credits one-time (6-month expiry) |
+
+## Breaking Changes
+
+### 1. CLI Command Changes
+
+```bash
+# v1.8: Separate commands for different analyses
+peakinfer scan ./src                    # Discovery only
+peakinfer profile ./src                 # Cost/latency profiling
+peakinfer drift ./src --events file.jsonl
+
+# v2.0: Unified analyze command
+peakinfer analyze ./src                           # Full analysis
+peakinfer analyze ./src --events production.jsonl # With runtime correlation
+```
+
+### 2. Output Format Changes
+
+The InferenceMap schema has been updated to v0.1:
+
+```diff
+{
+-  "version": "0.0.1",
++  "version": "0.1",
+   "root": "./src",
+   "generatedAt": "2024-12-21T10:00:00Z",
++  "metadata": {
++    "promptId": "unified-analyzer",
++    "promptVersion": "1.6.0",
++    "llmProvider": "anthropic",
++    "llmModel": "claude-sonnet-4-20250514"
++  },
+   "summary": { ... },
+   "callsites": [
+     {
+       "id": "src/chat.ts:42",
+       "file": "src/chat.ts",
+       "line": 42,
+       "provider": "openai",
+       "model": "gpt-4o",
++      "framework": "langchain",  // NEW: Framework detection
++      "runtime": null,           // NEW: Runtime detection (vllm, tgi, etc.)
+       "patterns": {
+         "streaming": true,
+-        "retry": true,           // Renamed
++        "retries": true,         // Renamed for consistency
+         "caching": false,
+-        "error_handling": true   // Removed
++        "fallback": true         // NEW: Fallback pattern detection
+       },
+       "confidence": 0.95
+     }
+   ]
+}
+```
+
+### 3. Configuration File Changes
+
+```yaml
+# v1.8: .peakinferrc.yaml
+scan:
+  extensions: [.ts, .js, .py]
+  ignore: [node_modules, dist]
+profile:
+  include_cost: true
+  include_latency: true
+
+# v2.0: .peakinfer.yaml (new name)
+analyze:
+  extensions: [.ts, .tsx, .js, .jsx, .py]  # More extensions supported
+  ignore: [node_modules, dist, .git, __pycache__]
+  prompt: unified-analyzer  # Configurable prompt pack
+output:
+  format: text  # text, json
+  save: true    # Auto-save to .peakinfer/
+```
+
+### 4. API Key Changes
+
+```bash
+# v1.8: Used PEAKINFER_API_KEY for managed mode
+export PEAKINFER_API_KEY=pk_xxx
+
+# v2.0: BYOK mode uses your Anthropic key directly
+export ANTHROPIC_API_KEY=sk-ant-xxx
+
+# For managed mode (GitHub Action), use PEAKINFER_TOKEN
+export PEAKINFER_TOKEN=pt_xxx
+```
+
+### 5. Credit System Changes
+
+| v1.8 | v2.0 |
+|------|------|
+| 300 credits free (10-day refresh) | 50 credits free (one-time, 6-month expiry) |
+| Pro: $20/500 credits | Starter: $19/200, Growth: $49/600, Scale: $149/2000 |
+| Monthly subscription | Credit packs (no subscription) |
+
+## Migration Steps
+
+### Step 1: Update CLI
+
+```bash
+# Uninstall v1.8
+npm uninstall -g @kalmantic/peakinfer
+
+# Install v2.0
+npm install -g @kalmantic/peakinfer
+```
+
+### Step 2: Update Configuration
+
+```bash
+# Rename config file
+mv .peakinferrc.yaml .peakinfer.yaml
+
+# Update config format (see example above)
+```
+
+### Step 3: Update API Key
+
+```bash
+# Add to your shell profile (.bashrc, .zshrc, etc.)
+export ANTHROPIC_API_KEY="your-key-here"
+
+# Or create .env file
+echo "ANTHROPIC_API_KEY=your-key-here" > .env
+```
+
+### Step 4: Update Scripts
+
+```diff
+# package.json scripts
+{
+  "scripts": {
+-   "peakinfer": "peakinfer scan ./src && peakinfer profile ./src"
++   "peakinfer": "peakinfer analyze ./src"
+  }
+}
+```
+
+### Step 5: Update GitHub Action
+
+```diff
+# .github/workflows/peakinfer.yml
+- uses: kalmantic/peakinfer-action@v1
++ uses: kalmantic/peakinfer-action@v2
+  with:
+    path: ./src
+-   mode: scan-and-profile
++   # mode is now automatic - unified analysis
+    events: ./events.jsonl
++   events-map: timestamp=time,model=model_name  # NEW: Field mapping
+```
+
+### Step 6: Update CI/CD Integration
+
+```diff
+# Check output format changes in your CI scripts
+- if jq -e '.callsites[] | select(.patterns.retry == false)' output.json; then
++ if jq -e '.callsites[] | select(.patterns.retries == false)' output.json; then
+    echo "Missing retry handling detected"
+    exit 1
+  fi
+```
+
+## New Features in v2.0
+
+### 1. Framework Detection
+
+v2.0 automatically detects LLM frameworks:
+
+```json
+{
+  "id": "src/rag.ts:25",
+  "framework": "langchain",  // langchain, llamaindex, haystack, etc.
+  "runtime": null
+}
+```
+
+### 2. Self-Hosted Runtime Detection
+
+```json
+{
+  "id": "src/inference.py:42",
+  "provider": null,
+  "runtime": "vllm"  // vllm, tgi, ollama, sglang, etc.
+}
+```
+
+### 3. Field Mapping for Runtime Events
+
+Handle non-standard event formats:
+
+```bash
+# v2.0: Map custom field names
+peakinfer analyze ./src \
+  --events logs.jsonl \
+  --events-map latency_ms=duration,model=model_name,input_tokens=prompt_tokens
+```
+
+### 4. What-If Analysis
+
+```bash
+# Predict impact of model changes
+peakinfer whatif --model gpt-4o-mini
+
+# Output: "Switching 5 inference points from gpt-4o to gpt-4o-mini
+#          would reduce monthly cost by $2,340 (67% reduction)"
+```
+
+### 5. Historical Comparison
+
+```bash
+# Compare current run with baseline
+peakinfer analyze ./src --compare-baseline
+
+# Compare two specific runs
+peakinfer history compare run_abc123 run_def456
+```
+
+### 6. Latency Prediction
+
+```bash
+# Predict p95 latency based on InferenceMAX envelope data
+peakinfer analyze ./src --predict --target-p95 2000
+```
+
+## Template Migration
+
+If you have custom templates, update them to v2.0 schema:
+
+```diff
+# Template changes
+{
+  "id": "my-custom-template",
+- "type": "insight",
++ "category": "cost",        # cost, drift, performance, waste
++ "severity": "warning",     # critical, warning, info
+  "match": {
+    "scope": "callsite",
+    "conditions": [
+-     { "field": "retry", "equals": false }
++     { "field": "patterns.retries", "op": "eq", "value": false }
+    ]
+  }
+}
+```
+
+## FAQ
+
+### Q: Will my v1.8 InferenceMap files still work?
+
+A: v2.0 can read v1.8 files but will convert them on save. Consider regenerating for full v2.0 benefits.
+
+### Q: Do I need to change my Anthropic API key?
+
+A: No, the same key works. Just update the environment variable name if using the old `PEAKINFER_API_KEY`.
+
+### Q: Are v1.8 templates compatible?
+
+A: Mostly yes. Check the template schema changes above and update `match.conditions` syntax.
+
+### Q: What happened to the `scan` command?
+
+A: It's now integrated into `analyze`. Use `peakinfer analyze ./src` for all analysis types.
+
+### Q: How do I get my free credits?
+
+A: Sign in at peakinfer.com with GitHub. First-time users get 50 credits automatically.
+
+## Getting Help
+
+- **Documentation:** https://peakinfer.com/docs
+- **GitHub Issues:** https://github.com/Kalmantic/peakinfer/issues
+- **Discord:** https://discord.gg/kalmantic

package/fixtures/demo/precomputed.json

@@ -0,0 +1,142 @@
+{
+  "version": "1.0.0",
+  "generated": "2025-12-24T00:00:00.000Z",
+  "description": "Pre-computed demo analysis for offline `peakinfer demo` command",
+  "inferencePoints": [
+    {
+      "id": "demo-ai-service-chat-1",
+      "file": "ai-service.ts",
+      "line": 13,
+      "function": "chat",
+      "provider": "anthropic",
+      "model": "claude-sonnet-4-20250514",
+      "streaming": true,
+      "costProfile": {
+        "estimatedCostPer1K": 0.015,
+        "inputTokens": 500,
+        "outputTokens": 2000
+      }
+    },
+    {
+      "id": "demo-ai-service-classify-2",
+      "file": "ai-service.ts",
+      "line": 29,
+      "function": "classifyIntent",
+      "provider": "openai",
+      "model": "gpt-4",
+      "streaming": false,
+      "costProfile": {
+        "estimatedCostPer1K": 0.03,
+        "inputTokens": 100,
+        "outputTokens": 50
+      }
+    },
+    {
+      "id": "demo-ai-service-summarize-3",
+      "file": "ai-service.ts",
+      "line": 42,
+      "function": "summarize",
+      "provider": "anthropic",
+      "model": "claude-sonnet-4-20250514",
+      "streaming": false,
+      "costProfile": {
+        "estimatedCostPer1K": 0.015,
+        "inputTokens": 2000,
+        "outputTokens": 500
+      }
+    },
+    {
+      "id": "demo-ai-service-batch-4",
+      "file": "ai-service.ts",
+      "line": 52,
+      "function": "batchAnalyze",
+      "provider": "anthropic",
+      "model": "claude-sonnet-4-20250514",
+      "streaming": false,
+      "costProfile": {
+        "estimatedCostPer1K": 0.015,
+        "inputTokens": 200,
+        "outputTokens": 200
+      }
+    }
+  ],
+  "drift": {
+    "detected": true,
+    "type": "streaming",
+    "description": "Code declares streaming but runtime shows 0% actual streams",
+    "evidence": {
+      "code": "stream: true",
+      "runtime": "0% streaming responses observed",
+      "impact": "p95 latency 2.4s instead of 400ms",
+      "duration": "23 days"
+    }
+  },
+  "issues": [
+    {
+      "id": "issue-1",
+      "severity": "critical",
+      "category": "drift",
+      "title": "Streaming Drift Detected",
+      "description": "Code says `stream: true` but runtime shows 0% streaming responses",
+      "impact": "6x slower response times, poor user experience",
+      "file": "ai-service.ts",
+      "line": 15,
+      "fix": {
+        "description": "Verify streaming is working end-to-end",
+        "effort": "1 hour"
+      }
+    },
+    {
+      "id": "issue-2",
+      "severity": "high",
+      "category": "cost",
+      "title": "Overpowered Model for Classification",
+      "description": "GPT-4 ($0.03/1K) used for simple intent classification that GPT-4o-mini ($0.00015/1K) handles equally well",
+      "impact": "200x cost waste on this endpoint",
+      "savings": "$4,200/month",
+      "file": "ai-service.ts",
+      "line": 30,
+      "fix": {
+        "description": "Switch to gpt-4o-mini for classification",
+        "effort": "5 minutes"
+      }
+    },
+    {
+      "id": "issue-3",
+      "severity": "high",
+      "category": "reliability",
+      "title": "No Error Handling",
+      "description": "LLM call has no try/catch, no retry logic, no timeout",
+      "impact": "Single API failure crashes the application",
+      "file": "ai-service.ts",
+      "line": 42,
+      "fix": {
+        "description": "Add retry with exponential backoff and timeout",
+        "effort": "30 minutes"
+      }
+    },
+    {
+      "id": "issue-4",
+      "severity": "medium",
+      "category": "throughput",
+      "title": "Sequential Batch Processing",
+      "description": "Items processed one at a time in a loop instead of parallel",
+      "impact": "50x throughput loss for batch operations",
+      "file": "ai-service.ts",
+      "line": 51,
+      "fix": {
+        "description": "Use Promise.all() for parallel processing",
+        "effort": "15 minutes"
+      }
+    }
+  ],
+  "summary": {
+    "totalInferencePoints": 4,
+    "providers": ["anthropic", "openai"],
+    "criticalIssues": 1,
+    "highIssues": 2,
+    "mediumIssues": 1,
+    "estimatedMonthlySavings": "$4,200",
+    "estimatedLatencyImprovement": "6x faster"
+  }
+}

package/fixtures/demo-project/README.md

@@ -0,0 +1,52 @@
+# PeakInfer v2.0 Demo Project
+
+This is a sample project for demonstrating PeakInfer v2.0 features.
+
+## Files
+
+- `src/llm-client.ts` - Sample LLM client with various inference patterns
+- `sample-events.jsonl` - Sample runtime events for combined analysis
+
+## Quick Demo
+
+### 1. Basic Analysis
+```bash
+peakinfer analyze .
+```
+
+### 2. With Comparison
+```bash
+# Run again to see comparison
+peakinfer analyze . --compare
+```
+
+### 3. With Prediction
+```bash
+peakinfer analyze . --predict --target-p95 3000
+```
+
+### 4. Combined Analysis (Static + Runtime)
+```bash
+peakinfer analyze . --events sample-events.jsonl --compare --predict --html --open
+```
+
+## Expected Results
+
+### Predictions
+- `chatWithGPT4`: HIGH risk (p95 ~5000ms)
+- `analyzeWithOpus`: HIGH risk (p95 ~8000ms)
+- `summarize`: MEDIUM risk (p95 ~4000ms)
+- `quickChat`: LOW risk (p95 ~1500ms)
+- `fastResponse`: LOW risk (p95 ~1500ms)
+
+### Counterfactuals
+- Model swap: gpt-4 → gpt-4o-mini (-75% latency, -90% cost)
+- Model swap: claude-3-opus → claude-3.5-sonnet (-60% latency)
+- Enable streaming for non-streaming calls
+- Add batching for embedding calls
+- Add caching layer
+
+### Drift (Combined Mode)
+Shows mismatches between code and runtime:
+- Inference points in code but not in runtime (dead code?)
+- Runtime events not mapped to code (dynamic calls?)

package/fixtures/demo-project/ai-service.ts

@@ -0,0 +1,65 @@
+/**
+ * Demo AI Service - Shows common LLM inference issues
+ * This file is used by `peakinfer demo` to demonstrate drift detection
+ */
+import Anthropic from '@anthropic-ai/sdk';
+import OpenAI from 'openai';
+
+const anthropic = new Anthropic();
+const openai = new OpenAI();
+
+// Issue 1: Streaming configured but may not be working in production
+export async function chat(prompt: string): Promise<string> {
+  const response = await anthropic.messages.create({
+    model: 'claude-sonnet-4-20250514',
+    max_tokens: 2000,
+    stream: true,  // <-- Code says streaming
+    messages: [{ role: 'user', content: prompt }],
+  });
+
+  let result = '';
+  for await (const event of response) {
+    if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {
+      result += event.delta.text;
+    }
+  }
+  return result;
+}
+
+// Issue 2: GPT-4 used for simple classification (overpowered model)
+export async function classifyIntent(message: string): Promise<string> {
+  const response = await openai.chat.completions.create({
+    model: 'gpt-4',  // <-- Expensive model for simple task
+    messages: [
+      { role: 'system', content: 'Classify the user intent as: question, complaint, feedback, or other' },
+      { role: 'user', content: message },
+    ],
+    max_tokens: 50,
+  });
+  return response.choices[0].message.content || 'other';
+}
+
+// Issue 3: No error handling, no retry logic
+export async function summarize(text: string): Promise<string> {
+  // No try/catch, no retry, no timeout
+  const response = await anthropic.messages.create({
+    model: 'claude-sonnet-4-20250514',
+    max_tokens: 500,
+    messages: [{ role: 'user', content: `Summarize: ${text}` }],
+  });
+  return response.content[0].type === 'text' ? response.content[0].text : '';
+}
+
+// Issue 4: Sequential processing (throughput bottleneck)
+export async function batchAnalyze(items: string[]): Promise<string[]> {
+  const results: string[] = [];
+  for (const item of items) {  // <-- Sequential, should be parallel
+    const response = await anthropic.messages.create({
+      model: 'claude-sonnet-4-20250514',
+      max_tokens: 200,
+      messages: [{ role: 'user', content: `Analyze: ${item}` }],
+    });
+    results.push(response.content[0].type === 'text' ? response.content[0].text : '');
+  }
+  return results;
+}

package/fixtures/demo-project/sample-events.jsonl

@@ -0,0 +1,15 @@
+{"id":"evt_001","ts":"2024-12-15T10:00:00Z","provider":"openai","model":"gpt-4","input_tokens":150,"output_tokens":500,"latency_ms":4500}
+{"id":"evt_002","ts":"2024-12-15T10:00:05Z","provider":"openai","model":"gpt-4","input_tokens":200,"output_tokens":800,"latency_ms":5200}
+{"id":"evt_003","ts":"2024-12-15T10:00:10Z","provider":"anthropic","model":"claude-3-opus-20240229","input_tokens":1000,"output_tokens":2000,"latency_ms":8500}
+{"id":"evt_004","ts":"2024-12-15T10:00:15Z","provider":"openai","model":"gpt-4-turbo","input_tokens":300,"output_tokens":400,"latency_ms":3800}
+{"id":"evt_005","ts":"2024-12-15T10:00:20Z","provider":"openai","model":"gpt-4o-mini","input_tokens":100,"output_tokens":150,"latency_ms":1200}
+{"id":"evt_006","ts":"2024-12-15T10:00:25Z","provider":"openai","model":"gpt-4o-mini","input_tokens":80,"output_tokens":120,"latency_ms":1100}
+{"id":"evt_007","ts":"2024-12-15T10:00:30Z","provider":"anthropic","model":"claude-3-haiku-20240307","input_tokens":50,"output_tokens":200,"latency_ms":900}
+{"id":"evt_008","ts":"2024-12-15T10:00:35Z","provider":"openai","model":"gpt-4o","input_tokens":200,"output_tokens":600,"latency_ms":2100,"streaming":true}
+{"id":"evt_009","ts":"2024-12-15T10:00:40Z","provider":"openai","model":"text-embedding-3-small","input_tokens":100,"output_tokens":0,"latency_ms":150}
+{"id":"evt_010","ts":"2024-12-15T10:00:45Z","provider":"openai","model":"gpt-4","input_tokens":180,"output_tokens":550,"latency_ms":4800}
+{"id":"evt_011","ts":"2024-12-15T10:01:00Z","provider":"anthropic","model":"claude-3-sonnet-20240229","input_tokens":400,"output_tokens":800,"latency_ms":3500}
+{"id":"evt_012","ts":"2024-12-15T10:01:05Z","provider":"openai","model":"gpt-4o-mini","input_tokens":90,"output_tokens":180,"latency_ms":1300}
+{"id":"evt_013","ts":"2024-12-15T10:01:10Z","provider":"openai","model":"gpt-4","input_tokens":220,"output_tokens":700,"latency_ms":5500}
+{"id":"evt_014","ts":"2024-12-15T10:01:15Z","provider":"anthropic","model":"claude-3-opus-20240229","input_tokens":1200,"output_tokens":2500,"latency_ms":9200}
+{"id":"evt_015","ts":"2024-12-15T10:01:20Z","provider":"openai","model":"text-embedding-3-small","input_tokens":150,"output_tokens":0,"latency_ms":180}

package/fixtures/demo-project/src/ai-service.ts

@@ -0,0 +1,128 @@
+/**
+ * AI Service for PeakInfer Demo
+ * Anthropic Claude only - demonstrates various inference patterns
+ */
+
+import Anthropic from '@anthropic-ai/sdk';
+
+const client = new Anthropic();
+
+// ============================================================================
+// HIGH LATENCY / HIGH COST (will trigger issues)
+// ============================================================================
+
+/**
+ * Complex analysis with Claude Opus (highest cost, no error handling)
+ * Issues expected:
+ * - Critical: No error handling
+ * - Warning: Expensive model for simple task
+ */
+export async function analyzeDocument(document: string): Promise<string> {
+  const response = await client.messages.create({
+    model: 'claude-opus-4-20250514',
+    max_tokens: 4000,
+    messages: [{ role: 'user', content: `Analyze: ${document}` }],
+  });
+  return response.content[0].type === 'text' ? response.content[0].text : '';
+}
+
+/**
+ * Chat completion without streaming (latency issue)
+ * Issues expected:
+ * - Warning: No streaming enabled
+ * - Critical: No error handling
+ */
+export async function chat(prompt: string): Promise<string> {
+  const response = await client.messages.create({
+    model: 'claude-sonnet-4-20250514',
+    max_tokens: 2000,
+    messages: [{ role: 'user', content: prompt }],
+  });
+  return response.content[0].type === 'text' ? response.content[0].text : '';
+}
+
+// ============================================================================
+// MEDIUM PATTERNS (some issues)
+// ============================================================================
+
+/**
+ * Translation service (no timeout configured)
+ * Issues expected:
+ * - Warning: No timeout configured
+ */
+export async function translate(text: string, targetLang: string): Promise<string> {
+  const response = await client.messages.create({
+    model: 'claude-3-5-sonnet-20241022',
+    max_tokens: 2000,
+    messages: [{ role: 'user', content: `Translate to ${targetLang}: ${text}` }],
+  });
+  return response.content[0].type === 'text' ? response.content[0].text : '';
+}
+
+// ============================================================================
+// GOOD PATTERNS (minimal issues)
+// ============================================================================
+
+/**
+ * Fast response with Claude Haiku (optimized)
+ * - Uses cheap model
+ * - Has error handling
+ * - Has streaming
+ */
+export async function* streamChat(prompt: string): AsyncGenerator<string> {
+  try {
+    const stream = await client.messages.stream({
+      model: 'claude-3-5-haiku-20241022',
+      max_tokens: 500,
+      messages: [{ role: 'user', content: prompt }],
+    });
+
+    for await (const event of stream) {
+      if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {
+        yield event.delta.text;
+      }
+    }
+  } catch (error) {
+    console.error('Stream error:', error);
+    throw error;
+  }
+}
+
+/**
+ * Robust API call with retry and fallback
+ * - Has error handling
+ * - Has retry logic
+ * - Has fallback model
+ */
+export async function robustChat(prompt: string): Promise<string> {
+  const maxRetries = 3;
+  let lastError: Error | null = null;
+
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      const response = await client.messages.create({
+        model: 'claude-3-5-sonnet-20241022',
+        max_tokens: 1000,
+        messages: [{ role: 'user', content: prompt }],
+      });
+      return response.content[0].type === 'text' ? response.content[0].text : '';
+    } catch (error) {
+      lastError = error as Error;
+      if (attempt < maxRetries - 1) {
+        await new Promise(r => setTimeout(r, 1000 * (attempt + 1)));
+      }
+    }
+  }
+
+  // Fallback to cheaper model
+  try {
+    const response = await client.messages.create({
+      model: 'claude-3-5-haiku-20241022',
+      max_tokens: 1000,
+      messages: [{ role: 'user', content: prompt }],
+    });
+    return response.content[0].type === 'text' ? response.content[0].text : '';
+  } catch {
+    throw lastError;
+  }
+}