@peakinfer/cli 1.0.133

package/tests/pr-comment.test.ts

@@ -0,0 +1,313 @@
+/**
+ * PR Comment Tests
+ * Per Test Cases v1.9.3 - PR/CI Integration Tests
+ */
+import { describe, test, expect } from 'vitest';
+
+// Types for PR comment generation
+interface InferencePoint {
+  id: string;
+  file: string;
+  line: number;
+  provider: string;
+  model: string;
+}
+
+interface Issue {
+  id: string;
+  severity: 'critical' | 'high' | 'medium' | 'low';
+  category: string;
+  title: string;
+  description: string;
+  file: string;
+  line: number;
+}
+
+interface AnalysisResult {
+  inferencePoints: InferencePoint[];
+  issues: Issue[];
+  summary: {
+    totalInferencePoints: number;
+    criticalIssues: number;
+    highIssues: number;
+    mediumIssues: number;
+    lowIssues: number;
+  };
+}
+
+// PR Comment generator (to be implemented in src/pr-comment.ts)
+function generatePRComment(result: AnalysisResult): string {
+  const lines: string[] = [];
+
+  // Header
+  lines.push('## PeakInfer Analysis');
+  lines.push('');
+
+  // Summary
+  lines.push(`Found **${result.summary.totalInferencePoints}** inference points`);
+  lines.push('');
+
+  // Issues by severity
+  if (result.summary.criticalIssues > 0 || result.summary.highIssues > 0) {
+    lines.push('### Issues Found');
+    lines.push('');
+
+    const critical = result.issues.filter(i => i.severity === 'critical');
+    const high = result.issues.filter(i => i.severity === 'high');
+
+    for (const issue of [...critical, ...high].slice(0, 5)) {
+      const emoji = issue.severity === 'critical' ? ':red_circle:' : ':orange_circle:';
+      lines.push(`${emoji} **${issue.severity.toUpperCase()}**: ${issue.title}`);
+      lines.push(`   - ${issue.file}:${issue.line}`);
+      lines.push(`   - ${issue.description}`);
+      lines.push('');
+    }
+  }
+
+  // No issues case
+  if (result.issues.length === 0) {
+    lines.push(':white_check_mark: No issues found');
+  }
+
+  // Footer
+  lines.push('---');
+  lines.push('*Generated by [PeakInfer](https://peakinfer.dev)*');
+
+  return lines.join('\n');
+}
+
+function generateCompactComment(result: AnalysisResult): string {
+  const criticalHigh = result.summary.criticalIssues + result.summary.highIssues;
+
+  if (criticalHigh === 0) {
+    return `:white_check_mark: PeakInfer: ${result.summary.totalInferencePoints} inference points, no critical issues`;
+  }
+
+  return `:warning: PeakInfer: ${result.summary.totalInferencePoints} inference points, ${criticalHigh} critical/high issues`;
+}
+
+function shouldBlockPR(result: AnalysisResult, threshold: 'critical' | 'high' | 'medium'): boolean {
+  if (threshold === 'critical') {
+    return result.summary.criticalIssues > 0;
+  }
+  if (threshold === 'high') {
+    return result.summary.criticalIssues > 0 || result.summary.highIssues > 0;
+  }
+  return result.summary.criticalIssues > 0 || result.summary.highIssues > 0 || result.summary.mediumIssues > 0;
+}
+
+describe('PR Comment Generation', () => {
+  test('Generates markdown comment with issues', () => {
+    const result: AnalysisResult = {
+      inferencePoints: [
+        { id: 'ip-1', file: 'src/chat.ts', line: 42, provider: 'anthropic', model: 'claude-sonnet-4-20250514' },
+      ],
+      issues: [
+        {
+          id: 'issue-1',
+          severity: 'critical',
+          category: 'cost',
+          title: 'Expensive model for simple task',
+          description: 'Using claude-opus for chat completion',
+          file: 'src/chat.ts',
+          line: 42,
+        },
+      ],
+      summary: {
+        totalInferencePoints: 1,
+        criticalIssues: 1,
+        highIssues: 0,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    const comment = generatePRComment(result);
+
+    expect(comment).toContain('## PeakInfer Analysis');
+    expect(comment).toContain('1 inference points');
+    expect(comment).toContain('CRITICAL');
+    expect(comment).toContain('Expensive model');
+    expect(comment).toContain('src/chat.ts:42');
+  });
+
+  test('Generates clean comment when no issues', () => {
+    const result: AnalysisResult = {
+      inferencePoints: [
+        { id: 'ip-1', file: 'src/chat.ts', line: 42, provider: 'anthropic', model: 'claude-sonnet-4-20250514' },
+      ],
+      issues: [],
+      summary: {
+        totalInferencePoints: 1,
+        criticalIssues: 0,
+        highIssues: 0,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    const comment = generatePRComment(result);
+
+    expect(comment).toContain('No issues found');
+    expect(comment).not.toContain('CRITICAL');
+  });
+
+  test('Limits issues to top 5', () => {
+    const issues: Issue[] = Array.from({ length: 10 }, (_, i) => ({
+      id: `issue-${i}`,
+      severity: 'high' as const,
+      category: 'cost',
+      title: `Issue ${i}`,
+      description: 'Test issue',
+      file: 'src/chat.ts',
+      line: i + 1,
+    }));
+
+    const result: AnalysisResult = {
+      inferencePoints: [],
+      issues,
+      summary: {
+        totalInferencePoints: 0,
+        criticalIssues: 0,
+        highIssues: 10,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    const comment = generatePRComment(result);
+
+    // Should only show 5 issues max
+    const highCount = (comment.match(/HIGH/g) || []).length;
+    expect(highCount).toBeLessThanOrEqual(5);
+  });
+
+  test('Includes PeakInfer branding', () => {
+    const result: AnalysisResult = {
+      inferencePoints: [],
+      issues: [],
+      summary: {
+        totalInferencePoints: 0,
+        criticalIssues: 0,
+        highIssues: 0,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    const comment = generatePRComment(result);
+
+    expect(comment).toContain('PeakInfer');
+    expect(comment).toContain('peakinfer.dev');
+  });
+});
+
+describe('Compact Comment', () => {
+  test('Shows success for no critical issues', () => {
+    const result: AnalysisResult = {
+      inferencePoints: [{ id: 'ip-1', file: 'test.ts', line: 1, provider: 'openai', model: 'gpt-4' }],
+      issues: [],
+      summary: {
+        totalInferencePoints: 1,
+        criticalIssues: 0,
+        highIssues: 0,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    const comment = generateCompactComment(result);
+
+    expect(comment).toContain(':white_check_mark:');
+    expect(comment).toContain('no critical issues');
+  });
+
+  test('Shows warning for critical/high issues', () => {
+    const result: AnalysisResult = {
+      inferencePoints: [],
+      issues: [
+        { id: 'i-1', severity: 'critical', category: 'cost', title: 'Test', description: 'Test', file: 'test.ts', line: 1 },
+      ],
+      summary: {
+        totalInferencePoints: 1,
+        criticalIssues: 1,
+        highIssues: 0,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    const comment = generateCompactComment(result);
+
+    expect(comment).toContain(':warning:');
+    expect(comment).toContain('1 critical/high issues');
+  });
+});
+
+describe('PR Blocking Logic', () => {
+  test('Blocks on critical when threshold is critical', () => {
+    const result: AnalysisResult = {
+      inferencePoints: [],
+      issues: [],
+      summary: {
+        totalInferencePoints: 0,
+        criticalIssues: 1,
+        highIssues: 0,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    expect(shouldBlockPR(result, 'critical')).toBe(true);
+  });
+
+  test('Does not block on high when threshold is critical', () => {
+    const result: AnalysisResult = {
+      inferencePoints: [],
+      issues: [],
+      summary: {
+        totalInferencePoints: 0,
+        criticalIssues: 0,
+        highIssues: 5,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    expect(shouldBlockPR(result, 'critical')).toBe(false);
+  });
+
+  test('Blocks on high when threshold is high', () => {
+    const result: AnalysisResult = {
+      inferencePoints: [],
+      issues: [],
+      summary: {
+        totalInferencePoints: 0,
+        criticalIssues: 0,
+        highIssues: 1,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    expect(shouldBlockPR(result, 'high')).toBe(true);
+  });
+
+  test('Does not block when no issues', () => {
+    const result: AnalysisResult = {
+      inferencePoints: [],
+      issues: [],
+      summary: {
+        totalInferencePoints: 5,
+        criticalIssues: 0,
+        highIssues: 0,
+        mediumIssues: 0,
+        lowIssues: 0,
+      },
+    };
+
+    expect(shouldBlockPR(result, 'critical')).toBe(false);
+    expect(shouldBlockPR(result, 'high')).toBe(false);
+    expect(shouldBlockPR(result, 'medium')).toBe(false);
+  });
+});

package/tests/prediction.test.ts

@@ -0,0 +1,296 @@
+import { describe, it, expect } from 'vitest';
+import {
+  generatePredictions,
+  formatPredictionSummary,
+  hasHighRiskPredictions,
+} from '../src/prediction.js';
+import type { Callsite, InferenceMap } from '../src/types.js';
+
+// =============================================================================
+// TEST FIXTURES
+// =============================================================================
+
+function createCallsite(overrides: Partial<Callsite> = {}): Callsite {
+  return {
+    id: `cs_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`,
+    file: 'src/agent.ts',
+    line: 42,
+    col: 10,
+    provider: 'openai',
+    model: 'gpt-4o',
+    framework: null,
+    runtime: null,
+    patterns: {},
+    confidence: 0.9,
+    ...overrides,
+  };
+}
+
+function createInferenceMap(callsites: Callsite[]): InferenceMap {
+  return {
+    version: '0.1',
+    callsites,
+    frameworks: {},
+    scanResult: {
+      root: '/test',
+      files: [],
+      summary: { totalFiles: 0, totalLoc: 0, languages: [], totalCandidates: 0 },
+    },
+  };
+}
+
+// =============================================================================
+// TESTS
+// =============================================================================
+
+describe('prediction', () => {
+  describe('generatePredictions', () => {
+    it('should generate predictions for callsites', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4o' }),
+        createCallsite({ file: 'src/b.ts', line: 20, model: 'gpt-4o-mini' }),
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+
+      expect(result.predictions.length).toBe(2);
+      expect(result.summary.totalPoints).toBe(2);
+    });
+
+    it('should assign risk levels based on latency', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'claude-3-opus' }), // high latency
+        createCallsite({ file: 'src/b.ts', line: 20, model: 'gemini-1.5-flash' }), // low latency
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+
+      const opusPrediction = result.predictions.find(p => p.model?.includes('opus'));
+      const flashPrediction = result.predictions.find(p => p.model?.includes('flash'));
+
+      expect(opusPrediction?.risk).toBe('high');
+      expect(flashPrediction?.risk).toBe('low');
+    });
+
+    it('should include prediction factors', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({
+          file: 'src/a.ts',
+          line: 10,
+          model: 'gpt-4o',
+          patterns: { streaming: true },
+        }),
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+
+      expect(result.predictions[0].factors.length).toBeGreaterThan(0);
+      const streamingFactor = result.predictions[0].factors.find(f =>
+        f.name.toLowerCase().includes('streaming')
+      );
+      expect(streamingFactor).toBeDefined();
+      expect(streamingFactor?.impact).toBe('positive');
+    });
+
+    it('should reduce latency for patterns like batching and caching', () => {
+      const baseInferenceMap = createInferenceMap([
+        createCallsite({
+          file: 'src/a.ts',
+          line: 10,
+          model: 'gpt-4o',
+          patterns: {},
+        }),
+      ]);
+
+      const optimizedInferenceMap = createInferenceMap([
+        createCallsite({
+          file: 'src/a.ts',
+          line: 10,
+          model: 'gpt-4o',
+          patterns: { batching: true, caching: true },
+        }),
+      ]);
+
+      const baseResult = generatePredictions(baseInferenceMap);
+      const optimizedResult = generatePredictions(optimizedInferenceMap);
+
+      expect(optimizedResult.predictions[0].predictedLatency.p95)
+        .toBeLessThan(baseResult.predictions[0].predictedLatency.p95);
+    });
+
+    it('should calculate correct summary statistics', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4' }), // high
+        createCallsite({ file: 'src/b.ts', line: 20, model: 'gpt-4o' }), // medium
+        createCallsite({ file: 'src/c.ts', line: 30, model: 'gpt-4o-mini' }), // low
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+
+      expect(result.summary.totalPoints).toBe(3);
+      expect(result.summary.highRiskCount).toBeGreaterThanOrEqual(0);
+      expect(result.summary.averageP95).toBeGreaterThan(0);
+      expect(result.summary.worstP95).toBeGreaterThanOrEqual(result.summary.averageP95);
+    });
+
+    it('should check budget when targetP95 is specified', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4' }), // p95 ~5000ms
+      ]);
+
+      const withinBudget = generatePredictions(inferenceMap, 0, { targetP95: 10000 });
+      const overBudget = generatePredictions(inferenceMap, 0, { targetP95: 1000 });
+
+      expect(withinBudget.summary.budgetExceeded).toBe(false);
+      expect(overBudget.summary.budgetExceeded).toBe(true);
+    });
+
+    it('should handle empty inference map', () => {
+      const inferenceMap = createInferenceMap([]);
+
+      const result = generatePredictions(inferenceMap);
+
+      expect(result.predictions.length).toBe(0);
+      expect(result.summary.totalPoints).toBe(0);
+      expect(result.summary.averageP95).toBe(0);
+    });
+
+    it('should handle unknown models with medium confidence when model is set', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({
+          file: 'src/a.ts',
+          line: 10,
+          model: 'custom-fine-tuned-model',
+        }),
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+
+      expect(result.predictions.length).toBe(1);
+      // When model is set (even unknown), confidence is medium; low only when no model
+      expect(result.predictions[0].confidence).toBe('medium');
+    });
+
+    it('should include location in predictions', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/agent.ts', line: 42 }),
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+
+      expect(result.predictions[0].location).toBe('src/agent.ts:42');
+    });
+  });
+
+  describe('formatPredictionSummary', () => {
+    it('should format summary with risk counts', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4' }),
+        createCallsite({ file: 'src/b.ts', line: 20, model: 'gpt-4o-mini' }),
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+      const summary = formatPredictionSummary(result);
+
+      expect(summary).toContain('inference points');
+      expect(summary).toContain('p95');
+    });
+
+    it('should show budget status when target specified', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4' }),
+      ]);
+
+      const result = generatePredictions(inferenceMap, 0, { targetP95: 1000 });
+      const summary = formatPredictionSummary(result);
+
+      expect(summary).toContain('Budget exceeded');
+    });
+  });
+
+  describe('hasHighRiskPredictions', () => {
+    it('should return true when high-risk predictions exist', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'claude-3-opus' }),
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+
+      expect(hasHighRiskPredictions(result)).toBe(true);
+    });
+
+    it('should return false when no high-risk predictions', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gemini-1.5-flash' }),
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+
+      expect(hasHighRiskPredictions(result)).toBe(false);
+    });
+
+    it('should return false for empty predictions', () => {
+      const inferenceMap = createInferenceMap([]);
+
+      const result = generatePredictions(inferenceMap);
+
+      expect(hasHighRiskPredictions(result)).toBe(false);
+    });
+  });
+
+  describe('model latency estimates', () => {
+    it('should estimate higher latency for Claude Opus vs Haiku', () => {
+      const opus = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'claude-3-opus' }),
+      ]);
+
+      const haiku = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'claude-3-haiku' }),
+      ]);
+
+      const opusResult = generatePredictions(opus);
+      const haikuResult = generatePredictions(haiku);
+
+      // Opus should have higher latency than Haiku
+      expect(opusResult.predictions[0].predictedLatency.p95)
+        .toBeGreaterThan(haikuResult.predictions[0].predictedLatency.p95);
+    });
+
+    it('should estimate higher latency for o1 models', () => {
+      const o1 = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'o1-preview' }),
+      ]);
+
+      const gpt4 = createInferenceMap([
+        createCallsite({ file: 'src/a.ts', line: 10, model: 'gpt-4' }),
+      ]);
+
+      const o1Result = generatePredictions(o1);
+      const gpt4Result = generatePredictions(gpt4);
+
+      expect(o1Result.predictions[0].predictedLatency.p95)
+        .toBeGreaterThan(gpt4Result.predictions[0].predictedLatency.p95);
+    });
+  });
+
+  describe('self-hosted providers', () => {
+    it('should identify self-hosted providers as positive factor', () => {
+      const inferenceMap = createInferenceMap([
+        createCallsite({
+          file: 'src/a.ts',
+          line: 10,
+          provider: 'vllm',
+          model: 'llama-3.1-70b',
+        }),
+      ]);
+
+      const result = generatePredictions(inferenceMap);
+
+      const selfHostedFactor = result.predictions[0].factors.find(f =>
+        f.name.toLowerCase().includes('self-hosted')
+      );
+      expect(selfHostedFactor).toBeDefined();
+      expect(selfHostedFactor?.impact).toBe('positive');
+    });
+  });
+});