@peakinfer/cli 1.0.133

package/src/commands/whatif.ts

@@ -0,0 +1,272 @@
+/**
+ * What-If Command (v1.6 - GAP 5)
+ *
+ * CLI command for counterfactual analysis:
+ * - Model swap: What if we used a different model?
+ * - Streaming: What if we enabled/disabled streaming?
+ * - Batching: What if we batched requests?
+ * - Provider: What if we used a different provider?
+ *
+ * Leverages existing src/counterfactuals.ts module.
+ */
+
+import { Command } from 'commander';
+import { existsSync, readFileSync } from 'fs';
+import { listRuns, loadRun } from '../history.js';
+import {
+  generateCounterfactuals,
+  rankCounterfactuals,
+  formatCounterfactualSummary,
+} from '../counterfactuals.js';
+import type {
+  InferenceMap,
+  Counterfactual,
+  CounterfactualResult,
+} from '../types.js';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+interface WhatIfOptions {
+  model?: string;
+  provider?: string;
+  streaming?: string;
+  batchSize?: string;
+  run?: string;
+  priority?: 'latency' | 'cost' | 'balanced';
+  output?: string;
+  limit?: number;
+}
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+/**
+ * Load inference map from history or file
+ */
+function loadInferenceMap(runId?: string): InferenceMap | null {
+  if (runId) {
+    // Check if it's a file path
+    if (existsSync(runId) && runId.endsWith('.json')) {
+      try {
+        const content = readFileSync(runId, 'utf-8');
+        return JSON.parse(content);
+      } catch {
+        return null;
+      }
+    }
+
+    // Load from history
+    const run = loadRun(runId);
+    return run?.data?.inferenceMap || null;
+  }
+
+  // Load latest run
+  const runs = listRuns();
+  if (runs.length === 0) {
+    return null;
+  }
+
+  const latestRun = loadRun(runs[0].runId);
+  return latestRun?.data?.inferenceMap || null;
+}
+
+/**
+ * Filter counterfactuals by type/criteria
+ */
+function filterCounterfactuals(
+  result: CounterfactualResult,
+  options: WhatIfOptions
+): Counterfactual[] {
+  let filtered = result.counterfactuals;
+
+  // Filter by type based on options
+  if (options.model) {
+    filtered = filtered.filter(c =>
+      c.type === 'model_swap' &&
+      c.proposedState.model?.toLowerCase().includes(options.model!.toLowerCase())
+    );
+  }
+
+  if (options.provider) {
+    filtered = filtered.filter(c =>
+      c.proposedState.provider?.toLowerCase().includes(options.provider!.toLowerCase())
+    );
+  }
+
+  if (options.streaming !== undefined) {
+    filtered = filtered.filter(c => c.type === 'streaming_enable');
+  }
+
+  if (options.batchSize) {
+    filtered = filtered.filter(c => c.type === 'batch_optimization');
+  }
+
+  return filtered;
+}
+
+/**
+ * Format counterfactual for display
+ */
+function formatCounterfactual(cf: Counterfactual, index: number): string {
+  const lines: string[] = [];
+
+  // Header with number and headline
+  lines.push(`\n${index + 1}. ${cf.headline}`);
+  lines.push('─'.repeat(50));
+
+  // Type badge
+  const typeBadges: Record<string, string> = {
+    model_swap: 'Model Swap',
+    batch_optimization: 'Batching',
+    cache_addition: 'Caching',
+    streaming_enable: 'Streaming',
+    provider_change: 'Provider',
+  };
+  lines.push(`   Type: ${typeBadges[cf.type] || cf.type}`);
+
+  // Current vs Proposed
+  lines.push('');
+  lines.push('   Current → Proposed:');
+  if (cf.currentState.model) {
+    lines.push(`     Model:    ${cf.currentState.model} → ${cf.proposedState.model || 'same'}`);
+  }
+  if (cf.currentState.provider || cf.proposedState.provider) {
+    lines.push(`     Provider: ${cf.currentState.provider || '-'} → ${cf.proposedState.provider || 'same'}`);
+  }
+  if (cf.currentState.pattern || cf.proposedState.pattern) {
+    lines.push(`     Pattern:  ${cf.currentState.pattern || '-'} → ${cf.proposedState.pattern || 'same'}`);
+  }
+
+  // Impact
+  lines.push('');
+  lines.push('   Impact:');
+  if (cf.impact.latencyDeltaPercent !== 0) {
+    const sign = cf.impact.latencyDeltaPercent < 0 ? '' : '+';
+    lines.push(`     Latency: ${sign}${cf.impact.latencyDeltaPercent}% (${sign}${cf.impact.latencyDelta}ms)`);
+  }
+  if (cf.impact.costDeltaPercent !== 0) {
+    const sign = cf.impact.costDeltaPercent < 0 ? '' : '+';
+    lines.push(`     Cost:    ${sign}${cf.impact.costDeltaPercent}%`);
+  }
+
+  // Effort and confidence
+  lines.push(`     Effort: ${cf.effort}   Confidence: ${cf.confidence}`);
+
+  // Tradeoffs
+  if (cf.impact.tradeoffs.length > 0) {
+    lines.push('');
+    lines.push('   Tradeoffs:');
+    for (const tradeoff of cf.impact.tradeoffs.slice(0, 3)) {
+      lines.push(`     • ${tradeoff}`);
+    }
+  }
+
+  // Affected points
+  if (cf.affectedPoints.length > 0) {
+    lines.push(`\n   Affects ${cf.affectedPoints.length} inference point${cf.affectedPoints.length !== 1 ? 's' : ''}`);
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Display counterfactual results
+ */
+function displayResults(counterfactuals: Counterfactual[], summary: CounterfactualResult['summary']): void {
+  if (counterfactuals.length === 0) {
+    console.log('\nNo optimization opportunities found matching your criteria.');
+    return;
+  }
+
+  console.log('\nWhat-If Analysis Results');
+  console.log('═'.repeat(50));
+  console.log(formatCounterfactualSummary({ counterfactuals, summary, generatedAt: new Date().toISOString() }));
+
+  for (let i = 0; i < counterfactuals.length; i++) {
+    console.log(formatCounterfactual(counterfactuals[i], i));
+  }
+
+  console.log('');
+}
+
+// =============================================================================
+// COMMAND
+// =============================================================================
+
+/**
+ * Register what-if command
+ */
+export function registerWhatIfCommand(program: Command): void {
+  program
+    .command('whatif')
+    .description('run counterfactual analysis ("what if" scenarios)')
+    .option('--model <model>', 'what if we used this model?')
+    .option('--provider <provider>', 'what if we used this provider?')
+    .option('--streaming <bool>', 'what if streaming was enabled/disabled?')
+    .option('--batch-size <n>', 'what if we batched requests?')
+    .option('--run <runId>', 'run ID or inference-map.json path (default: latest)')
+    .option('--priority <type>', 'ranking priority: latency, cost, balanced (default)', 'balanced')
+    .option('--output <format>', 'output format: text (default) or json', 'text')
+    .option('--limit <n>', 'limit number of results', parseInt)
+    .action(async (options: WhatIfOptions) => {
+      try {
+        // Load inference map
+        const inferenceMap = loadInferenceMap(options.run);
+
+        if (!inferenceMap) {
+          console.error('No analysis data found.');
+          console.log('\nRun "peakinfer analyze ." first, or specify:');
+          console.log('  --run <runId>  Run ID from history');
+          console.log('  --run <file>   Path to inference-map.json');
+          process.exit(1);
+        }
+
+        if (inferenceMap.callsites.length === 0) {
+          console.error('No inference points found in analysis.');
+          process.exit(1);
+        }
+
+        // Generate counterfactuals
+        const result = generateCounterfactuals(inferenceMap);
+
+        // Filter based on options
+        let counterfactuals = filterCounterfactuals(result, options);
+
+        // Rank by priority
+        if (options.priority && ['latency', 'cost', 'balanced'].includes(options.priority)) {
+          counterfactuals = rankCounterfactuals(
+            { ...result, counterfactuals },
+            options.priority as 'latency' | 'cost' | 'balanced'
+          );
+        }
+
+        // Apply limit
+        if (options.limit && options.limit > 0) {
+          counterfactuals = counterfactuals.slice(0, options.limit);
+        }
+
+        // Output
+        if (options.output === 'json') {
+          console.log(JSON.stringify({
+            counterfactuals,
+            summary: result.summary,
+            generatedAt: new Date().toISOString(),
+          }, null, 2));
+        } else {
+          displayResults(counterfactuals, result.summary);
+
+          // Show hints for more specific queries
+          if (!options.model && !options.provider && !options.streaming && !options.batchSize) {
+            console.log('Tip: Use --model, --provider, --streaming, or --batch-size to filter results.');
+            console.log('Example: peakinfer whatif --model gpt-4o-mini --priority cost');
+          }
+        }
+      } catch (error) {
+        console.error('Error:', error instanceof Error ? error.message : 'What-if analysis failed');
+        process.exit(1);
+      }
+    });
+}

package/src/comparison.ts

@@ -0,0 +1,283 @@
+/**
+ * Historical Comparison Module (v1.5)
+ *
+ * Compares current analysis with previous runs to surface:
+ * - New inference points (pre-deploy validation)
+ * - Removed inference points (cleanup validation)
+ * - Changed configurations (drift detection)
+ *
+ * Enables tracking changes over time for informed deployment decisions.
+ */
+
+import type {
+  Callsite,
+  Insight,
+  ComparisonResult,
+  ChangedInferencePoint,
+  FieldChange,
+  HistoryManifest,
+} from './types.js';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+export interface CompareOptions {
+  /** Specific run ID to compare against (default: latest) */
+  baseRunId?: string;
+  /** Include insight comparison (default: true) */
+  compareInsights?: boolean;
+}
+
+export interface AnalysisSnapshot {
+  runId: string;
+  timestamp: string;
+  callsites: Callsite[];
+  insights?: Insight[];
+}
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+/**
+ * Create a unique key for an inference point based on file:line.
+ * Used for matching inference points across runs.
+ */
+function getCallsiteKey(callsite: Callsite): string {
+  return `${callsite.file}:${callsite.line}`;
+}
+
+/**
+ * Compare two values for equality (deep comparison for objects).
+ */
+function valuesEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (a === null || b === null) return a === b;
+  if (typeof a !== typeof b) return false;
+  if (typeof a === 'object') {
+    return JSON.stringify(a) === JSON.stringify(b);
+  }
+  return false;
+}
+
+/**
+ * Find changes between two inference points.
+ */
+function findCallsiteChanges(before: Callsite, after: Callsite): FieldChange[] {
+  const changes: FieldChange[] = [];
+  const fieldsToCompare: (keyof Callsite)[] = [
+    'provider', 'model', 'framework', 'runtime', 'confidence'
+  ];
+
+  for (const field of fieldsToCompare) {
+    if (!valuesEqual(before[field], after[field])) {
+      changes.push({
+        field,
+        before: before[field],
+        after: after[field],
+      });
+    }
+  }
+
+  // Compare patterns object
+  const patternKeys = new Set([
+    ...Object.keys(before.patterns || {}),
+    ...Object.keys(after.patterns || {}),
+  ]);
+
+  for (const pattern of patternKeys) {
+    const beforeVal = (before.patterns as Record<string, unknown>)?.[pattern];
+    const afterVal = (after.patterns as Record<string, unknown>)?.[pattern];
+    if (!valuesEqual(beforeVal, afterVal)) {
+      changes.push({
+        field: `patterns.${pattern}`,
+        before: beforeVal,
+        after: afterVal,
+      });
+    }
+  }
+
+  return changes;
+}
+
+/**
+ * Compare insights between runs to find new/resolved issues.
+ */
+function compareInsights(
+  baseInsights: Insight[],
+  currentInsights: Insight[]
+): { newCritical: number; resolvedCritical: number; newWarnings: number; resolvedWarnings: number } {
+  const baseIds = new Set(baseInsights.map(i => i.id || `${i.headline}:${i.location}`));
+  const currentIds = new Set(currentInsights.map(i => i.id || `${i.headline}:${i.location}`));
+
+  // New insights (in current but not in base)
+  const newInsights = currentInsights.filter(i => {
+    const id = i.id || `${i.headline}:${i.location}`;
+    return !baseIds.has(id);
+  });
+
+  // Resolved insights (in base but not in current)
+  const resolvedInsights = baseInsights.filter(i => {
+    const id = i.id || `${i.headline}:${i.location}`;
+    return !currentIds.has(id);
+  });
+
+  return {
+    newCritical: newInsights.filter(i => i.severity === 'critical').length,
+    resolvedCritical: resolvedInsights.filter(i => i.severity === 'critical').length,
+    newWarnings: newInsights.filter(i => i.severity === 'warning').length,
+    resolvedWarnings: resolvedInsights.filter(i => i.severity === 'warning').length,
+  };
+}
+
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+
+/**
+ * Compare two analysis snapshots.
+ */
+export function compareSnapshots(
+  baseline: AnalysisSnapshot,
+  current: AnalysisSnapshot,
+  options: CompareOptions = {}
+): ComparisonResult {
+  const { compareInsights: shouldCompareInsights = true } = options;
+
+  // Build lookup maps
+  const baseMap = new Map<string, Callsite>();
+  for (const cs of baseline.callsites) {
+    baseMap.set(getCallsiteKey(cs), cs);
+  }
+
+  const currentMap = new Map<string, Callsite>();
+  for (const cs of current.callsites) {
+    currentMap.set(getCallsiteKey(cs), cs);
+  }
+
+  // Find added, removed, and changed inference points
+  const added: Callsite[] = [];
+  const removed: Callsite[] = [];
+  const changed: ChangedInferencePoint[] = [];
+
+  // Check current callsites against baseline
+  for (const [key, currentCs] of currentMap) {
+    const baseCs = baseMap.get(key);
+    if (!baseCs) {
+      // New inference point
+      added.push(currentCs);
+    } else {
+      // Check for changes
+      const changes = findCallsiteChanges(baseCs, currentCs);
+      if (changes.length > 0) {
+        changed.push({ point: currentCs, changes });
+      }
+    }
+  }
+
+  // Check for removed callsites
+  for (const [key, baseCs] of baseMap) {
+    if (!currentMap.has(key)) {
+      removed.push(baseCs);
+    }
+  }
+
+  // Build result
+  const result: ComparisonResult = {
+    baseRunId: baseline.runId,
+    baseTimestamp: baseline.timestamp,
+    currentRunId: current.runId,
+    currentTimestamp: current.timestamp,
+    added,
+    removed,
+    changed,
+    metrics: {
+      totalBefore: baseline.callsites.length,
+      totalAfter: current.callsites.length,
+      addedCount: added.length,
+      removedCount: removed.length,
+      changedCount: changed.length,
+      netChange: added.length - removed.length,
+    },
+  };
+
+  // Add insight deltas if requested
+  if (shouldCompareInsights && baseline.insights && current.insights) {
+    result.insightDeltas = compareInsights(baseline.insights, current.insights);
+  }
+
+  return result;
+}
+
+/**
+ * Format a comparison result as a human-readable summary.
+ * Provides concise, actionable summary for pre-deploy review.
+ */
+export function formatComparisonSummary(comparison: ComparisonResult): string {
+  const lines: string[] = [];
+
+  // Header with delta
+  const delta = comparison.metrics.netChange;
+  const deltaStr = delta > 0 ? `+${delta}` : delta.toString();
+  lines.push(`Comparing with run from ${new Date(comparison.baseTimestamp).toLocaleDateString()}`);
+  lines.push(`Inference points: ${comparison.metrics.totalBefore} → ${comparison.metrics.totalAfter} (${deltaStr})`);
+  lines.push('');
+
+  // Changes summary
+  if (comparison.metrics.addedCount > 0) {
+    lines.push(`  + ${comparison.metrics.addedCount} new inference point${comparison.metrics.addedCount !== 1 ? 's' : ''}`);
+  }
+  if (comparison.metrics.removedCount > 0) {
+    lines.push(`  - ${comparison.metrics.removedCount} removed inference point${comparison.metrics.removedCount !== 1 ? 's' : ''}`);
+  }
+  if (comparison.metrics.changedCount > 0) {
+    lines.push(`  ~ ${comparison.metrics.changedCount} modified inference point${comparison.metrics.changedCount !== 1 ? 's' : ''}`);
+  }
+
+  // Insight deltas (if available)
+  if (comparison.insightDeltas) {
+    const { newCritical, resolvedCritical, newWarnings, resolvedWarnings } = comparison.insightDeltas;
+    if (newCritical > 0) {
+      lines.push(`  ! ${newCritical} new critical issue${newCritical !== 1 ? 's' : ''}`);
+    }
+    if (resolvedCritical > 0) {
+      lines.push(`  [OK] ${resolvedCritical} critical issue${resolvedCritical !== 1 ? 's' : ''} resolved`);
+    }
+    if (newWarnings > 0) {
+      lines.push(`  ! ${newWarnings} new warning${newWarnings !== 1 ? 's' : ''}`);
+    }
+    if (resolvedWarnings > 0) {
+      lines.push(`  [OK] ${resolvedWarnings} warning${resolvedWarnings !== 1 ? 's' : ''} resolved`);
+    }
+  }
+
+  // No changes case
+  if (comparison.metrics.addedCount === 0 &&
+      comparison.metrics.removedCount === 0 &&
+      comparison.metrics.changedCount === 0) {
+    lines.push('  No changes detected');
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Check if comparison has significant changes that warrant attention.
+ * Used to highlight important changes in the output.
+ */
+export function hasSignificantChanges(comparison: ComparisonResult): boolean {
+  // Any added/removed/changed inference points are significant
+  if (comparison.metrics.addedCount > 0 ||
+      comparison.metrics.removedCount > 0 ||
+      comparison.metrics.changedCount > 0) {
+    return true;
+  }
+
+  // New critical insights are significant
+  if (comparison.insightDeltas?.newCritical && comparison.insightDeltas.newCritical > 0) {
+    return true;
+  }
+
+  return false;
+}