@peakinfer/cli 1.0.133

package/src/artifacts.ts

@@ -0,0 +1,218 @@
+import { mkdirSync, writeFileSync, existsSync, readFileSync, symlinkSync, unlinkSync } from 'fs';
+import { join, relative } from 'path';
+import type { InferenceMap, Insight, JoinedOutput, RuntimeSummary } from './types.js';
+import { generateRunId, getRunDir, createManifest, canResume, loadCachedArtifacts, type RunInputs, type RunManifest } from './runid.js';
+
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+
+const OUTPUT_DIR = '.peakinfer';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+export interface ArtifactData {
+  inferenceMap?: InferenceMap;
+  insights?: Insight[];
+  joined?: JoinedOutput;
+  runtime?: RuntimeSummary;
+  html?: string;
+}
+
+export interface SaveOptions {
+  runId?: string;
+  inputs?: RunInputs;
+  projectName?: string;  // For human-friendly report naming
+}
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+/**
+ * Convert project name to a URL/file-safe slug
+ * Julie Zhou: Human-friendly naming for shareability
+ */
+function toSlug(name: string): string {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '_')  // Replace non-alphanumeric with underscore
+    .replace(/^_+|_+$/g, '')      // Trim leading/trailing underscores
+    .substring(0, 50);            // Limit length
+}
+
+function ensureDir(dir: string): void {
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+}
+
+function writeJSON(filePath: string, data: unknown): void {
+  writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
+}
+
+function updateLatestSymlink(baseDir: string, runId: string): void {
+  const latestPath = join(baseDir, 'latest');
+  const targetPath = join('runs', runId);
+
+  try {
+    // Remove existing symlink
+    if (existsSync(latestPath)) {
+      unlinkSync(latestPath);
+    }
+    // Create new symlink
+    symlinkSync(targetPath, latestPath);
+  } catch {
+    // Symlinks may not work on all systems (e.g., Windows without admin)
+    // Fallback: write a text file with the run ID
+    writeFileSync(latestPath, runId, 'utf-8');
+  }
+}
+
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+
+/**
+ * Save all analysis artifacts to .peakinfer/runs/<runId>/ directory
+ * Also maintains backward compatibility with root-level artifacts
+ */
+export function saveArtifacts(
+  data: ArtifactData,
+  outputDir: string = OUTPUT_DIR,
+  options: SaveOptions = {}
+): string[] {
+  const savedFiles: string[] = [];
+  const artifactNames: string[] = [];
+
+  // Determine run directory
+  let runDir = outputDir;
+  if (options.runId) {
+    runDir = getRunDir(outputDir, options.runId);
+  }
+
+  ensureDir(runDir);
+
+  // 1. InferenceMap - the core output
+  if (data.inferenceMap) {
+    const filePath = join(runDir, 'inferencemap.json');
+    writeJSON(filePath, data.inferenceMap);
+    savedFiles.push(filePath);
+    artifactNames.push('inferencemap.json');
+  }
+
+  // 2. Insights - the findings
+  if (data.insights) {
+    const filePath = join(runDir, 'insights.json');
+    writeJSON(filePath, data.insights);
+    savedFiles.push(filePath);
+    artifactNames.push('insights.json');
+  }
+
+  // 3. Joined data - static + runtime correlation
+  if (data.joined) {
+    const filePath = join(runDir, 'joined.json');
+    writeJSON(filePath, data.joined);
+    savedFiles.push(filePath);
+    artifactNames.push('joined.json');
+  }
+
+  // 4. Runtime summary - aggregated metrics
+  if (data.runtime) {
+    const filePath = join(runDir, 'runtime.json');
+    writeJSON(filePath, data.runtime);
+    savedFiles.push(filePath);
+    artifactNames.push('runtime.json');
+  }
+
+  // 5. HTML report - human-friendly naming for shareability
+  // Julie Zhou: "Reports exist to enable sharing, not exploration"
+  const reportFileName = options.projectName
+    ? `${toSlug(options.projectName)}_peakinfer_report.html`
+    : 'report.html';
+
+  if (data.html) {
+    const filePath = join(runDir, reportFileName);
+    writeFileSync(filePath, data.html, 'utf-8');
+    savedFiles.push(filePath);
+    artifactNames.push(reportFileName);
+  }
+
+  // 6. Save manifest if runId provided
+  if (options.runId && options.inputs) {
+    const manifest = createManifest(options.runId, options.inputs, artifactNames, 'complete');
+    const manifestPath = join(runDir, 'manifest.json');
+    writeJSON(manifestPath, manifest);
+
+    // Update latest symlink
+    updateLatestSymlink(outputDir, options.runId);
+  }
+
+  // 7. Also save to root level for backward compatibility
+  if (options.runId && runDir !== outputDir) {
+    ensureDir(outputDir);
+    if (data.inferenceMap) {
+      writeJSON(join(outputDir, 'inferencemap.json'), data.inferenceMap);
+    }
+    if (data.insights) {
+      writeJSON(join(outputDir, 'insights.json'), data.insights);
+    }
+    if (data.joined) {
+      writeJSON(join(outputDir, 'joined.json'), data.joined);
+    }
+    if (data.runtime) {
+      writeJSON(join(outputDir, 'runtime.json'), data.runtime);
+    }
+    if (data.html) {
+      writeFileSync(join(outputDir, reportFileName), data.html, 'utf-8');
+    }
+  }
+
+  return savedFiles;
+}
+
+/**
+ * Get the output directory path
+ */
+export function getOutputDir(): string {
+  return OUTPUT_DIR;
+}
+
+/**
+ * Check if artifacts exist from a previous run
+ */
+export function artifactsExist(outputDir: string = OUTPUT_DIR): boolean {
+  return existsSync(join(outputDir, 'inferencemap.json'));
+}
+
+/**
+ * Check if a run can be resumed with cached artifacts
+ */
+export function checkResumable(inputs: RunInputs, outputDir: string = OUTPUT_DIR): {
+  canResume: boolean;
+  runId: string;
+  runDir: string;
+} {
+  const runId = generateRunId(inputs);
+  const runDir = getRunDir(outputDir, runId);
+
+  return {
+    canResume: canResume(runDir, inputs),
+    runId,
+    runDir,
+  };
+}
+
+/**
+ * Load artifacts from a previous run
+ */
+export function loadArtifacts(runDir: string): ArtifactData {
+  return loadCachedArtifacts(runDir);
+}
+
+/**
+ * Get a new run ID for given inputs
+ */
+export { generateRunId } from './runid.js';

package/src/benchmarks/index.ts

@@ -0,0 +1,309 @@
+/**
+ * InferenceMAX Benchmark Integration
+ *
+ * Provides benchmark comparison for LLM inference performance.
+ * Data sourced from the InferenceMAX benchmark suite.
+ */
+
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+// Get the directory of the current module
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+interface BenchmarkMetrics {
+  ttft_ms: number;
+  p50_latency_ms: number;
+  p95_latency_ms: number;
+  p99_latency_ms: number;
+  throughput_tps: number;
+  cost_per_1k_input: number;
+  cost_per_1k_output: number;
+}
+
+interface BenchmarkEntry {
+  model: string;
+  provider: string;
+  framework: string;
+  hardware: string;
+  metrics: BenchmarkMetrics;
+  optimal_config?: Record<string, unknown>;
+  notes?: string;
+}
+
+interface BenchmarkData {
+  version: string;
+  last_updated: string;
+  source: string;
+  benchmarks: Record<string, BenchmarkEntry>;
+  model_aliases: Record<string, string>;
+}
+
+export interface BenchmarkComparison {
+  pointId: string;
+  model: string;
+  framework: string;
+  hardware: string;
+  your_metrics: {
+    p95_latency_ms?: number;
+    ttft_ms?: number;
+    throughput_tps?: number;
+  };
+  benchmark_metrics: BenchmarkMetrics;
+  gaps: {
+    p95_latency?: { value: number; percent: number; description: string };
+    ttft?: { value: number; percent: number; description: string };
+    throughput?: { value: number; percent: number; description: string };
+  };
+  overall_gap: string;
+  optimal_config?: Record<string, unknown>;
+}
+
+let benchmarkData: BenchmarkData | null = null;
+
+/**
+ * Load benchmark data from JSON file
+ */
+function loadBenchmarks(): BenchmarkData {
+  if (benchmarkData) return benchmarkData;
+
+  try {
+    const dataPath = join(__dirname, '../../data/inferencemax.json');
+    const content = readFileSync(dataPath, 'utf-8');
+    benchmarkData = JSON.parse(content) as BenchmarkData;
+    return benchmarkData;
+  } catch (error) {
+    throw new Error(`Failed to load benchmark data: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}
+
+/**
+ * Normalize model name for lookup
+ */
+function normalizeModel(model: string): string {
+  return model
+    .toLowerCase()
+    .replace(/[_\s]/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-|-$/g, '');
+}
+
+/**
+ * Get benchmark for a specific model
+ */
+export function getBenchmark(
+  model: string,
+  framework = 'api',
+  hardware = 'api'
+): BenchmarkEntry | null {
+  const data = loadBenchmarks();
+  const normalizedModel = normalizeModel(model);
+
+  // Try exact key match first
+  const exactKey = `${normalizedModel}:${framework}:${hardware}`;
+  if (data.benchmarks[exactKey]) {
+    return data.benchmarks[exactKey];
+  }
+
+  // Try alias lookup
+  const alias = data.model_aliases[normalizedModel];
+  if (alias && data.benchmarks[alias]) {
+    return data.benchmarks[alias];
+  }
+
+  // Try model name with default framework:hardware
+  const defaultKey = `${normalizedModel}:api:api`;
+  if (data.benchmarks[defaultKey]) {
+    return data.benchmarks[defaultKey];
+  }
+
+  // Try partial model name matches
+  for (const key of Object.keys(data.benchmarks)) {
+    const benchModel = normalizeModel(data.benchmarks[key].model);
+    if (benchModel.includes(normalizedModel) || normalizedModel.includes(benchModel)) {
+      return data.benchmarks[key];
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Compare user metrics to benchmark
+ */
+export function compareToBenchmark(
+  pointId: string,
+  model: string,
+  userMetrics: {
+    p95_latency_ms?: number;
+    ttft_ms?: number;
+    throughput_tps?: number;
+  },
+  framework = 'api',
+  hardware = 'api'
+): BenchmarkComparison | null {
+  const benchmark = getBenchmark(model, framework, hardware);
+  if (!benchmark) return null;
+
+  const gaps: BenchmarkComparison['gaps'] = {};
+
+  // P95 Latency gap
+  if (userMetrics.p95_latency_ms && benchmark.metrics.p95_latency_ms) {
+    const diff = userMetrics.p95_latency_ms - benchmark.metrics.p95_latency_ms;
+    const percent = Math.round((diff / benchmark.metrics.p95_latency_ms) * 100);
+    gaps.p95_latency = {
+      value: diff,
+      percent,
+      description: formatGapDescription(diff, percent, 'ms', 'slower', 'faster'),
+    };
+  }
+
+  // TTFT gap
+  if (userMetrics.ttft_ms && benchmark.metrics.ttft_ms) {
+    const diff = userMetrics.ttft_ms - benchmark.metrics.ttft_ms;
+    const percent = Math.round((diff / benchmark.metrics.ttft_ms) * 100);
+    gaps.ttft = {
+      value: diff,
+      percent,
+      description: formatGapDescription(diff, percent, 'ms', 'slower', 'faster'),
+    };
+  }
+
+  // Throughput gap (inverse - higher is better)
+  if (userMetrics.throughput_tps && benchmark.metrics.throughput_tps) {
+    const diff = benchmark.metrics.throughput_tps - userMetrics.throughput_tps;
+    const percent = Math.round((diff / benchmark.metrics.throughput_tps) * 100);
+    gaps.throughput = {
+      value: -diff, // Negative means user is faster
+      percent: -percent,
+      description: formatGapDescription(diff, percent, 'tps', 'below', 'above'),
+    };
+  }
+
+  // Calculate overall gap description
+  const overallGap = calculateOverallGap(gaps);
+
+  return {
+    pointId,
+    model: benchmark.model,
+    framework: benchmark.framework,
+    hardware: benchmark.hardware,
+    your_metrics: userMetrics,
+    benchmark_metrics: benchmark.metrics,
+    gaps,
+    overall_gap: overallGap,
+    optimal_config: benchmark.optimal_config,
+  };
+}
+
+function formatGapDescription(
+  diff: number,
+  percent: number,
+  unit: string,
+  worseWord: string,
+  betterWord: string
+): string {
+  if (diff === 0 || percent === 0) {
+    return 'On par with benchmark';
+  }
+
+  const absPercent = Math.abs(percent);
+  const absDiff = Math.abs(diff);
+  const word = diff > 0 ? worseWord : betterWord;
+
+  if (absPercent > 100) {
+    const multiplier = (absPercent / 100 + 1).toFixed(1);
+    return `${multiplier}x ${word}`;
+  }
+
+  return `${absPercent}% ${word} (${diff > 0 ? '+' : ''}${absDiff}${unit})`;
+}
+
+function calculateOverallGap(gaps: BenchmarkComparison['gaps']): string {
+  const issues: string[] = [];
+
+  if (gaps.p95_latency && gaps.p95_latency.percent > 50) {
+    issues.push(`latency ${gaps.p95_latency.description}`);
+  }
+
+  if (gaps.ttft && gaps.ttft.percent > 50) {
+    issues.push(`TTFT ${gaps.ttft.description}`);
+  }
+
+  if (gaps.throughput && gaps.throughput.percent < -30) {
+    issues.push(`throughput ${gaps.throughput.description}`);
+  }
+
+  if (issues.length === 0) {
+    if (Object.keys(gaps).length === 0) {
+      return 'No metrics to compare';
+    }
+    return 'Performing within benchmark range';
+  }
+
+  return issues.join(', ');
+}
+
+/**
+ * Get all available benchmarks
+ */
+export function listBenchmarks(): BenchmarkEntry[] {
+  const data = loadBenchmarks();
+  return Object.values(data.benchmarks);
+}
+
+/**
+ * Get benchmark data version
+ */
+export function getBenchmarkVersion(): { version: string; lastUpdated: string } {
+  const data = loadBenchmarks();
+  return {
+    version: data.version,
+    lastUpdated: data.last_updated,
+  };
+}
+
+/**
+ * Check if benchmark data is available for a model
+ */
+export function hasBenchmark(model: string): boolean {
+  return getBenchmark(model) !== null;
+}
+
+/**
+ * Format benchmark comparison for display
+ */
+export function formatBenchmarkComparison(comparison: BenchmarkComparison): string {
+  const lines: string[] = [];
+
+  lines.push(`Model: ${comparison.model}`);
+  lines.push(`Framework: ${comparison.framework} | Hardware: ${comparison.hardware}`);
+  lines.push('');
+
+  if (comparison.gaps.p95_latency) {
+    lines.push(`P95 Latency: Your ${comparison.your_metrics.p95_latency_ms}ms | Benchmark ${comparison.benchmark_metrics.p95_latency_ms}ms | ${comparison.gaps.p95_latency.description}`);
+  }
+
+  if (comparison.gaps.ttft) {
+    lines.push(`TTFT: Your ${comparison.your_metrics.ttft_ms}ms | Benchmark ${comparison.benchmark_metrics.ttft_ms}ms | ${comparison.gaps.ttft.description}`);
+  }
+
+  if (comparison.gaps.throughput) {
+    lines.push(`Throughput: Your ${comparison.your_metrics.throughput_tps} tps | Benchmark ${comparison.benchmark_metrics.throughput_tps} tps | ${comparison.gaps.throughput.description}`);
+  }
+
+  lines.push('');
+  lines.push(`Overall: ${comparison.overall_gap}`);
+
+  if (comparison.optimal_config) {
+    lines.push('');
+    lines.push('Optimal Config:');
+    for (const [key, value] of Object.entries(comparison.optimal_config)) {
+      lines.push(`  ${key}: ${value}`);
+    }
+  }
+
+  return lines.join('\n');
+}