@peakinfer/cli 1.0.133

package/src/connectors/types.ts

@@ -0,0 +1,180 @@
+/**
+ * Shared types for runtime data connectors
+ *
+ * These types define the normalized format for runtime events
+ * from various sources (Helicone, LangSmith, etc.)
+ */
+
+export interface NormalizedEvent {
+  id: string;
+  timestamp: string;
+  model: string;
+  provider: string;
+  latency_ms: number;
+  prompt_tokens?: number;
+  completion_tokens?: number;
+  total_tokens?: number;
+  cost_usd?: number;
+  success: boolean;
+  error?: string;
+  streaming?: boolean;
+  // Trace metadata
+  trace_id?: string;
+  span_id?: string;
+  parent_span_id?: string;
+  // Request metadata
+  request_path?: string;
+  user_id?: string;
+  session_id?: string;
+  // Raw data for debugging
+  raw?: Record<string, unknown>;
+}
+
+export interface ConnectorSummary {
+  total_requests: number;
+  total_cost_usd: number;
+  avg_latency_ms: number;
+  p50_latency_ms: number;
+  p95_latency_ms: number;
+  p99_latency_ms: number;
+  error_rate: number;
+  streaming_rate: number;
+  by_model: Record<string, {
+    count: number;
+    cost: number;
+    avg_latency_ms: number;
+    p95_latency_ms: number;
+    error_rate: number;
+  }>;
+  by_provider: Record<string, {
+    count: number;
+    cost: number;
+  }>;
+  time_range: {
+    start: string;
+    end: string;
+  };
+}
+
+export interface ConnectorResult {
+  events: NormalizedEvent[];
+  summary: ConnectorSummary;
+  metadata: {
+    source: 'helicone' | 'langsmith';
+    fetched_at: string;
+    total_fetched: number;
+    truncated: boolean;
+    api_version?: string;
+  };
+}
+
+export interface ConnectorConfig {
+  apiKey: string;
+  limit?: number;
+  startDate?: string;
+  endDate?: string;
+  filter?: {
+    model?: string;
+    provider?: string;
+    success?: boolean;
+  };
+}
+
+export class ConnectorError extends Error {
+  constructor(
+    message: string,
+    public readonly source: 'helicone' | 'langsmith',
+    public readonly statusCode?: number,
+    public readonly code?: string
+  ) {
+    super(message);
+    this.name = 'ConnectorError';
+  }
+}
+
+// Helper functions for calculating summary statistics
+export function calculatePercentile(values: number[], percentile: number): number {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.ceil((percentile / 100) * sorted.length) - 1;
+  return sorted[Math.max(0, index)];
+}
+
+export function calculateSummary(events: NormalizedEvent[]): ConnectorSummary {
+  if (events.length === 0) {
+    return {
+      total_requests: 0,
+      total_cost_usd: 0,
+      avg_latency_ms: 0,
+      p50_latency_ms: 0,
+      p95_latency_ms: 0,
+      p99_latency_ms: 0,
+      error_rate: 0,
+      streaming_rate: 0,
+      by_model: {},
+      by_provider: {},
+      time_range: { start: '', end: '' },
+    };
+  }
+
+  const latencies = events.map(e => e.latency_ms).filter(l => l > 0);
+  const costs = events.map(e => e.cost_usd || 0);
+  const errors = events.filter(e => !e.success).length;
+  const streaming = events.filter(e => e.streaming).length;
+
+  // Group by model
+  const byModel: ConnectorSummary['by_model'] = {};
+  for (const event of events) {
+    const model = event.model || 'unknown';
+    if (!byModel[model]) {
+      byModel[model] = { count: 0, cost: 0, avg_latency_ms: 0, p95_latency_ms: 0, error_rate: 0 };
+    }
+    byModel[model].count++;
+    byModel[model].cost += event.cost_usd || 0;
+  }
+
+  // Calculate per-model stats
+  for (const model of Object.keys(byModel)) {
+    const modelEvents = events.filter(e => (e.model || 'unknown') === model);
+    const modelLatencies = modelEvents.map(e => e.latency_ms).filter(l => l > 0);
+    const modelErrors = modelEvents.filter(e => !e.success).length;
+
+    byModel[model].avg_latency_ms = modelLatencies.length > 0
+      ? Math.round(modelLatencies.reduce((a, b) => a + b, 0) / modelLatencies.length)
+      : 0;
+    byModel[model].p95_latency_ms = calculatePercentile(modelLatencies, 95);
+    byModel[model].error_rate = modelEvents.length > 0 ? modelErrors / modelEvents.length : 0;
+  }
+
+  // Group by provider
+  const byProvider: ConnectorSummary['by_provider'] = {};
+  for (const event of events) {
+    const provider = event.provider || 'unknown';
+    if (!byProvider[provider]) {
+      byProvider[provider] = { count: 0, cost: 0 };
+    }
+    byProvider[provider].count++;
+    byProvider[provider].cost += event.cost_usd || 0;
+  }
+
+  // Time range
+  const timestamps = events.map(e => new Date(e.timestamp).getTime()).filter(t => !isNaN(t));
+  const timeRange = {
+    start: timestamps.length > 0 ? new Date(Math.min(...timestamps)).toISOString() : '',
+    end: timestamps.length > 0 ? new Date(Math.max(...timestamps)).toISOString() : '',
+  };
+
+  return {
+    total_requests: events.length,
+    total_cost_usd: costs.reduce((a, b) => a + b, 0),
+    avg_latency_ms: latencies.length > 0 ? Math.round(latencies.reduce((a, b) => a + b, 0) / latencies.length) : 0,
+    p50_latency_ms: calculatePercentile(latencies, 50),
+    p95_latency_ms: calculatePercentile(latencies, 95),
+    p99_latency_ms: calculatePercentile(latencies, 99),
+    error_rate: events.length > 0 ? errors / events.length : 0,
+    streaming_rate: events.length > 0 ? streaming / events.length : 0,
+    by_model: byModel,
+    by_provider: byProvider,
+    time_range: timeRange,
+  };
+}

package/src/cost-estimator.ts

@@ -0,0 +1,146 @@
+/**
+ * Cost Estimation Module for PeakInfer CLI
+ *
+ * Estimates LLM API costs before running analysis to prevent surprise bills
+ * on large repositories. Uses LiteLLM pricing data with 24hr cache.
+ *
+ * PRD v1.9.3 Section 2.3: Cost Estimation (Pre-Analysis Transparency)
+ */
+
+import { scan } from './scanner.js';
+import { loadPricing, getModelCost } from './costs.js';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+export interface CostEstimate {
+  model: string;
+  filesToScan: number;
+  estimatedInputTokens: number;
+  estimatedOutputTokens: number;
+  inputCost: number;
+  outputCost: number;
+  totalCost: number;
+  pricing: {
+    inputPerMillion: number;
+    outputPerMillion: number;
+    source: 'litellm' | 'fallback';
+  };
+  warnings: CostWarning[];
+}
+
+export interface CostWarning {
+  level: 'yellow' | 'red' | 'critical';
+  message: string;
+}
+
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+
+// Token estimation constants (based on empirical analysis of codebases)
+const AVG_TOKENS_PER_FILE = 2000;  // Average input tokens per code file
+const AVG_OUTPUT_RATIO = 0.35;     // Output tokens as ratio of input
+
+// Warning thresholds (in USD)
+const WARNING_THRESHOLD_YELLOW = 5;
+const WARNING_THRESHOLD_RED = 20;
+const WARNING_THRESHOLD_CRITICAL = 100;
+
+// Default model if not specified (matches agent.ts default)
+const DEFAULT_MODEL = 'claude-sonnet-4-20250514';
+
+// Fallback pricing if LiteLLM fetch fails (Claude Sonnet pricing)
+const FALLBACK_PRICING = {
+  input: 3.00,   // $3.00 per 1M input tokens
+  output: 15.00, // $15.00 per 1M output tokens
+};
+
+// =============================================================================
+// MAIN FUNCTION
+// =============================================================================
+
+/**
+ * Estimate the cost of analyzing a codebase before running the analysis.
+ *
+ * @param path - Path to the codebase to analyze
+ * @param options - Optional model override
+ * @returns Cost estimate with warnings
+ */
+export async function estimateAnalysisCost(
+  path: string,
+  options: { model?: string } = {}
+): Promise<CostEstimate> {
+  const model = options.model || DEFAULT_MODEL;
+
+  // Step 1: Scan codebase to count files
+  const scanResult = await scan(path);
+  const filesToScan = scanResult.summary.totalFiles;
+
+  // Step 2: Estimate tokens
+  const estimatedInputTokens = filesToScan * AVG_TOKENS_PER_FILE;
+  const estimatedOutputTokens = Math.round(estimatedInputTokens * AVG_OUTPUT_RATIO);
+
+  // Step 3: Load pricing data
+  await loadPricing();
+  const modelCost = getModelCost(model);
+
+  // Determine if using LiteLLM or fallback
+  const useFallback = modelCost.input === 0 && modelCost.output === 0;
+  const pricing = useFallback
+    ? { input: FALLBACK_PRICING.input, output: FALLBACK_PRICING.output }
+    : { input: modelCost.input, output: modelCost.output };
+
+  // Step 4: Calculate costs
+  const inputCost = (estimatedInputTokens * pricing.input) / 1_000_000;
+  const outputCost = (estimatedOutputTokens * pricing.output) / 1_000_000;
+  const totalCost = inputCost + outputCost;
+
+  // Step 5: Generate warnings based on cost
+  const warnings: CostWarning[] = [];
+
+  if (totalCost > WARNING_THRESHOLD_CRITICAL) {
+    warnings.push({
+      level: 'critical',
+      message: 'Very high cost estimate. Strongly recommend limiting scope.',
+    });
+  } else if (totalCost > WARNING_THRESHOLD_RED) {
+    warnings.push({
+      level: 'red',
+      message: 'High cost estimate. Consider analyzing a subdirectory.',
+    });
+  } else if (totalCost > WARNING_THRESHOLD_YELLOW) {
+    warnings.push({
+      level: 'yellow',
+      message: 'Moderate cost. Consider analyzing a subdirectory for faster results.',
+    });
+  }
+
+  return {
+    model,
+    filesToScan,
+    estimatedInputTokens,
+    estimatedOutputTokens,
+    inputCost,
+    outputCost,
+    totalCost,
+    pricing: {
+      inputPerMillion: pricing.input,
+      outputPerMillion: pricing.output,
+      source: useFallback ? 'fallback' : 'litellm',
+    },
+    warnings,
+  };
+}
+
+/**
+ * Check if estimated cost exceeds a threshold.
+ *
+ * @param estimate - Cost estimate to check
+ * @param maxCost - Maximum allowed cost in USD
+ * @returns true if cost exceeds threshold
+ */
+export function exceedsMaxCost(estimate: CostEstimate, maxCost: number): boolean {
+  return estimate.totalCost > maxCost;
+}

package/src/costs.ts

@@ -0,0 +1,347 @@
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
+import { join } from 'path';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+export interface ModelCost {
+  input: number;  // per 1M tokens
+  output: number; // per 1M tokens
+}
+
+interface PricingCache {
+  data: Record<string, ModelCost>;
+  fetchedAt: number;
+  source: string;
+}
+
+// =============================================================================
+// PRICING PROVIDER INTERFACE
+// =============================================================================
+// This interface allows swapping LiteLLM for a local implementation.
+// To replace: implement PricingProvider and call setPricingProvider().
+
+export interface PricingProvider {
+  name: string;
+  fetch(): Promise<Record<string, ModelCost>>;
+}
+
+// =============================================================================
+// LITELLM PROVIDER (Default)
+// =============================================================================
+
+const LITELLM_PRICING_URL = 'https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json';
+
+function normalizeLiteLLMPricing(litellmData: Record<string, unknown>): Record<string, ModelCost> {
+  const result: Record<string, ModelCost> = {};
+
+  for (const [model, info] of Object.entries(litellmData)) {
+    if (typeof info !== 'object' || info === null) continue;
+
+    const data = info as Record<string, unknown>;
+    const inputCost = data.input_cost_per_token;
+    const outputCost = data.output_cost_per_token;
+
+    if (typeof inputCost === 'number' && typeof outputCost === 'number') {
+      // Convert per-token to per-1M-tokens
+      result[model] = {
+        input: inputCost * 1_000_000,
+        output: outputCost * 1_000_000,
+      };
+    }
+  }
+
+  return result;
+}
+
+const litellmProvider: PricingProvider = {
+  name: 'litellm',
+  async fetch(): Promise<Record<string, ModelCost>> {
+    const response = await fetch(LITELLM_PRICING_URL);
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
+    const rawData = await response.json() as Record<string, unknown>;
+    return normalizeLiteLLMPricing(rawData);
+  },
+};
+
+// =============================================================================
+// LOCAL PROVIDER (Ready for future use)
+// =============================================================================
+// Uncomment and populate to use local pricing instead of LiteLLM.
+// Then call: setPricingProvider(localProvider)
+
+/*
+const localProvider: PricingProvider = {
+  name: 'local',
+  async fetch(): Promise<Record<string, ModelCost>> {
+    // Local pricing data - $/1M tokens
+    return {
+      'gpt-4o': { input: 2.50, output: 10.00 },
+      'gpt-4o-mini': { input: 0.15, output: 0.60 },
+      'gpt-4-turbo': { input: 10.00, output: 30.00 },
+      'claude-3-5-sonnet': { input: 3.00, output: 15.00 },
+      'claude-3-haiku': { input: 0.25, output: 1.25 },
+      'claude-sonnet-4-20250514': { input: 3.00, output: 15.00 },
+      // Add more models as needed
+    };
+  },
+};
+*/
+
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+
+const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
+const CACHE_DIR = '.peakinfer/cache';
+const CACHE_FILE = 'pricing.json';
+
+// =============================================================================
+// STATE
+// =============================================================================
+
+let pricingCache: PricingCache | null = null;
+let activeProvider: PricingProvider = litellmProvider;
+
+// =============================================================================
+// PROVIDER MANAGEMENT
+// =============================================================================
+
+/**
+ * Set a custom pricing provider.
+ * Call this before loadPricing() to use a different data source.
+ *
+ * Example:
+ *   setPricingProvider({ name: 'local', fetch: async () => ({ ... }) });
+ */
+export function setPricingProvider(provider: PricingProvider): void {
+  activeProvider = provider;
+  // Invalidate cache when provider changes
+  pricingCache = null;
+}
+
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+function getCachePath(): string {
+  return join(process.cwd(), CACHE_DIR, CACHE_FILE);
+}
+
+function loadCacheFromDisk(): PricingCache | null {
+  const cachePath = getCachePath();
+  if (!existsSync(cachePath)) {
+    return null;
+  }
+  try {
+    const raw = readFileSync(cachePath, 'utf-8');
+    return JSON.parse(raw) as PricingCache;
+  } catch {
+    return null;
+  }
+}
+
+function saveCacheToDisk(cache: PricingCache): void {
+  const cachePath = getCachePath();
+  const cacheDir = join(process.cwd(), CACHE_DIR);
+  if (!existsSync(cacheDir)) {
+    mkdirSync(cacheDir, { recursive: true });
+  }
+  writeFileSync(cachePath, JSON.stringify(cache, null, 2));
+}
+
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+
+export function isCacheValid(): boolean {
+  if (!pricingCache) return false;
+  // Also invalidate if provider changed
+  if (pricingCache.source !== activeProvider.name) return false;
+  return Date.now() - pricingCache.fetchedAt < CACHE_TTL_MS;
+}
+
+export async function loadPricing(): Promise<void> {
+  // Check memory cache
+  if (isCacheValid()) {
+    return;
+  }
+
+  // Check disk cache (only if same provider)
+  const diskCache = loadCacheFromDisk();
+  if (diskCache &&
+      diskCache.source === activeProvider.name &&
+      Date.now() - diskCache.fetchedAt < CACHE_TTL_MS) {
+    pricingCache = diskCache;
+    return;
+  }
+
+  // Fetch from active provider
+  try {
+    const data = await activeProvider.fetch();
+
+    pricingCache = {
+      data,
+      fetchedAt: Date.now(),
+      source: activeProvider.name,
+    };
+
+    saveCacheToDisk(pricingCache);
+  } catch (error) {
+    // Fall back to stale cache if available and same provider
+    if (diskCache && diskCache.source === activeProvider.name) {
+      console.warn(`[costs] Failed to fetch from ${activeProvider.name}, using stale cache`);
+      pricingCache = diskCache;
+      return;
+    }
+
+    // No cache at all - use empty with warning
+    console.warn(`[costs] Failed to fetch from ${activeProvider.name}, no cache available`);
+    pricingCache = {
+      data: {},
+      fetchedAt: Date.now(),
+      source: activeProvider.name,
+    };
+  }
+}
+
+export function getModelCost(model: string): ModelCost {
+  if (!pricingCache) {
+    return { input: 0, output: 0 };
+  }
+
+  // Try exact match
+  if (pricingCache.data[model]) {
+    return pricingCache.data[model];
+  }
+
+  // Try with provider prefix variations
+  const variations = [
+    model,
+    `openai/${model}`,
+    `anthropic/${model}`,
+    `azure/${model}`,
+    `together_ai/${model}`,
+    `fireworks_ai/${model}`,
+  ];
+
+  for (const variant of variations) {
+    if (pricingCache.data[variant]) {
+      return pricingCache.data[variant];
+    }
+  }
+
+  // Try partial match (model name contains)
+  const lowerModel = model.toLowerCase();
+  for (const [key, cost] of Object.entries(pricingCache.data)) {
+    if (key.toLowerCase().includes(lowerModel)) {
+      return cost;
+    }
+  }
+
+  return { input: 0, output: 0 };
+}
+
+export function calculateCost(
+  model: string,
+  inputTokens: number,
+  outputTokens: number
+): number {
+  const cost = getModelCost(model);
+  return (inputTokens * cost.input + outputTokens * cost.output) / 1_000_000;
+}
+
+export function setTestPricing(data: Record<string, { input: number; output: number }>): void {
+  pricingCache = {
+    data,
+    fetchedAt: Date.now(),
+    source: 'test',
+  };
+}
+
+// =============================================================================
+// PRICING CONTEXT FOR LLM
+// =============================================================================
+
+/**
+ * Pricing tier classification based on cost per 1M tokens
+ */
+export type PricingTier = 'expensive' | 'moderate' | 'cheap' | 'unknown';
+
+const EXPENSIVE_THRESHOLD = 10.0; // >$10/1M = expensive
+const MODERATE_THRESHOLD = 1.0;   // $1-10/1M = moderate
+
+/**
+ * Classify a model into pricing tiers
+ */
+export function classifyModelCost(model: string): PricingTier {
+  const cost = getModelCost(model);
+  if (cost.input === 0 && cost.output === 0) {
+    return 'unknown';
+  }
+
+  // Use average of input/output cost for classification
+  const avgCost = (cost.input + cost.output) / 2;
+
+  if (avgCost > EXPENSIVE_THRESHOLD) {
+    return 'expensive';
+  } else if (avgCost > MODERATE_THRESHOLD) {
+    return 'moderate';
+  } else {
+    return 'cheap';
+  }
+}
+
+/**
+ * Get pricing context for LLM analysis
+ * Returns a condensed pricing map for models used in the data
+ */
+export interface PricingContext {
+  models: Record<string, {
+    input: number;   // $/1M tokens
+    output: number;  // $/1M tokens
+    tier: PricingTier;
+  }>;
+  thresholds: {
+    expensive: number;
+    moderate: number;
+  };
+}
+
+export function getPricingContext(models: string[]): PricingContext {
+  const result: PricingContext = {
+    models: {},
+    thresholds: {
+      expensive: EXPENSIVE_THRESHOLD,
+      moderate: MODERATE_THRESHOLD,
+    },
+  };
+
+  for (const model of models) {
+    const cost = getModelCost(model);
+    result.models[model] = {
+      input: cost.input,
+      output: cost.output,
+      tier: classifyModelCost(model),
+    };
+  }
+
+  return result;
+}
+
+/**
+ * Calculate total cost for a set of events
+ */
+export function calculateTotalCost(events: Array<{
+  model: string;
+  input_tokens: number;
+  output_tokens: number;
+}>): number {
+  return events.reduce((total, event) => {
+    return total + calculateCost(event.model, event.input_tokens, event.output_tokens);
+  }, 0);
+}