@peakinfer/cli 1.0.133

package/src/insights.ts

@@ -0,0 +1,341 @@
+import type {
+  Insight,
+  InsightTemplate,
+  TemplateCondition,
+  Callsite,
+  EnrichedCallsite,
+  JoinedOutput,
+  PerformanceEnvelope,
+} from './types.js';
+import { getEnvelope, getThroughputPercent } from './envelopes.js';
+import { getModelCost, calculateCost } from './costs.js';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+interface EvaluationContext {
+  callsites?: Callsite[] | EnrichedCallsite[];
+  joined?: JoinedOutput;
+  envelopes?: Record<string, PerformanceEnvelope>;
+}
+
+interface GlobalStats {
+  totalCost: number;
+  costByCallsite: Map<string, number>;
+  top_callsite_cost_percent: number;
+  top_callsite_id: string;
+  top_callsite_model: string;
+  top_callsite_location: string;
+}
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+function getNestedValue(obj: unknown, path: string): unknown {
+  const parts = path.split('.');
+  let current: unknown = obj;
+
+  for (const part of parts) {
+    if (current === null || current === undefined) return undefined;
+    if (typeof current !== 'object') return undefined;
+    current = (current as Record<string, unknown>)[part];
+  }
+
+  return current;
+}
+
+function evaluateCondition(condition: TemplateCondition, context: Record<string, unknown>): boolean {
+  const fieldValue = getNestedValue(context, condition.field);
+
+  switch (condition.op) {
+    case 'eq':
+      return fieldValue === condition.value;
+
+    case 'neq':
+      return fieldValue !== condition.value;
+
+    case 'gt':
+      return typeof fieldValue === 'number' && typeof condition.value === 'number' && fieldValue > condition.value;
+
+    case 'lt':
+      return typeof fieldValue === 'number' && typeof condition.value === 'number' && fieldValue < condition.value;
+
+    case 'gte':
+      return typeof fieldValue === 'number' && typeof condition.value === 'number' && fieldValue >= condition.value;
+
+    case 'lte':
+      return typeof fieldValue === 'number' && typeof condition.value === 'number' && fieldValue <= condition.value;
+
+    case 'exists':
+      return fieldValue !== null && fieldValue !== undefined;
+
+    case 'in':
+      return Array.isArray(condition.value) && condition.value.includes(fieldValue as string);
+
+    case 'ratio_gt': {
+      if (!condition.compare_to || typeof condition.value !== 'number') return false;
+      const compareValue = getNestedValue(context, condition.compare_to);
+      if (typeof fieldValue !== 'number' || typeof compareValue !== 'number' || compareValue === 0) return false;
+      return (fieldValue / compareValue) > condition.value;
+    }
+
+    case 'ratio_lt': {
+      if (!condition.compare_to || typeof condition.value !== 'number') return false;
+      const compareValue = getNestedValue(context, condition.compare_to);
+      if (typeof fieldValue !== 'number' || typeof compareValue !== 'number' || compareValue === 0) return false;
+      return (fieldValue / compareValue) < condition.value;
+    }
+
+    case 'has_pattern': {
+      if (!condition.pattern || !Array.isArray(fieldValue)) return false;
+      const pattern = condition.pattern.toLowerCase();
+      const matchCount = (fieldValue as Callsite[]).filter(c =>
+        c.patterns.fallback === true ||
+        c.file.toLowerCase().includes(pattern)
+      ).length;
+      return condition.count_gt !== undefined ? matchCount > condition.count_gt : matchCount > 0;
+    }
+
+    default:
+      return false;
+  }
+}
+
+function interpolate(template: string, vars: Record<string, unknown>): string {
+  return template.replace(/\{\{(\w+)\}\}/g, (_, key) => {
+    const value = vars[key];
+    if (value === undefined || value === null) return `{{${key}}}`;
+    if (typeof value === 'number') {
+      return Number.isInteger(value) ? value.toString() : value.toFixed(1);
+    }
+    return String(value);
+  });
+}
+
+function computeGlobalStats(callsites: EnrichedCallsite[]): GlobalStats {
+  const costByCallsite = new Map<string, number>();
+  let totalCost = 0;
+
+  for (const cs of callsites) {
+    if (cs.usage) {
+      const cost = calculateCost(cs.model || 'unknown', cs.usage.tokens_in, cs.usage.tokens_out);
+      costByCallsite.set(cs.id, cost);
+      totalCost += cost;
+    }
+  }
+
+  let topId = '';
+  let topCost = 0;
+  for (const [id, cost] of costByCallsite) {
+    if (cost > topCost) {
+      topCost = cost;
+      topId = id;
+    }
+  }
+
+  const topCallsite = callsites.find(c => c.id === topId);
+
+  return {
+    totalCost,
+    costByCallsite,
+    top_callsite_cost_percent: totalCost > 0 ? Math.round((topCost / totalCost) * 100) : 0,
+    top_callsite_id: topId,
+    top_callsite_model: topCallsite?.model || 'unknown',
+    top_callsite_location: topCallsite ? `${topCallsite.file}:${topCallsite.line}` : '',
+  };
+}
+
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+
+export function evaluate(
+  data: JoinedOutput | { callsites: Callsite[] | EnrichedCallsite[] },
+  templates: InsightTemplate[],
+  envelopes: Record<string, PerformanceEnvelope> = {}
+): Insight[] {
+  const insights: Insight[] = [];
+
+  const callsites = 'callsites' in data ? data.callsites : [];
+  const joined = 'codeOnly' in data ? data as JoinedOutput : null;
+  const globalStats = computeGlobalStats(callsites as EnrichedCallsite[]);
+
+  for (const template of templates) {
+    const { match, output, severity, category, id } = template;
+
+    switch (match.scope) {
+      case 'callsite': {
+        // Evaluate against each callsite
+        for (const callsite of callsites) {
+          const enriched = callsite as EnrichedCallsite;
+          const usage = enriched.usage;
+
+          // Build context with computed fields for condition evaluation
+          const context = {
+            ...callsite,
+            globalStats,
+            // Computed fields needed by templates
+            avg_tokens: usage ? Math.round(usage.tokens_out / usage.calls) : 0,
+            avg_tokens_in: usage ? Math.round(usage.tokens_in / usage.calls) : 0,
+            input_output_ratio: usage && usage.tokens_out > 0
+              ? Math.round(usage.tokens_in / usage.tokens_out)
+              : 0,
+          };
+          const allMatch = match.conditions.every(c => evaluateCondition(c, context));
+
+          if (allMatch) {
+            const vars: Record<string, unknown> = {
+              ...callsite,
+              location: `${callsite.file}:${callsite.line}`,
+              // Latency metrics
+              ratio: usage
+                ? (usage.latency_p99 / usage.latency_p50).toFixed(1)
+                : 'N/A',
+              p50: usage?.latency_p50,
+              p95: usage?.latency_p95,
+              p99: usage?.latency_p99,
+              // Token metrics
+              tokens_in: usage?.tokens_in || 0,
+              tokens_out: usage?.tokens_out || 0,
+              calls: usage?.calls || 0,
+              avg_tokens: usage
+                ? Math.round(usage.tokens_out / usage.calls)
+                : 0,
+              avg_tokens_in: usage
+                ? Math.round(usage.tokens_in / usage.calls)
+                : 0,
+              // Input/output ratio for prompt bloat detection
+              input_output_ratio: usage && usage.tokens_out > 0
+                ? Math.round(usage.tokens_in / usage.tokens_out)
+                : 0,
+            };
+
+            insights.push({
+              severity,
+              category,
+              templateId: id,
+              headline: interpolate(output.headline, vars),
+              evidence: interpolate(output.evidence, vars),
+              location: `${callsite.file}:${callsite.line}`,
+              source: 'template',
+            });
+          }
+        }
+        break;
+      }
+
+      case 'joined': {
+        if (!joined) continue;
+
+        const context = {
+          codeOnly: joined.codeOnly,
+          runtimeOnly: joined.runtimeOnly,
+          drift: joined.drift,
+          'codeOnly.length': joined.codeOnly.length,
+          'runtimeOnly.length': joined.runtimeOnly.length,
+        };
+
+        const allMatch = match.conditions.every(c => evaluateCondition(c, context));
+
+        if (allMatch) {
+          const vars: Record<string, unknown> = {
+            count: joined.codeOnly.length,
+            locations: joined.codeOnly
+              .slice(0, 3)
+              .map(c => `${c.file}:${c.line}`)
+              .join(', ') + (joined.codeOnly.length > 3 ? '...' : ''),
+          };
+
+          insights.push({
+            severity,
+            category,
+            templateId: id,
+            headline: interpolate(output.headline, vars),
+            evidence: interpolate(output.evidence, vars),
+            source: 'template',
+          });
+        }
+        break;
+      }
+
+      case 'global': {
+        const context = globalStats as unknown as Record<string, unknown>;
+        const allMatch = match.conditions.every(c => evaluateCondition(c, context));
+
+        if (allMatch) {
+          const vars: Record<string, unknown> = {
+            percent: globalStats.top_callsite_cost_percent,
+            model: globalStats.top_callsite_model,
+            location: globalStats.top_callsite_location,
+          };
+
+          insights.push({
+            severity,
+            category,
+            templateId: id,
+            headline: interpolate(output.headline, vars),
+            evidence: interpolate(output.evidence, vars),
+            location: globalStats.top_callsite_location,
+            source: 'template',
+          });
+        }
+        break;
+      }
+
+      case 'envelope': {
+        // Evaluate against each callsite with envelope comparison
+        for (const callsite of callsites) {
+          const enriched = callsite as EnrichedCallsite;
+          if (!enriched.usage || !callsite.model) continue;
+
+          const envelope = getEnvelope(callsite.model);
+          if (!envelope) continue;
+
+          // Calculate actual TPS (tokens per second)
+          const avgLatencySec = enriched.usage.latency_p50 / 1000;
+          const avgOutputTokens = enriched.usage.tokens_out / enriched.usage.calls;
+          const actualTps = avgLatencySec > 0 ? avgOutputTokens / avgLatencySec : 0;
+
+          const context = {
+            actual_tps: actualTps,
+            envelope,
+            'envelope.tps_median': envelope.tps_median,
+            'envelope.tps_peak': envelope.tps_peak,
+          };
+
+          const allMatch = match.conditions.every(c => evaluateCondition(c, context));
+
+          if (allMatch) {
+            const percent = getThroughputPercent(callsite.model, actualTps) || 0;
+            const vars: Record<string, unknown> = {
+              percent,
+              model: callsite.model,
+              actual: actualTps.toFixed(0),
+              reference: envelope.tps_median,
+              location: `${callsite.file}:${callsite.line}`,
+            };
+
+            insights.push({
+              severity,
+              category,
+              templateId: id,
+              headline: interpolate(output.headline, vars),
+              evidence: interpolate(output.evidence, vars),
+              location: `${callsite.file}:${callsite.line}`,
+              source: 'template',
+            });
+          }
+        }
+        break;
+      }
+    }
+  }
+
+  // Sort by severity: critical > warning > info
+  const severityOrder = { critical: 0, warning: 1, info: 2 };
+  insights.sort((a, b) => severityOrder[a.severity] - severityOrder[b.severity]);
+
+  return insights;
+}

package/src/joiner.ts

@@ -0,0 +1,289 @@
+import { percentile } from './runtime.js';
+import type { Callsite, InferenceEvent, JoinedOutput, DriftSignal, EnrichedCallsite, UsageStats } from './types.js';
+
+// =============================================================================
+// CONFIGURABLE THRESHOLDS
+// =============================================================================
+
+export interface DriftThresholds {
+  /** Percentage threshold for streaming drift (0-100). Default: 50 */
+  streamingDriftPercent: number;
+  /** Minimum events before flagging retry/fallback drift. Default: 10 */
+  minEventsForPatternDrift: number;
+}
+
+const DEFAULT_THRESHOLDS: DriftThresholds = {
+  streamingDriftPercent: 50,
+  minEventsForPatternDrift: 10,
+};
+
+const currentThresholds = { ...DEFAULT_THRESHOLDS };
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+function computeUsageStats(events: InferenceEvent[]): UsageStats {
+  const latencies = events.map(e => e.latency_ms);
+  return {
+    calls: events.length,
+    tokens_in: events.reduce((sum, e) => sum + e.input_tokens, 0),
+    tokens_out: events.reduce((sum, e) => sum + e.output_tokens, 0),
+    latency_p50: percentile(latencies, 50),
+    latency_p95: percentile(latencies, 95),
+    latency_p99: percentile(latencies, 99),
+  };
+}
+
+function makeKey(provider: string | null, model: string | null): string {
+  return `${provider || 'unknown'}:${model || 'unknown'}`;
+}
+
+/**
+ * Detect pattern drift between code-declared patterns and runtime behavior.
+ *
+ * Pattern types:
+ * - streaming: Code says stream=true but runtime shows non-streaming (or vice versa)
+ * - batching: Code declares batching but runtime shows no batch_id/batch_size
+ * - fallback: Code declares fallback but no fallback_used events seen
+ * - caching: Code declares caching but no cached events seen
+ * - retries: Code declares retries but no retry_count > 0 events seen
+ */
+function detectPatternDrift(
+  callsite: Callsite,
+  events: InferenceEvent[]
+): DriftSignal[] {
+  const driftSignals: DriftSignal[] = [];
+  const patterns = callsite.patterns;
+
+  if (events.length === 0) {
+    return driftSignals;
+  }
+
+  // Count events with each pattern
+  const streamingEvents = events.filter(e => e.streaming === true).length;
+  const nonStreamingEvents = events.filter(e => e.streaming === false).length;
+  const batchedEvents = events.filter(e => e.batch_id || (e.batch_size && e.batch_size > 1)).length;
+  const cachedEvents = events.filter(e => e.cached === true).length;
+  const retriedEvents = events.filter(e => e.retry_count && e.retry_count > 0).length;
+  const fallbackEvents = events.filter(e => e.fallback_used === true).length;
+
+  // Calculate percentages
+  const streamingPercent = (streamingEvents / events.length) * 100;
+  const nonStreamingPercent = (nonStreamingEvents / events.length) * 100;
+
+  // 1. Streaming drift detection
+  if (patterns.streaming === true) {
+    // Code declares streaming, check if runtime confirms
+    if (nonStreamingPercent > currentThresholds.streamingDriftPercent) {
+      driftSignals.push({
+        type: 'patternDrift',
+        callsiteId: callsite.id,
+        provider: callsite.provider || undefined,
+        model: callsite.model || undefined,
+        message: `Streaming declared in code but ${nonStreamingPercent.toFixed(0)}% of runtime calls are non-streaming`,
+      });
+    } else if (streamingEvents === 0 && nonStreamingEvents === 0) {
+      // No streaming info available - soft warning
+      driftSignals.push({
+        type: 'patternDrift',
+        callsiteId: callsite.id,
+        provider: callsite.provider || undefined,
+        model: callsite.model || undefined,
+        message: `Streaming declared in code but cannot verify from runtime data (no TTFT metrics)`,
+      });
+    }
+  } else if (patterns.streaming === false) {
+    // Code explicitly non-streaming, but runtime shows streaming
+    if (streamingPercent > currentThresholds.streamingDriftPercent) {
+      driftSignals.push({
+        type: 'patternDrift',
+        callsiteId: callsite.id,
+        provider: callsite.provider || undefined,
+        model: callsite.model || undefined,
+        message: `Non-streaming declared in code but ${streamingPercent.toFixed(0)}% of runtime calls appear to stream`,
+      });
+    }
+  }
+
+  // 2. Batching drift detection
+  if (patterns.batching === true && batchedEvents === 0) {
+    driftSignals.push({
+      type: 'patternDrift',
+      callsiteId: callsite.id,
+      provider: callsite.provider || undefined,
+      model: callsite.model || undefined,
+      message: `Batching declared in code but no batched requests observed at runtime`,
+    });
+  }
+
+  // 3. Caching drift detection
+  if (patterns.caching === true && cachedEvents === 0) {
+    driftSignals.push({
+      type: 'patternDrift',
+      callsiteId: callsite.id,
+      provider: callsite.provider || undefined,
+      model: callsite.model || undefined,
+      message: `Caching declared in code but no cache hits observed at runtime`,
+    });
+  }
+
+  // 4. Retry drift detection
+  if (patterns.retries === true && retriedEvents === 0) {
+    // This could be good (no failures) or indicate retries aren't working
+    // Only flag if there are a significant number of events
+    if (events.length >= currentThresholds.minEventsForPatternDrift) {
+      driftSignals.push({
+        type: 'patternDrift',
+        callsiteId: callsite.id,
+        provider: callsite.provider || undefined,
+        model: callsite.model || undefined,
+        message: `Retries declared in code but no retry events observed (${events.length} calls, 0 retries)`,
+      });
+    }
+  }
+
+  // 5. Fallback drift detection
+  if (patterns.fallback === true && fallbackEvents === 0) {
+    // This could be good (primary always works) or indicate fallback isn't working
+    // Only flag if there are a significant number of events
+    if (events.length >= currentThresholds.minEventsForPatternDrift) {
+      driftSignals.push({
+        type: 'patternDrift',
+        callsiteId: callsite.id,
+        provider: callsite.provider || undefined,
+        model: callsite.model || undefined,
+        message: `Fallback declared in code but never triggered at runtime (${events.length} calls)`,
+      });
+    }
+  }
+
+  return driftSignals;
+}
+
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+
+export function join(callsites: Callsite[], events: InferenceEvent[]): JoinedOutput {
+  const codeOnly: Callsite[] = [];
+  const runtimeOnly: InferenceEvent[] = [];
+  const drift: DriftSignal[] = [];
+  const enrichedCallsites: EnrichedCallsite[] = [];
+
+  // Separate events: those with callsite_id vs those without
+  const eventsByCallsiteId = new Map<string, InferenceEvent[]>();
+  const eventsWithoutCallsiteId: InferenceEvent[] = [];
+
+  for (const event of events) {
+    if (event.callsite_id) {
+      // Events with callsite_id ONLY match by callsite_id, not by provider+model
+      if (!eventsByCallsiteId.has(event.callsite_id)) {
+        eventsByCallsiteId.set(event.callsite_id, []);
+      }
+      eventsByCallsiteId.get(event.callsite_id)!.push(event);
+    } else {
+      // Events without callsite_id will be matched by provider+model
+      eventsWithoutCallsiteId.push(event);
+    }
+  }
+
+  // Group events without callsite_id by provider+model
+  const eventsByKey = new Map<string, InferenceEvent[]>();
+  for (const event of eventsWithoutCallsiteId) {
+    const key = makeKey(event.provider, event.model);
+    if (!eventsByKey.has(key)) {
+      eventsByKey.set(key, []);
+    }
+    eventsByKey.get(key)!.push(event);
+  }
+
+  // Track which keys have been matched
+  const matchedKeys = new Set<string>();
+  const matchedCallsiteIds = new Set<string>();
+
+  // Match callsites to events
+  for (const callsite of callsites) {
+    let matchedEvents: InferenceEvent[] = [];
+
+    // Priority 1: Match by callsite_id (events that explicitly reference this callsite)
+    if (eventsByCallsiteId.has(callsite.id)) {
+      matchedEvents = eventsByCallsiteId.get(callsite.id)!;
+      matchedCallsiteIds.add(callsite.id);
+    }
+
+    // Priority 2: Match by provider+model (only for events without callsite_id)
+    if (matchedEvents.length === 0) {
+      const key = makeKey(callsite.provider, callsite.model);
+      const keyEvents = eventsByKey.get(key);
+      if (keyEvents && keyEvents.length > 0) {
+        matchedEvents = keyEvents;
+        matchedKeys.add(key);
+      }
+    }
+
+    // Build enriched callsite
+    if (matchedEvents.length > 0) {
+      const usage = computeUsageStats(matchedEvents);
+      enrichedCallsites.push({ ...callsite, usage });
+
+      // Detect pattern drift for matched callsites
+      const patternDrift = detectPatternDrift(callsite, matchedEvents);
+      drift.push(...patternDrift);
+    } else {
+      // No matching events - this is code-only
+      enrichedCallsites.push(callsite);
+      codeOnly.push(callsite);
+
+      drift.push({
+        type: 'codeOnly',
+        provider: callsite.provider || undefined,
+        model: callsite.model || undefined,
+        callsiteId: callsite.id,
+        message: `Callsite ${callsite.file}:${callsite.line} has no runtime events`,
+      });
+    }
+  }
+
+  // Find runtime-only events
+  // 1. Events with callsite_id that don't match any callsite
+  for (const [callsiteId, evts] of eventsByCallsiteId) {
+    if (!matchedCallsiteIds.has(callsiteId)) {
+      runtimeOnly.push(...evts);
+    }
+  }
+
+  // 2. Events without callsite_id whose key wasn't matched
+  for (const [key, evts] of eventsByKey) {
+    if (!matchedKeys.has(key)) {
+      runtimeOnly.push(...evts);
+    }
+  }
+
+  // Generate drift signals for runtime-only events (grouped by provider+model)
+  const runtimeOnlyByKey = new Map<string, InferenceEvent[]>();
+  for (const event of runtimeOnly) {
+    const key = makeKey(event.provider, event.model);
+    if (!runtimeOnlyByKey.has(key)) {
+      runtimeOnlyByKey.set(key, []);
+    }
+    runtimeOnlyByKey.get(key)!.push(event);
+  }
+
+  for (const [key, evts] of runtimeOnlyByKey) {
+    const [provider, model] = key.split(':');
+    drift.push({
+      type: 'runtimeOnly',
+      provider: provider || undefined,
+      model: model || undefined,
+      message: `${evts.length} events for ${key} with no matching code`,
+    });
+  }
+
+  return {
+    callsites: enrichedCallsites,
+    codeOnly,
+    runtimeOnly,
+    drift,
+  };
+}