@peakinfer/cli 1.0.133

package/src/runid.ts

@@ -0,0 +1,261 @@
+import { createHash } from 'crypto';
+import { existsSync, readFileSync, statSync, readdirSync } from 'fs';
+import { join } from 'path';
+import type { InferenceMap, Insight, JoinedOutput, RuntimeSummary } from './types.js';
+import { VERSION } from './version.js';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+export interface RunInputs {
+  repoRoot?: string;
+  eventsPath?: string;
+  offline?: boolean;
+}
+
+export interface RunManifest {
+  runId: string;
+  version: string;
+  createdAt: string;
+  inputs: {
+    repoRoot?: string;
+    repoHash?: string;
+    eventsPath?: string;
+    eventsHash?: string;
+    offline: boolean;
+  };
+  artifacts: string[];
+  status: 'complete' | 'partial' | 'failed';
+}
+
+export interface CachedArtifacts {
+  inferenceMap?: InferenceMap;
+  insights?: Insight[];
+  joined?: JoinedOutput;
+  runtime?: RuntimeSummary;
+}
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+/**
+ * Hash a file's content for change detection
+ */
+function hashFile(filePath: string): string {
+  try {
+    const content = readFileSync(filePath, 'utf-8');
+    return createHash('sha256').update(content).digest('hex').slice(0, 16);
+  } catch {
+    return 'missing';
+  }
+}
+
+/**
+ * Hash a directory structure (file paths + sizes + mtimes)
+ * This is fast and detects most changes without reading all file contents
+ */
+function hashDirectory(dirPath: string, maxDepth: number = 5): string {
+  const hash = createHash('sha256');
+
+  function walk(dir: string, depth: number): void {
+    if (depth > maxDepth) return;
+
+    try {
+      const entries = readdirSync(dir, { withFileTypes: true });
+      entries.sort((a, b) => a.name.localeCompare(b.name));
+
+      for (const entry of entries) {
+        const fullPath = join(dir, entry.name);
+
+        // Skip common non-code directories
+        if (entry.isDirectory()) {
+          if (['node_modules', 'dist', '.git', '__pycache__', '.peakinfer', '.venv', 'venv'].includes(entry.name)) {
+            continue;
+          }
+          walk(fullPath, depth + 1);
+        } else {
+          // Only hash source files
+          if (/\.(ts|tsx|js|jsx|py|go|java|rs|rb|php|cs)$/.test(entry.name)) {
+            try {
+              const stat = statSync(fullPath);
+              hash.update(`${fullPath}:${stat.size}:${stat.mtimeMs}`);
+            } catch {
+              // Skip files we can't stat
+            }
+          }
+        }
+      }
+    } catch {
+      // Skip directories we can't read
+    }
+  }
+
+  walk(dirPath, 0);
+  return hash.digest('hex').slice(0, 16);
+}
+
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+
+/**
+ * Generate a deterministic run ID based on inputs
+ *
+ * runId = hash(version, repoHash?, eventsHash?, offline)
+ *
+ * This ensures:
+ * - Same inputs = same runId = can resume
+ * - Changed inputs = new runId = fresh analysis
+ */
+export function generateRunId(inputs: RunInputs): string {
+  const hash = createHash('sha256');
+
+  // Version ensures cache invalidation on tool updates
+  hash.update(`v:${VERSION}`);
+
+  // Hash repo structure if provided
+  if (inputs.repoRoot && existsSync(inputs.repoRoot)) {
+    const repoHash = hashDirectory(inputs.repoRoot);
+    hash.update(`repo:${repoHash}`);
+  }
+
+  // Hash events file if provided
+  if (inputs.eventsPath && existsSync(inputs.eventsPath)) {
+    const eventsHash = hashFile(inputs.eventsPath);
+    hash.update(`events:${eventsHash}`);
+  }
+
+  // Offline mode affects template loading
+  hash.update(`offline:${inputs.offline ?? false}`);
+
+  return hash.digest('hex').slice(0, 12);
+}
+
+/**
+ * Get the run directory path
+ */
+export function getRunDir(baseDir: string, runId: string): string {
+  return join(baseDir, 'runs', runId);
+}
+
+/**
+ * Create run manifest
+ */
+export function createManifest(
+  runId: string,
+  inputs: RunInputs,
+  artifacts: string[],
+  status: 'complete' | 'partial' | 'failed'
+): RunManifest {
+  return {
+    runId,
+    version: VERSION,
+    createdAt: new Date().toISOString(),
+    inputs: {
+      repoRoot: inputs.repoRoot,
+      repoHash: inputs.repoRoot && existsSync(inputs.repoRoot)
+        ? hashDirectory(inputs.repoRoot)
+        : undefined,
+      eventsPath: inputs.eventsPath,
+      eventsHash: inputs.eventsPath && existsSync(inputs.eventsPath)
+        ? hashFile(inputs.eventsPath)
+        : undefined,
+      offline: inputs.offline ?? false,
+    },
+    artifacts,
+    status,
+  };
+}
+
+/**
+ * Load run manifest if it exists
+ */
+export function loadManifest(runDir: string): RunManifest | null {
+  const manifestPath = join(runDir, 'manifest.json');
+
+  if (!existsSync(manifestPath)) {
+    return null;
+  }
+
+  try {
+    const content = readFileSync(manifestPath, 'utf-8');
+    return JSON.parse(content) as RunManifest;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Check if a run can be resumed (all artifacts exist and inputs haven't changed)
+ */
+export function canResume(runDir: string, inputs: RunInputs): boolean {
+  const manifest = loadManifest(runDir);
+
+  if (!manifest) {
+    return false;
+  }
+
+  // Check status
+  if (manifest.status !== 'complete') {
+    return false;
+  }
+
+  // Check version match
+  if (manifest.version !== VERSION) {
+    return false;
+  }
+
+  // Verify input hashes still match
+  if (inputs.repoRoot && existsSync(inputs.repoRoot)) {
+    const currentRepoHash = hashDirectory(inputs.repoRoot);
+    if (currentRepoHash !== manifest.inputs.repoHash) {
+      return false;
+    }
+  }
+
+  if (inputs.eventsPath && existsSync(inputs.eventsPath)) {
+    const currentEventsHash = hashFile(inputs.eventsPath);
+    if (currentEventsHash !== manifest.inputs.eventsHash) {
+      return false;
+    }
+  }
+
+  // Verify all artifacts exist
+  for (const artifact of manifest.artifacts) {
+    if (!existsSync(join(runDir, artifact))) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/**
+ * Load cached artifacts from a previous run
+ */
+export function loadCachedArtifacts(runDir: string): CachedArtifacts {
+  const artifacts: CachedArtifacts = {};
+
+  const files = [
+    { name: 'inferencemap.json', key: 'inferenceMap' },
+    { name: 'insights.json', key: 'insights' },
+    { name: 'joined.json', key: 'joined' },
+    { name: 'runtime.json', key: 'runtime' },
+  ] as const;
+
+  for (const file of files) {
+    const filePath = join(runDir, file.name);
+    if (existsSync(filePath)) {
+      try {
+        const content = readFileSync(filePath, 'utf-8');
+        (artifacts as Record<string, unknown>)[file.key] = JSON.parse(content);
+      } catch {
+        // Skip corrupted files
+      }
+    }
+  }
+
+  return artifacts;
+}

package/src/runtime.ts

@@ -0,0 +1,450 @@
+import { readFileSync, existsSync } from 'fs';
+import { extname } from 'path';
+import { InferenceEvent, RuntimeSummary, ProviderStats, NormalizationOptions, NormalizationResult } from './types.js';
+import { detectFormat, normalizeRuntimeEvents, requiresAgentNormalization } from './format-normalizer.js';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+interface ParseError {
+  line: number;
+  field: string;
+  message: string;
+}
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+function validateEvent(data: unknown, lineNum: number): InferenceEvent {
+  if (typeof data !== 'object' || data === null) {
+    throw new Error(`Line ${lineNum}: Expected object, got ${typeof data}`);
+  }
+
+  const obj = data as Record<string, unknown>;
+  const errors: string[] = [];
+
+  // Required fields
+  if (typeof obj.id !== 'string') {
+    errors.push(`Missing or invalid 'id' field`);
+  }
+  if (typeof obj.ts !== 'string') {
+    errors.push(`Missing or invalid 'ts' field`);
+  }
+  if (typeof obj.provider !== 'string') {
+    errors.push(`Missing or invalid 'provider' field`);
+  }
+  if (typeof obj.model !== 'string') {
+    errors.push(`Missing or invalid 'model' field`);
+  }
+  if (typeof obj.input_tokens !== 'number') {
+    errors.push(`Missing or invalid 'input_tokens' field`);
+  }
+  if (typeof obj.output_tokens !== 'number') {
+    errors.push(`Missing or invalid 'output_tokens' field`);
+  }
+  if (typeof obj.latency_ms !== 'number') {
+    errors.push(`Missing or invalid 'latency_ms' field`);
+  }
+
+  if (errors.length > 0) {
+    throw new Error(`Line ${lineNum}: ${errors.join(', ')}`);
+  }
+
+  // Build event with optional fields
+  const event: InferenceEvent = {
+    id: obj.id as string,
+    ts: obj.ts as string,
+    provider: obj.provider as InferenceEvent['provider'],
+    model: obj.model as string,
+    input_tokens: obj.input_tokens as number,
+    output_tokens: obj.output_tokens as number,
+    latency_ms: obj.latency_ms as number,
+    intent: obj.intent as string | undefined,
+    callsite_id: obj.callsite_id as string | undefined,
+    // Runtime pattern fields
+    streaming: typeof obj.streaming === 'boolean' ? obj.streaming : undefined,
+    ttft_ms: typeof obj.ttft_ms === 'number' ? obj.ttft_ms : undefined,
+    batch_size: typeof obj.batch_size === 'number' ? obj.batch_size : undefined,
+    batch_id: typeof obj.batch_id === 'string' ? obj.batch_id : undefined,
+    cached: typeof obj.cached === 'boolean' ? obj.cached : undefined,
+    retry_count: typeof obj.retry_count === 'number' ? obj.retry_count : undefined,
+    fallback_used: typeof obj.fallback_used === 'boolean' ? obj.fallback_used : undefined,
+    original_model: typeof obj.original_model === 'string' ? obj.original_model : undefined,
+  };
+
+  // Infer streaming if not explicitly provided but ttft_ms is present
+  if (event.streaming === undefined && event.ttft_ms !== undefined) {
+    event.streaming = true;
+  }
+
+  return event;
+}
+
+/**
+ * Detect streaming based on heuristics when not explicitly provided.
+ *
+ * Heuristics:
+ * 1. If ttft_ms is present and much smaller than total latency → streaming
+ * 2. If output_tokens/latency_ms ratio suggests real-time generation → likely streaming
+ * 3. If latency pattern shows consistent per-token timing → streaming
+ */
+function inferStreamingFromHeuristics(events: InferenceEvent[]): InferenceEvent[] {
+  return events.map(event => {
+    // If streaming is already known, skip
+    if (event.streaming !== undefined) {
+      return event;
+    }
+
+    // Heuristic 1: If ttft_ms present and < 50% of total latency, likely streaming
+    if (event.ttft_ms !== undefined) {
+      event.streaming = event.ttft_ms < event.latency_ms * 0.5;
+      return event;
+    }
+
+    // Heuristic 2: Calculate tokens per second
+    // Streaming typically shows 20-150 TPS, non-streaming shows all tokens at once
+    const outputTokens = event.output_tokens;
+    const latencySeconds = event.latency_ms / 1000;
+    const tps = outputTokens / latencySeconds;
+
+    // Very high TPS (>200) with significant output suggests non-streaming (batch return)
+    // Very low TPS (<5) with significant output suggests non-streaming (slow model, one response)
+    // TPS in 20-150 range with reasonable output suggests streaming
+    if (outputTokens > 50) {
+      if (tps >= 20 && tps <= 150) {
+        // Likely streaming - consistent token generation rate
+        event.streaming = true;
+      } else if (tps > 200) {
+        // Likely non-streaming - tokens returned all at once (cached or fast response)
+        event.streaming = false;
+      }
+      // If TPS < 20, could be either - leave as undefined
+    }
+
+    return event;
+  });
+}
+
+function parseJSONL(content: string): InferenceEvent[] {
+  const lines = content.trim().split('\n').filter(l => l.trim());
+  const events: InferenceEvent[] = [];
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line) continue;
+
+    try {
+      const data = JSON.parse(line);
+      events.push(validateEvent(data, i + 1));
+    } catch (error) {
+      if (error instanceof SyntaxError) {
+        throw new Error(`Line ${i + 1}: Invalid JSON`);
+      }
+      throw error;
+    }
+  }
+
+  return events;
+}
+
+function parseJSONArray(content: string): InferenceEvent[] {
+  let data: unknown[];
+  try {
+    data = JSON.parse(content);
+  } catch {
+    throw new Error('Invalid JSON');
+  }
+
+  if (!Array.isArray(data)) {
+    throw new Error('Expected JSON array');
+  }
+
+  return data.map((item, i) => validateEvent(item, i + 1));
+}
+
+function parseCSV(content: string): InferenceEvent[] {
+  const lines = content.trim().split('\n');
+  if (lines.length < 2) {
+    return [];
+  }
+
+  const headers = lines[0].split(',').map(h => h.trim());
+  const events: InferenceEvent[] = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const values = lines[i].split(',').map(v => v.trim());
+    const obj: Record<string, unknown> = {};
+
+    for (let j = 0; j < headers.length; j++) {
+      const header = headers[j];
+      const value = values[j];
+
+      // Convert numeric fields
+      if (['input_tokens', 'output_tokens', 'latency_ms'].includes(header)) {
+        obj[header] = parseFloat(value);
+      } else {
+        obj[header] = value;
+      }
+    }
+
+    events.push(validateEvent(obj, i + 1));
+  }
+
+  return events;
+}
+
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+
+/**
+ * Extended parse result with normalization metadata.
+ */
+export interface ParseResult {
+  events: InferenceEvent[];
+  normalization?: NormalizationResult;
+  warnings: string[];
+}
+
+/**
+ * Parse runtime events from a file.
+ *
+ * This function implements the format detection pipeline (PRD §6.4):
+ * 1. Try direct parsing for known formats (JSONL, JSON, CSV)
+ * 2. Fall back to agent-based normalization for unknown formats
+ * 3. Apply streaming inference heuristics
+ *
+ * @param path - Path to the events file
+ * @param options - Normalization options (format hints, mappings, etc.)
+ */
+export async function parseEvents(
+  path: string,
+  options: NormalizationOptions = {},
+): Promise<InferenceEvent[]> {
+  const result = await parseEventsWithMetadata(path, options);
+  return result.events;
+}
+
+/**
+ * Parse runtime events with full normalization metadata.
+ * Use this for detailed reporting on format detection and field mappings.
+ */
+export async function parseEventsWithMetadata(
+  path: string,
+  options: NormalizationOptions = {},
+): Promise<ParseResult> {
+  if (!existsSync(path)) {
+    throw new Error(`File not found: ${path}`);
+  }
+
+  const content = readFileSync(path, 'utf-8');
+  const ext = extname(path).toLowerCase();
+  const warnings: string[] = [];
+
+  // Step 1: Try direct parsing for standard formats
+  if (!options.format_hint) {
+    try {
+      let events: InferenceEvent[] | null = null;
+
+      switch (ext) {
+        case '.jsonl':
+        case '.ndjson':
+          events = parseJSONL(content);
+          break;
+        case '.json':
+          // Could be JSON array or complex format (OTEL, Jaeger, etc.)
+          if (content.trim().startsWith('[')) {
+            try {
+              events = parseJSONArray(content);
+            } catch {
+              // JSON array but not InferenceEvent schema - needs normalization
+            }
+          }
+          break;
+        case '.csv':
+          events = parseCSV(content);
+          break;
+        case '.tsv':
+          events = parseTSV(content);
+          break;
+      }
+
+      if (events && events.length > 0) {
+        // Direct parse succeeded
+        return {
+          events: inferStreamingFromHeuristics(events),
+          warnings,
+        };
+      }
+    } catch (error) {
+      // Direct parse failed - will try format normalization
+      warnings.push(`Direct parse failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  // Step 2: Detect format and check if agent normalization is needed
+  const detection = detectFormat(content, path);
+
+  // If user provided strict mode and we need agent, fail early
+  if (options.strict && detection.requires_agent) {
+    throw new Error(
+      `Format "${detection.format_type}" requires agent normalization. ` +
+      `Use --format to specify the format, or remove --strict flag.`
+    );
+  }
+
+  // If direct-parse format detected with low confidence, warn but try
+  if (!detection.requires_agent && detection.confidence < 0.9) {
+    warnings.push(
+      `Format detection confidence is ${(detection.confidence * 100).toFixed(0)}%. ` +
+      `Results may be incomplete.`
+    );
+  }
+
+  // Step 3: Use format normalizer for complex formats
+  if (detection.requires_agent || options.format_hint) {
+    const { events, normalization, errors } = await normalizeRuntimeEvents(content, options);
+
+    if (errors.length > 0 && options.strict) {
+      throw new Error(`Normalization errors: ${errors.slice(0, 3).join('; ')}`);
+    }
+
+    warnings.push(...normalization.warnings);
+
+    return {
+      events: inferStreamingFromHeuristics(events),
+      normalization,
+      warnings,
+    };
+  }
+
+  // Step 4: Final fallback - try auto-detection
+  let events: InferenceEvent[];
+
+  if (content.trim().startsWith('[')) {
+    events = parseJSONArray(content);
+  } else if (content.trim().startsWith('{')) {
+    events = parseJSONL(content);
+  } else {
+    throw new Error(
+      `Unknown file format: ${ext}. ` +
+      `Supported formats: .jsonl, .json, .csv, .tsv, or observability exports (OTEL, Jaeger, Zipkin). ` +
+      `Use --format to specify the format type.`
+    );
+  }
+
+  return {
+    events: inferStreamingFromHeuristics(events),
+    warnings,
+  };
+}
+
+/**
+ * Parse TSV (tab-separated values) format.
+ */
+function parseTSV(content: string): InferenceEvent[] {
+  const lines = content.trim().split('\n');
+  if (lines.length < 2) {
+    return [];
+  }
+
+  const headers = lines[0].split('\t').map(h => h.trim());
+  const events: InferenceEvent[] = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const values = lines[i].split('\t').map(v => v.trim());
+    const obj: Record<string, unknown> = {};
+
+    for (let j = 0; j < headers.length; j++) {
+      const header = headers[j];
+      const value = values[j];
+
+      // Convert numeric fields
+      if (['input_tokens', 'output_tokens', 'latency_ms', 'ttft_ms', 'batch_size', 'retry_count'].includes(header)) {
+        obj[header] = parseFloat(value);
+      } else if (['streaming', 'cached', 'fallback_used'].includes(header)) {
+        obj[header] = value.toLowerCase() === 'true';
+      } else {
+        obj[header] = value;
+      }
+    }
+
+    events.push(validateEvent(obj, i + 1));
+  }
+
+  return events;
+}
+
+export function percentile(values: number[], p: number): number {
+  if (values.length === 0) return 0;
+
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.ceil((p / 100) * sorted.length) - 1;
+  return sorted[Math.max(0, index)];
+}
+
+export function aggregate(events: InferenceEvent[]): RuntimeSummary {
+  if (events.length === 0) {
+    return {
+      totalEvents: 0,
+      byProvider: {},
+      byModel: {},
+      global: { p50: 0, p95: 0, p99: 0 },
+    };
+  }
+
+  const byProvider: Record<string, InferenceEvent[]> = {};
+  const byModel: Record<string, InferenceEvent[]> = {};
+  const allLatencies: number[] = [];
+
+  for (const event of events) {
+    // Group by provider
+    if (!byProvider[event.provider]) {
+      byProvider[event.provider] = [];
+    }
+    byProvider[event.provider].push(event);
+
+    // Group by model
+    if (!byModel[event.model]) {
+      byModel[event.model] = [];
+    }
+    byModel[event.model].push(event);
+
+    allLatencies.push(event.latency_ms);
+  }
+
+  const computeStats = (group: InferenceEvent[]): ProviderStats => {
+    const latencies = group.map(e => e.latency_ms);
+    return {
+      calls: group.length,
+      tokens_in: group.reduce((sum, e) => sum + e.input_tokens, 0),
+      tokens_out: group.reduce((sum, e) => sum + e.output_tokens, 0),
+      latency_p50: percentile(latencies, 50),
+      latency_p95: percentile(latencies, 95),
+      latency_p99: percentile(latencies, 99),
+    };
+  };
+
+  const providerStats: Record<string, ProviderStats> = {};
+  for (const [provider, group] of Object.entries(byProvider)) {
+    providerStats[provider] = computeStats(group);
+  }
+
+  const modelStats: Record<string, ProviderStats> = {};
+  for (const [model, group] of Object.entries(byModel)) {
+    modelStats[model] = computeStats(group);
+  }
+
+  return {
+    totalEvents: events.length,
+    byProvider: providerStats,
+    byModel: modelStats,
+    global: {
+      p50: percentile(allLatencies, 50),
+      p95: percentile(allLatencies, 95),
+      p99: percentile(allLatencies, 99),
+    },
+  };
+}