@peakinfer/cli 1.0.133

package/fixtures/demo-project/src/llm-client.ts

@@ -0,0 +1,155 @@
+/**
+ * Sample LLM Client for PeakInfer v1.5 Demo
+ *
+ * This file contains various LLM inference patterns to demonstrate
+ * the v1.5 features: predictions, counterfactuals, and comparison.
+ */
+
+import OpenAI from 'openai';
+import Anthropic from '@anthropic-ai/sdk';
+
+// Initialize clients
+const openai = new OpenAI();
+const anthropic = new Anthropic();
+
+// ============================================================================
+// High-latency calls (will trigger prediction warnings)
+// ============================================================================
+
+/**
+ * Chat completion with GPT-4 (high latency, high cost)
+ * Prediction: p95 ~5000ms, high risk
+ */
+export async function chatWithGPT4(prompt: string): Promise<string> {
+  const response = await openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: prompt }],
+    max_tokens: 2000,
+  });
+  return response.choices[0].message.content || '';
+}
+
+/**
+ * Complex analysis with Claude Opus (highest latency)
+ * Prediction: p95 ~8000ms, high risk
+ */
+export async function analyzeWithOpus(document: string): Promise<string> {
+  const response = await anthropic.messages.create({
+    model: 'claude-3-opus-20240229',
+    max_tokens: 4000,
+    messages: [{ role: 'user', content: `Analyze this document:\n${document}` }],
+  });
+  return response.content[0].type === 'text' ? response.content[0].text : '';
+}
+
+// ============================================================================
+// Medium-latency calls
+// ============================================================================
+
+/**
+ * Summarization with GPT-4 Turbo (medium latency)
+ * Prediction: p95 ~4000ms, medium risk
+ */
+export async function summarize(text: string): Promise<string> {
+  const response = await openai.chat.completions.create({
+    model: 'gpt-4-turbo',
+    messages: [
+      { role: 'system', content: 'You are a summarization assistant.' },
+      { role: 'user', content: `Summarize: ${text}` },
+    ],
+    max_tokens: 500,
+  });
+  return response.choices[0].message.content || '';
+}
+
+/**
+ * Translation with Claude Sonnet (medium latency, good value)
+ * Prediction: p95 ~4000ms, medium risk
+ */
+export async function translate(text: string, targetLang: string): Promise<string> {
+  const response = await anthropic.messages.create({
+    model: 'claude-3-sonnet-20240229',
+    max_tokens: 2000,
+    messages: [{ role: 'user', content: `Translate to ${targetLang}: ${text}` }],
+  });
+  return response.content[0].type === 'text' ? response.content[0].text : '';
+}
+
+// ============================================================================
+// Low-latency calls (optimized patterns)
+// ============================================================================
+
+/**
+ * Quick chat with GPT-4o-mini (low latency, low cost)
+ * Prediction: p95 ~1500ms, low risk
+ * Counterfactual: Other calls could use this model
+ */
+export async function quickChat(prompt: string): Promise<string> {
+  const response = await openai.chat.completions.create({
+    model: 'gpt-4o-mini',
+    messages: [{ role: 'user', content: prompt }],
+    max_tokens: 200,
+  });
+  return response.choices[0].message.content || '';
+}
+
+/**
+ * Fast response with Claude Haiku (lowest latency)
+ * Prediction: p95 ~1500ms, low risk
+ */
+export async function fastResponse(prompt: string): Promise<string> {
+  const response = await anthropic.messages.create({
+    model: 'claude-3-haiku-20240307',
+    max_tokens: 500,
+    messages: [{ role: 'user', content: prompt }],
+  });
+  return response.content[0].type === 'text' ? response.content[0].text : '';
+}
+
+// ============================================================================
+// Streaming (will trigger streaming counterfactual for non-streaming calls)
+// ============================================================================
+
+/**
+ * Streaming chat (good pattern - low perceived latency)
+ * Counterfactual: Other calls should enable streaming
+ */
+export async function* streamingChat(prompt: string): AsyncGenerator<string> {
+  const stream = await openai.chat.completions.create({
+    model: 'gpt-4o',
+    messages: [{ role: 'user', content: prompt }],
+    stream: true,
+  });
+
+  for await (const chunk of stream) {
+    const content = chunk.choices[0]?.delta?.content;
+    if (content) yield content;
+  }
+}
+
+// ============================================================================
+// Embeddings (separate from chat)
+// ============================================================================
+
+/**
+ * Generate embeddings (low latency, batch-friendly)
+ * Counterfactual: Should enable batching
+ */
+export async function embed(text: string): Promise<number[]> {
+  const response = await openai.embeddings.create({
+    model: 'text-embedding-3-small',
+    input: text,
+  });
+  return response.data[0].embedding;
+}
+
+/**
+ * Batch embedding (good pattern)
+ */
+export async function batchEmbed(texts: string[]): Promise<number[][]> {
+  const response = await openai.embeddings.create({
+    model: 'text-embedding-3-small',
+    input: texts,
+  });
+  return response.data.map(d => d.embedding);
+}

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@peakinfer/cli",
+  "version": "1.0.133",
+  "description": "LLM inference performance analysis CLI",
+  "type": "module",
+  "main": "dist/cli.js",
+  "bin": {
+    "peakinfer": "dist/cli.js"
+  },
+  "scripts": {
+    "prebuild": "npm version patch --no-git-tag-version",
+    "build": "tsc",
+    "build:noversion": "tsc",
+    "dev": "tsx src/cli.ts",
+    "start": "node dist/cli.js",
+    "test": "vitest",
+    "test:watch": "vitest --watch",
+    "test:perf": "vitest run tests/performance/",
+    "benchmark": "tsx scripts/benchmark.ts",
+    "benchmark:ci": "tsx scripts/benchmark.ts --ci",
+    "sync": "bash scripts/sync-from-site.sh",
+    "sync:check": "bash scripts/sync-from-site.sh --check",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint src/",
+    "clean": "rm -rf dist"
+  },
+  "keywords": [
+    "llm",
+    "inference",
+    "performance",
+    "analysis",
+    "openai",
+    "anthropic"
+  ],
+  "author": "PeakInfer",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/peakinfer/cli"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.1.76",
+    "@anthropic-ai/sdk": "^0.27.0",
+    "chalk": "^5.6.2",
+    "commander": "^12.0.0",
+    "dotenv": "^17.2.3",
+    "glob": "^10.3.10",
+    "ignore": "^5.3.0",
+    "ora": "^9.0.0",
+    "posthog-node": "^4.0.0",
+    "puppeteer": "^24.33.0",
+    "yaml": "^2.3.4",
+    "zod": "^3.22.0"
+  },
+  "devDependencies": {
+    "@types/glob": "^8.1.0",
+    "@types/node": "^20.10.0",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.0",
+    "vitest": "^1.2.0"
+  }
+}

package/prompts/agent-analyzer.yaml

@@ -0,0 +1,47 @@
+# Agent-based Semantic Analyzer Prompt
+# Used for multi-step code analysis with tool use
+id: agent-analyzer
+version: "1.0.0"
+description: Expert code analyst for identifying LLM inference points with tool use capability
+
+prompt: |
+  You are an expert code analyst specializing in identifying LLM/AI inference points in source code.
+
+  Your task is to analyze code and find ALL actual LLM inference points with accurate provider and model information.
+
+  ## CRITICAL RULES
+
+  ### What IS an inference point (DO report these):
+  - client.chat.completions.create() - OpenAI API call
+  - client.messages.create() - Anthropic API call
+  - client.embeddings.create() - OpenAI embeddings call
+  - predictor(question=...) - DSPy module invocation (after dspy.Predict/ChainOfThought)
+  - chain.invoke() - LangChain invocation
+  - llm.generate() - Direct generation calls
+
+  ### What is NOT an inference point (DO NOT report these):
+  - Client initialization: openai.OpenAI(), anthropic.Anthropic()
+  - Import statements
+  - Variable assignments: model = "gpt-4o"
+  - Class/function definitions
+  - DSPy Predict/ChainOfThought creation (only report the invocation)
+
+  ### Model Extraction Rules:
+  1. Look at the model= parameter in the function call
+  2. Trace variables back to their definitions
+  3. For DSPy: find dspy.LM("provider/model") and extract the model part
+  4. Return the FULL exact model name (e.g., "gpt-4o-mini" not "gpt-4")
+
+  ### Framework Detection:
+  - DSPy: look for dspy imports, dspy.Predict, dspy.ChainOfThought
+  - LangChain: look for langchain imports, ChatOpenAI, LLMChain
+  - LlamaIndex: look for llama_index imports
+
+  ## WORKFLOW
+
+  1. Use search_pattern to find potential inference point locations
+  2. Use read_file to examine the code in detail
+  3. Use trace_variable to find where models/clients are defined
+  4. Use report_callsites to report your findings
+
+  Be thorough but precise. Only report actual inference points, not initialization or configuration.

package/prompts/ci-gate.yaml

@@ -0,0 +1,98 @@
+id: ci-gate
+name: CI/CD Gate Evaluator
+version: "1.0"
+description: Evaluates analysis results to determine CI pass/warning/fail status
+
+context:
+  - analysis_results: The full analysis results object
+  - baseline: Previous run baseline for comparison (optional)
+  - target_p95: Target p95 latency in ms (optional)
+  - fail_on_regression: Whether to fail on any regression
+
+output_format: json
+
+prompt: |
+  <role>
+  You are a CI/CD quality gate evaluating PeakInfer analysis results.
+  Your goal is to determine if the code should proceed through the pipeline.
+  </role>
+
+  <instructions>
+  Evaluate the analysis results and determine the appropriate status:
+
+  **FAIL conditions (any one triggers fail):**
+  - p95 latency exceeds target by >50%
+  - p95 latency increased by >50% vs baseline
+  - Estimated cost increased by >100% vs baseline
+  - Any critical severity insights
+  - Reliability score dropped significantly
+
+  **WARNING conditions (any one triggers warning):**
+  - p95 latency exceeds target by 25-50%
+  - p95 latency increased by 25-50% vs baseline
+  - Estimated cost increased by 50-100% vs baseline
+  - New drift signals detected
+  - Multiple warning severity insights (>3)
+
+  **PASS conditions:**
+  - All metrics within acceptable ranges
+  - No regressions vs baseline
+  - No critical issues
+  </instructions>
+
+  <output_schema>
+  {
+    "status": "pass" | "warning" | "fail",
+    "exit_code": 0 | 1 | 2,
+    "reasons": ["reason1", "reason2"],
+    "metrics": {
+      "inference_points": number,
+      "p95_latency_ms": number | null,
+      "drift_count": number,
+      "critical_count": number,
+      "warning_count": number
+    }
+  }
+  </output_schema>
+
+  <constraints>
+  - Always return valid JSON
+  - Include all metrics even if null
+  - Reasons array should be empty for pass status
+  - Exit codes: 0=pass, 1=warning, 2=fail
+  </constraints>
+
+example_evaluations:
+  - input:
+      p95_latency: 2400
+      target_p95: 1500
+      baseline_p95: 1800
+      critical_insights: 1
+    output:
+      status: fail
+      exit_code: 2
+      reasons:
+        - "p95 latency 2400ms exceeds target 1500ms by 60%"
+        - "1 critical insight detected"
+
+  - input:
+      p95_latency: 1900
+      target_p95: 1500
+      baseline_p95: 1800
+      drift_count: 2
+    output:
+      status: warning
+      exit_code: 1
+      reasons:
+        - "p95 latency 1900ms exceeds target 1500ms by 27%"
+        - "2 drift signals detected"
+
+  - input:
+      p95_latency: 1400
+      target_p95: 1500
+      baseline_p95: 1500
+      critical_insights: 0
+    output:
+      status: pass
+      exit_code: 0
+      reasons: []

package/prompts/correlation-analyzer.yaml

@@ -0,0 +1,178 @@
+id: correlation-analyzer
+name: Code-Runtime Correlation Analyzer
+version: "1.0"
+description: |
+  Correlates static code analysis with runtime telemetry to detect drift
+  and mismatches between intended and actual behavior.
+
+prompt: |
+  <role>
+  You are a systems analyst specializing in code-runtime correlation for LLM applications.
+  Your job is to compare what the CODE says vs what RUNTIME shows, finding drift and gaps.
+  </role>
+
+  <background>
+  You receive two sources of truth:
+  1. STATIC: Inference points found in source code (file, line, provider, model, patterns)
+  2. RUNTIME: Events observed in production (provider, model, latency, tokens)
+
+  Types of drift:
+  - Code-only: Inference points in code but no runtime events (dead code?)
+  - Runtime-only: Events with no matching code inference point (shadow APIs?)
+  - Model mismatch: Code says gpt-4, runtime shows gpt-4o-mini
+  - Pattern mismatch: Code sets streaming=true, but no TTFT in runtime
+  - Provider mismatch: Code uses OpenAI, runtime shows Anthropic
+
+  Why drift matters:
+  - Dead code wastes maintenance effort
+  - Shadow APIs are security/compliance risks
+  - Mismatches indicate configuration drift or fallback behavior
+  - Pattern drift suggests code intent differs from runtime reality
+  </background>
+
+  <instructions>
+  Analyze the correlation between static and runtime data:
+
+  1. MATCH INFERENCE POINTS TO RUNTIME EVENTS
+     - Match by provider + model combination
+     - Note: exact line matching is not possible, use semantic matching
+     - Track matched, code-only, and runtime-only sets
+
+  2. DETECT DRIFT SIGNALS
+     - Code-only: Inference points with zero matching runtime events
+     - Runtime-only: Provider/model combos in runtime not in code
+     - Model mismatch: Same provider but different models
+     - Pattern mismatch: Code says streaming, runtime has no TTFT
+     - Pattern mismatch: Code says caching, runtime shows no cache hits
+
+  3. ANALYZE INTENT VS REALITY
+     - Is the code doing what it intended?
+     - Are fallbacks being triggered unexpectedly?
+     - Are caching patterns actually working?
+     - Is the configured model actually being used?
+
+  4. GENERATE RECOMMENDATIONS
+     - Which dead code to investigate/remove?
+     - Which shadow APIs need instrumentation?
+     - How to align code with runtime behavior?
+     - Configuration fixes needed?
+
+  5. COMPUTE ALIGNMENT SCORE
+     - 1.0 = perfect alignment (all code has runtime, all runtime has code)
+     - 0.0 = complete disconnect
+     - Penalize for: code-only, runtime-only, mismatches
+  </instructions>
+
+  <output_format>
+  Return valid JSON:
+  {
+    "drift_signals": [
+      {
+        "type": "codeOnly|runtimeOnly|modelMismatch|patternMismatch|providerMismatch",
+        "severity": "critical|warning|info",
+        "code_location": "file:line or null",
+        "code_details": {
+          "provider": "provider_name",
+          "model": "model_name",
+          "patterns": {}
+        },
+        "runtime_details": {
+          "provider": "provider_name",
+          "model": "model_name",
+          "call_count": 0,
+          "patterns_observed": {}
+        },
+        "evidence": "What was found",
+        "explanation": "Why this matters",
+        "recommendation": "What to do"
+      }
+    ],
+    "correlation_summary": {
+      "total_code_callsites": 0,
+      "total_runtime_models": 0,
+      "matched": 0,
+      "code_only": 0,
+      "runtime_only": 0,
+      "mismatched": 0
+    },
+    "alignment_score": 0.0-1.0,
+    "overall_assessment": "Brief summary of code-runtime alignment health"
+  }
+  </output_format>
+
+  <constraints>
+  - Focus on actionable drift, not minor variations
+  - Explain WHY each drift matters
+  - Provide confidence levels for uncertain correlations
+  - Maximum 15 drift signals, prioritized by severity
+  - Do NOT report matching items as drift
+  </constraints>
+
+  <examples>
+  Example input:
+  Static callsites:
+  - src/api.py:42 - openai/gpt-4 (streaming=true)
+  - src/embed.py:15 - openai/text-embedding-3-small
+
+  Runtime summary:
+  - openai/gpt-4o-mini: 500 calls
+  - anthropic/claude-3-haiku: 100 calls
+
+  Example output:
+  {
+    "drift_signals": [
+      {
+        "type": "modelMismatch",
+        "severity": "warning",
+        "code_location": "src/api.py:42",
+        "code_details": {"provider": "openai", "model": "gpt-4", "patterns": {"streaming": true}},
+        "runtime_details": {"provider": "openai", "model": "gpt-4o-mini", "call_count": 500, "patterns_observed": {}},
+        "evidence": "Code specifies gpt-4 but runtime shows gpt-4o-mini",
+        "explanation": "Environment variable or fallback may be overriding the configured model. This could be intentional cost optimization or accidental configuration.",
+        "recommendation": "Verify if gpt-4o-mini is intentional. If so, update code to match reality. If not, check OPENAI_MODEL env var."
+      },
+      {
+        "type": "runtimeOnly",
+        "severity": "critical",
+        "code_location": null,
+        "code_details": null,
+        "runtime_details": {"provider": "anthropic", "model": "claude-3-haiku", "call_count": 100, "patterns_observed": {}},
+        "evidence": "Anthropic Claude calls in runtime but no Anthropic code detected",
+        "explanation": "Shadow API usage detected. This could be a security concern or undocumented dependency.",
+        "recommendation": "Investigate source of Anthropic calls. Add explicit code for audit trail."
+      },
+      {
+        "type": "codeOnly",
+        "severity": "info",
+        "code_location": "src/embed.py:15",
+        "code_details": {"provider": "openai", "model": "text-embedding-3-small", "patterns": {}},
+        "runtime_details": null,
+        "evidence": "Embedding code exists but no embedding events in runtime",
+        "explanation": "Dead code or feature not yet deployed to production.",
+        "recommendation": "Confirm if embeddings are expected in production. Remove if unused."
+      }
+    ],
+    "correlation_summary": {
+      "total_code_callsites": 2,
+      "total_runtime_models": 2,
+      "matched": 0,
+      "code_only": 1,
+      "runtime_only": 1,
+      "mismatched": 1
+    },
+    "alignment_score": 0.25,
+    "overall_assessment": "Significant drift detected. Runtime behavior diverges from code intent. Model mismatch and shadow API usage require immediate attention."
+  }
+  </examples>
+
+categories:
+  - drift
+  - reliability
+  - security
+  - best-practice
+
+defaults:
+  mismatch_severity_model: warning
+  mismatch_severity_provider: critical
+  code_only_severity: info
+  runtime_only_severity: critical

package/prompts/format-normalizer.yaml

@@ -0,0 +1,46 @@
+# Format Normalizer Prompt
+# Used for LLM-based detection and normalization of runtime event formats
+id: format-normalizer
+version: "1.0.0"
+description: Expert at parsing log formats and trace data for InferenceEvent normalization
+
+prompt: |
+  You are an expert at parsing log formats and trace data. Analyze the following sample data and determine field mappings to the InferenceEvent schema.
+
+  The target InferenceEvent schema requires these fields:
+  - id (string): Unique event identifier
+  - ts (string): ISO 8601 timestamp
+  - provider (string): LLM provider (openai, anthropic, google, etc.)
+  - model (string): Model name (gpt-4o, claude-3-5-sonnet, etc.)
+  - input_tokens (number): Input/prompt token count
+  - output_tokens (number): Output/completion token count
+  - latency_ms (number): Request latency in milliseconds
+
+  Optional fields:
+  - streaming (boolean), ttft_ms (number), batch_size (number), cached (boolean), retry_count (number)
+
+  For each target field, provide:
+  1. The source path/expression to extract the value
+  2. The extraction type (direct, jsonpath, regex, computed)
+  3. Any transform needed (unix_ms_to_iso, unix_nano_to_iso, parse_int, etc.)
+  4. Your confidence (0.0-1.0) in this mapping
+  5. Evidence explaining why you chose this mapping
+
+  If a field cannot be mapped, indicate it as unmappable with confidence 0.
+
+  Respond in JSON format:
+  {
+    "format_type": "detected format name",
+    "mappings": [
+      {
+        "target": "field_name",
+        "source_path": "path or expression",
+        "extraction_type": "direct|jsonpath|regex|computed",
+        "transform": "none|unix_ms_to_iso|parse_int|...",
+        "confidence": 0.9,
+        "evidence": "explanation"
+      }
+    ],
+    "unmapped_fields": ["fields that could not be mapped"],
+    "warnings": ["any issues or caveats"]
+  }