@peakinfer/cli 1.0.133

package/templates/optimizations/tensorrt-llm-performance.yaml

@@ -0,0 +1,77 @@
+id: tensorrt-llm-performance
+name: TensorRT-LLM Maximum Performance
+description: Deploy models with TensorRT-LLM for maximum inference performance on NVIDIA GPUs
+category: runtime_optimization
+confidence: 0.88
+success_count: 789
+verified_environments: 38
+contributors:
+  - tensorrt_specialist
+  - nvidia_partner
+last_updated: "2025-01-03"
+
+environment_match:
+  gpu_vendor: nvidia
+  performance_requirement: maximum
+  model_support: transformer
+
+optimization:
+  technique: tensorrt_llm_deployment
+  expected_throughput_improvement: "3-5x"
+  expected_latency_improvement: "50-70%"
+  effort_estimate: "2-3 weeks"
+  risk_level: medium
+
+economics:
+  projected_improvement:
+    throughput_multiplier: 4
+    cost_per_token_reduction: 0.75
+  implementation_cost:
+    engineering_hours: 120
+    total_cost: 24000
+
+implementation:
+  prerequisites:
+    - requirement: "NVIDIA GPU (A100/H100 recommended)"
+    - requirement: "TensorRT-LLM installation"
+      validation_command: "python -c 'import tensorrt_llm'"
+    - requirement: "Sufficient disk space for compiled engines"
+  automated_steps:
+    - step_id: engine_build
+      name: TensorRT Engine Build
+      executable: true
+      commands:
+        - "python scripts/convert_to_trt.py --model ./model --dtype fp16"
+        - "python scripts/build_trt_engine.py --max-batch-size 64 --max-input-len 2048"
+      validation:
+        command: "python scripts/validate_engine.py"
+        success_criteria: "engine_valid AND accuracy > 0.99"
+    - step_id: deployment
+      name: TensorRT Deployment
+      executable: true
+      commands:
+        - "python scripts/deploy_trt_server.py --engine ./engine"
+        - "python scripts/configure_inflight_batching.py"
+      validation:
+        command: "python scripts/benchmark_trt.py"
+        success_criteria: "throughput > baseline * 3"
+        rollback_command: "python scripts/fallback_to_pytorch.py"
+
+monitoring:
+  key_metrics:
+    - metric: throughput_tokens_per_second
+      target: ">5000"
+      alert_threshold: "<3000"
+    - metric: gpu_memory_utilization
+      target: "70-85%"
+      alert_threshold: ">95%"
+  rollback_triggers:
+    - condition: "engine_error_rate > 0.01 for 5 minutes"
+      action: automatic_rollback
+
+results:
+  recent_implementations:
+    - environment: high_volume_api
+      baseline_throughput: 800
+      optimized_throughput: 3500
+      improvement_factor: 4.4

package/templates/optimizations/vllm-high-throughput-optimization.yaml

@@ -0,0 +1,93 @@
+id: vllm-high-throughput-optimization
+name: vLLM Continuous Batching for High-Volume Production
+description: Optimize vLLM deployment for maximum throughput in high-traffic scenarios
+category: batching_optimization
+confidence: 0.91
+success_count: 1923
+verified_environments: 67
+contributors:
+  - scaling_team
+  - vllm_expert
+  - production_engineer
+last_updated: "2025-01-14"
+
+environment_match:
+  runtime: vllm
+  monthly_requests: ">1M"
+  current_batch_size: "<8"
+  gpu_utilization: "<70%"
+  latency_requirements: flexible
+
+optimization:
+  technique: continuous_batching
+  expected_throughput_improvement: "3-5x"
+  expected_cost_reduction: "60-75%"
+  effort_estimate: "1-2 weeks"
+  risk_level: low
+
+economics:
+  baseline_calculation:
+    current_throughput_factor: 1.0
+  projected_improvement:
+    new_throughput_factor: 4.0
+    gpu_reduction_factor: 0.25
+  implementation_cost:
+    engineering_hours: 80
+    total_cost: 16000
+
+implementation:
+  prerequisites:
+    - requirement: "vLLM 0.2.7+"
+      validation_command: "python -c 'import vllm; print(vllm.__version__)'"
+    - requirement: "CUDA 11.8+"
+      validation_command: "nvcc --version | grep 'release 11.8'"
+    - requirement: "16GB+ GPU memory"
+      validation_command: "nvidia-smi --query-gpu=memory.total --format=csv,noheader | awk '{if($1<16000) exit 1}'"
+  automated_steps:
+    - step_id: batch_configuration
+      name: Optimal Batch Configuration
+      executable: true
+      commands:
+        - "python scripts/configure_vllm.py --max-num-batched-tokens 8192 --max-num-seqs 32"
+        - "python scripts/start_vllm_server.py --model meta-llama/Llama-2-7b-hf --gpu-memory-utilization 0.85"
+      validation:
+        command: "python scripts/test_batch_performance.py --target-batch-size 16"
+        success_criteria: "average_batch_size > 12"
+        rollback_command: "python scripts/revert_vllm_config.py"
+    - step_id: memory_optimization
+      name: Memory Optimization
+      executable: true
+      commands:
+        - "python scripts/enable_prefix_caching.py"
+        - "python scripts/configure_swap_space.py --swap-size 4GB"
+      validation:
+        command: "python scripts/check_memory_efficiency.py"
+        success_criteria: "memory_utilization > 0.8 AND memory_utilization < 0.9"
+        rollback_command: "python scripts/disable_optimizations.py"
+
+monitoring:
+  key_metrics:
+    - metric: average_batch_size
+      target: ">16"
+      alert_threshold: "<12"
+    - metric: throughput_tokens_per_second
+      target: ">3000"
+      alert_threshold: "<2000"
+    - metric: gpu_memory_utilization
+      target: "0.8-0.85"
+      alert_threshold: ">0.9"
+  rollback_triggers:
+    - condition: "average_batch_size < 8 for 20 minutes"
+      action: automatic_rollback
+    - condition: "gpu_memory_utilization > 0.95 for 10 minutes"
+      action: automatic_rollback
+    - condition: "throughput_degradation > 30% for 15 minutes"
+      action: alert_and_investigation
+
+results:
+  recent_implementations:
+    - environment: video_streaming_recommendations
+      baseline_throughput: 800
+      optimized_throughput: 3200
+      throughput_improvement: 4.0
+      implementation_days: 8

package/templates/optimizations/vllm-migration-memory-bound.yaml

@@ -0,0 +1,78 @@
+id: vllm-migration-memory-bound
+name: vLLM Migration from Memory-Bound Workloads
+description: Migrate from traditional serving to vLLM for memory-bound inference workloads
+category: runtime_optimization
+confidence: 0.90
+success_count: 1123
+verified_environments: 52
+contributors:
+  - vllm_specialist
+  - migration_engineer
+last_updated: "2025-01-10"
+
+environment_match:
+  current_runtime:
+    - huggingface
+    - pytorch
+  memory_bound: true
+  batch_size: "<4"
+
+optimization:
+  technique: vllm_migration
+  expected_throughput_improvement: "3-6x"
+  expected_cost_reduction: "60-80%"
+  effort_estimate: "1-2 weeks"
+  risk_level: low
+
+economics:
+  projected_improvement:
+    throughput_multiplier: 4.5
+    cost_reduction_percent: 70
+  implementation_cost:
+    engineering_hours: 60
+    total_cost: 12000
+
+implementation:
+  prerequisites:
+    - requirement: "vLLM compatible model"
+      validation_command: "python scripts/check_vllm_compatibility.py --model ./model"
+    - requirement: "GPU with 16GB+ memory"
+  automated_steps:
+    - step_id: compatibility_check
+      name: Compatibility Verification
+      executable: true
+      commands:
+        - "python scripts/verify_model_format.py"
+        - "python scripts/test_vllm_loading.py"
+      validation:
+        command: "python scripts/validate_loading.py"
+        success_criteria: "model_loads_successfully"
+    - step_id: migration
+      name: vLLM Migration
+      executable: true
+      commands:
+        - "python scripts/setup_vllm_server.py --model ./model --tensor-parallel-size 1"
+        - "python scripts/configure_batching.py --max-tokens 8192"
+      validation:
+        command: "python scripts/benchmark_vllm.py"
+        success_criteria: "throughput > baseline * 3"
+        rollback_command: "python scripts/revert_to_original.py"
+
+monitoring:
+  key_metrics:
+    - metric: throughput_rps
+      target: ">baseline * 3"
+      alert_threshold: "<baseline * 2"
+    - metric: latency_p99
+      target: "<baseline * 1.2"
+      alert_threshold: ">baseline * 2"
+  rollback_triggers:
+    - condition: "throughput < baseline for 15 minutes"
+      action: automatic_rollback
+
+results:
+  recent_implementations:
+    - environment: api_inference_service
+      baseline_throughput: 50
+      optimized_throughput: 220
+      improvement_factor: 4.4

package/templates/overpowered-extraction.yaml

@@ -0,0 +1,32 @@
+# Based on: https://www.kalmantic.com/posts/gpt5-model-selection-economics-extraction-tasks
+# "Why Premium Models Waste Money on Extraction Tasks"
+
+id: overpowered-extraction
+name: Overpowered Model for Simple Tasks
+version: "1.0"
+category: cost
+severity: warning
+layer: model
+
+match:
+  scope: callsite
+  conditions:
+    - field: model
+      op: in
+      value: ["gpt-4o", "gpt-4", "gpt-4-turbo", "claude-3-opus", "claude-3.5-sonnet"]
+    - field: avg_tokens
+      op: lt
+      value: 100
+
+output:
+  headline: "Using {{model}} for {{avg_tokens}}-token outputs"
+  evidence: "{{location}}: Consider gpt-4o-mini or claude-3-haiku for simple extraction tasks"
+
+defaults:
+  small_output_threshold: 100
+  premium_models:
+    - gpt-4o
+    - gpt-4
+    - gpt-4-turbo
+    - claude-3-opus
+    - claude-3.5-sonnet

package/templates/overpowered-model.yaml

@@ -0,0 +1,31 @@
+id: overpowered-model
+name: Overpowered Model Detection
+version: "1.0"
+category: waste
+severity: info
+layer: model
+
+source:
+  url: https://openai.com/pricing
+  title: "Model Pricing and Capability Tiers"
+
+match:
+  scope: callsite
+  conditions:
+    - field: model
+      op: in
+      value: ["gpt-4o", "gpt-4", "gpt-4-turbo", "claude-3-opus", "claude-3-opus-20240229"]
+    - field: usage.avg_output_tokens
+      op: lt
+      value: 100
+    - field: usage.calls
+      op: gt
+      value: 100
+
+output:
+  headline: "{{model}} used for short outputs (avg {{avg_tokens}} tokens)"
+  evidence: "Premium models have minimum cost overhead regardless of output length"
+
+defaults:
+  output_threshold: 100
+  calls_threshold: 100

package/templates/prompt-bloat.yaml

@@ -0,0 +1,24 @@
+# Based on: https://www.kalmantic.com/posts/system-prompt-optimization-stop-paying-redundant-instructions
+# "Stop Paying 40x More for Redundant AI Instructions"
+
+id: prompt-bloat
+name: Prompt Bloat Detection
+version: "1.0"
+category: cost
+severity: warning
+layer: model
+
+match:
+  scope: callsite
+  conditions:
+    - field: usage.tokens_in
+      op: ratio_gt
+      compare_to: usage.tokens_out
+      value: 20
+
+output:
+  headline: "{{ratio}}x more input than output tokens"
+  evidence: "{{location}}: {{tokens_in}} tokens in → {{tokens_out}} tokens out. Consider prompt optimization."
+
+defaults:
+  input_output_ratio_threshold: 20

package/templates/retry-explosion.yaml

@@ -0,0 +1,28 @@
+# Based on: https://www.kalmantic.com/posts/ai-retry-logic-error-handling-multiplies-costs
+# "How Bad Error Handling Turns $10 Failures into $1000 Bills"
+
+id: retry-explosion
+name: Retry Storm Detection
+version: "1.0"
+category: cost
+severity: critical
+layer: api
+
+match:
+  scope: callsite
+  conditions:
+    - field: usage.calls
+      op: gt
+      value: 10
+    - field: usage.latency_p99
+      op: ratio_gt
+      compare_to: usage.latency_p50
+      value: 5
+
+output:
+  headline: "Possible retry storm at {{location}}"
+  evidence: "{{calls}} calls with p99/p50 ratio of {{ratio}}x - check retry logic"
+
+defaults:
+  min_calls: 10
+  latency_ratio_threshold: 5

package/templates/schema/insight.schema.json

@@ -0,0 +1,113 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/Kalmantic/peakinfer_templates/schema/insight.schema.json",
+  "title": "PeakInfer Insight Template",
+  "description": "Schema for insight detection templates",
+  "type": "object",
+  "required": ["id", "version", "name", "category", "severity", "match", "output"],
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z][a-z0-9-]*$",
+      "description": "Unique identifier (kebab-case)"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+$",
+      "description": "Template version (semver major.minor)"
+    },
+    "name": {
+      "type": "string",
+      "description": "Human-readable name"
+    },
+    "description": {
+      "type": "string",
+      "description": "Detailed description of what this insight detects"
+    },
+    "source": {
+      "type": "object",
+      "properties": {
+        "url": { "type": "string", "format": "uri" },
+        "title": { "type": "string" }
+      },
+      "description": "Attribution to blog post, research, etc."
+    },
+    "category": {
+      "type": "string",
+      "enum": ["cost", "drift", "performance", "waste", "reliability", "latency", "throughput"],
+      "description": "Primary category for grouping"
+    },
+    "severity": {
+      "type": "string",
+      "enum": ["critical", "warning", "info"],
+      "description": "Impact severity"
+    },
+    "tags": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Searchable tags"
+    },
+    "match": {
+      "type": "object",
+      "required": ["scope", "conditions"],
+      "properties": {
+        "scope": {
+          "type": "string",
+          "enum": ["callsite", "joined", "global", "envelope"],
+          "description": "What data context to evaluate against"
+        },
+        "conditions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["field", "op"],
+            "properties": {
+              "field": { "type": "string" },
+              "op": {
+                "type": "string",
+                "enum": ["eq", "neq", "gt", "lt", "gte", "lte", "exists", "in", "ratio_gt", "ratio_lt", "has_pattern"]
+              },
+              "value": {},
+              "compare_to": { "type": "string" },
+              "pattern": { "type": "string" },
+              "count_gt": { "type": "number" }
+            }
+          }
+        }
+      }
+    },
+    "output": {
+      "type": "object",
+      "required": ["headline", "evidence"],
+      "properties": {
+        "headline": {
+          "type": "string",
+          "description": "Short summary with {{variables}}"
+        },
+        "evidence": {
+          "type": "string",
+          "description": "Supporting details with {{variables}}"
+        }
+      }
+    },
+    "recommends": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "optimization": { "type": "string" },
+          "relevance": { "type": "number", "minimum": 0, "maximum": 1 },
+          "reason": { "type": "string" }
+        }
+      },
+      "description": "Links to optimization templates"
+    },
+    "defaults": {
+      "type": "object",
+      "description": "Default threshold values"
+    },
+    "author": { "type": "string" },
+    "created": { "type": "string", "format": "date" },
+    "updated": { "type": "string", "format": "date" }
+  }
+}

package/templates/schema/optimization.schema.json

@@ -0,0 +1,180 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/Kalmantic/peakinfer_templates/schema/optimization.schema.json",
+  "title": "PeakInfer Optimization Template",
+  "description": "Schema for optimization recommendation templates",
+  "type": "object",
+  "required": ["id", "name", "description", "category", "optimization", "implementation"],
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z][a-z0-9-]*$",
+      "description": "Unique identifier (kebab-case)"
+    },
+    "name": {
+      "type": "string",
+      "description": "Human-readable name"
+    },
+    "description": {
+      "type": "string",
+      "description": "Detailed description of the optimization"
+    },
+    "source": {
+      "type": "object",
+      "properties": {
+        "url": { "type": "string", "format": "uri" },
+        "title": { "type": "string" },
+        "authors": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      },
+      "description": "Attribution to research paper, blog post, etc."
+    },
+    "category": {
+      "type": "string",
+      "enum": [
+        "api_optimization",
+        "memory_optimization",
+        "latency_optimization",
+        "cost_optimization",
+        "reliability_optimization",
+        "throughput_optimization",
+        "serving_optimization"
+      ],
+      "description": "Primary optimization category"
+    },
+    "confidence": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "description": "Confidence score based on verified implementations"
+    },
+    "success_count": {
+      "type": "integer",
+      "minimum": 0,
+      "description": "Number of successful implementations"
+    },
+    "verified_environments": {
+      "type": "integer",
+      "minimum": 0,
+      "description": "Number of verified deployment environments"
+    },
+    "contributors": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Contributors to this optimization"
+    },
+    "last_updated": {
+      "type": "string",
+      "format": "date",
+      "description": "Last update date"
+    },
+    "environment_match": {
+      "type": "object",
+      "description": "Conditions for when this optimization applies",
+      "properties": {
+        "model_size": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "memory_pressure": { "type": "string" },
+        "quality_tolerance": { "type": "string" },
+        "deployment": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "optimization": {
+      "type": "object",
+      "required": ["technique"],
+      "properties": {
+        "technique": { "type": "string" },
+        "expected_memory_reduction": { "type": "string" },
+        "expected_quality_retention": { "type": "string" },
+        "expected_latency_improvement": { "type": "string" },
+        "expected_cost_reduction": { "type": "string" },
+        "effort_estimate": { "type": "string" },
+        "risk_level": {
+          "type": "string",
+          "enum": ["low", "medium", "high"]
+        }
+      }
+    },
+    "economics": {
+      "type": "object",
+      "properties": {
+        "baseline_calculation": { "type": "object" },
+        "projected_improvement": { "type": "object" },
+        "implementation_cost": { "type": "object" }
+      }
+    },
+    "implementation": {
+      "type": "object",
+      "properties": {
+        "prerequisites": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "requirement": { "type": "string" },
+              "validation_command": { "type": "string" }
+            }
+          }
+        },
+        "automated_steps": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "step_id": { "type": "string" },
+              "name": { "type": "string" },
+              "executable": { "type": "boolean" },
+              "commands": {
+                "type": "array",
+                "items": { "type": "string" }
+              },
+              "validation": { "type": "object" }
+            }
+          }
+        }
+      }
+    },
+    "monitoring": {
+      "type": "object",
+      "properties": {
+        "key_metrics": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "metric": { "type": "string" },
+              "target": { "type": "string" },
+              "alert_threshold": { "type": "string" }
+            }
+          }
+        },
+        "rollback_triggers": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "condition": { "type": "string" },
+              "action": { "type": "string" }
+            }
+          }
+        }
+      }
+    },
+    "results": {
+      "type": "object",
+      "properties": {
+        "recent_implementations": {
+          "type": "array",
+          "items": { "type": "object" }
+        }
+      }
+    }
+  }
+}

package/templates/streaming-drift.yaml

@@ -0,0 +1,30 @@
+id: streaming-drift
+name: Streaming Drift Detection
+version: "1.0"
+category: latency
+severity: critical
+layer: application
+
+source:
+  url: https://anthropic.com/research/streaming-tokens
+  title: "Token Streaming for Real-Time Applications"
+
+match:
+  scope: callsite
+  conditions:
+    - field: patterns.streaming
+      op: eq
+      value: true
+    - field: usage
+      op: exists
+    - field: usage.latency_p99
+      op: ratio_gt
+      compare_to: usage.latency_p50
+      value: 5
+
+output:
+  headline: "Streaming enabled but responses arrive in bursts"
+  evidence: "p99/p50 ratio is {{ratio}}x — true streaming would be under 2x"
+
+defaults:
+  threshold: 2