@peakinfer/cli 1.0.133

package/dist/envelopes.js

@@ -0,0 +1,790 @@
+// =============================================================================
+// INFERENCE MAX REFERENCE ENVELOPES
+// =============================================================================
+//
+// Performance benchmarks from InferenceMax testing.
+// These represent achievable performance under optimal conditions.
+// Key: model name or model:runtime combination
+//
+// TTFT = Time To First Token
+// TPS = Tokens Per Second (output generation speed)
+// =============================================================================
+export const ENVELOPES = {
+    // ==========================================================================
+    // OpenAI Models
+    // ==========================================================================
+    'gpt-4o': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 500,
+        tps_median: 80,
+        tps_peak: 120,
+    },
+    'gpt-4o-2024-11-20': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 500,
+        tps_median: 80,
+        tps_peak: 120,
+    },
+    'gpt-4o-mini': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 350,
+        tps_median: 100,
+        tps_peak: 150,
+    },
+    'gpt-4o-mini-2024-07-18': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 350,
+        tps_median: 100,
+        tps_peak: 150,
+    },
+    'gpt-4-turbo': {
+        ttft_p50_ms: 300,
+        ttft_p95_ms: 800,
+        tps_median: 60,
+        tps_peak: 90,
+    },
+    'gpt-4-turbo-2024-04-09': {
+        ttft_p50_ms: 300,
+        ttft_p95_ms: 800,
+        tps_median: 60,
+        tps_peak: 90,
+    },
+    'gpt-4.1': {
+        ttft_p50_ms: 180,
+        ttft_p95_ms: 450,
+        tps_median: 90,
+        tps_peak: 130,
+    },
+    'gpt-4.1-2025-04-14': {
+        ttft_p50_ms: 180,
+        ttft_p95_ms: 450,
+        tps_median: 90,
+        tps_peak: 130,
+    },
+    'gpt-4.1-mini': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 300,
+        tps_median: 110,
+        tps_peak: 160,
+    },
+    'gpt-4.1-nano': {
+        ttft_p50_ms: 80,
+        ttft_p95_ms: 200,
+        tps_median: 140,
+        tps_peak: 200,
+    },
+    'gpt-3.5-turbo': {
+        ttft_p50_ms: 100,
+        ttft_p95_ms: 250,
+        tps_median: 120,
+        tps_peak: 180,
+    },
+    'o1': {
+        ttft_p50_ms: 2000,
+        ttft_p95_ms: 8000,
+        tps_median: 30,
+        tps_peak: 50,
+    },
+    'o1-preview': {
+        ttft_p50_ms: 2000,
+        ttft_p95_ms: 8000,
+        tps_median: 30,
+        tps_peak: 50,
+    },
+    'o1-mini': {
+        ttft_p50_ms: 800,
+        ttft_p95_ms: 3000,
+        tps_median: 50,
+        tps_peak: 80,
+    },
+    'o3': {
+        ttft_p50_ms: 1500,
+        ttft_p95_ms: 6000,
+        tps_median: 40,
+        tps_peak: 60,
+    },
+    'o3-mini': {
+        ttft_p50_ms: 600,
+        ttft_p95_ms: 2000,
+        tps_median: 60,
+        tps_peak: 90,
+    },
+    // ==========================================================================
+    // Anthropic Models
+    // ==========================================================================
+    'claude-3-opus': {
+        ttft_p50_ms: 400,
+        ttft_p95_ms: 1200,
+        tps_median: 40,
+        tps_peak: 60,
+    },
+    'claude-3-opus-20240229': {
+        ttft_p50_ms: 400,
+        ttft_p95_ms: 1200,
+        tps_median: 40,
+        tps_peak: 60,
+    },
+    'claude-3-sonnet': {
+        ttft_p50_ms: 250,
+        ttft_p95_ms: 600,
+        tps_median: 70,
+        tps_peak: 100,
+    },
+    'claude-3-sonnet-20240229': {
+        ttft_p50_ms: 250,
+        ttft_p95_ms: 600,
+        tps_median: 70,
+        tps_peak: 100,
+    },
+    'claude-3-haiku': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 350,
+        tps_median: 100,
+        tps_peak: 150,
+    },
+    'claude-3-haiku-20240307': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 350,
+        tps_median: 100,
+        tps_peak: 150,
+    },
+    'claude-3-5-sonnet': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 500,
+        tps_median: 80,
+        tps_peak: 120,
+    },
+    'claude-3-5-sonnet-20241022': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 500,
+        tps_median: 80,
+        tps_peak: 120,
+    },
+    'claude-3-5-haiku': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 300,
+        tps_median: 110,
+        tps_peak: 160,
+    },
+    'claude-3-5-haiku-20241022': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 300,
+        tps_median: 110,
+        tps_peak: 160,
+    },
+    // Claude 4 models
+    'claude-sonnet-4-20250514': {
+        ttft_p50_ms: 180,
+        ttft_p95_ms: 450,
+        tps_median: 90,
+        tps_peak: 130,
+    },
+    'claude-4-sonnet': {
+        ttft_p50_ms: 180,
+        ttft_p95_ms: 450,
+        tps_median: 90,
+        tps_peak: 130,
+    },
+    'claude-opus-4-20250514': {
+        ttft_p50_ms: 350,
+        ttft_p95_ms: 1000,
+        tps_median: 50,
+        tps_peak: 75,
+    },
+    'claude-4-opus': {
+        ttft_p50_ms: 350,
+        ttft_p95_ms: 1000,
+        tps_median: 50,
+        tps_peak: 75,
+    },
+    // ==========================================================================
+    // Google Models
+    // ==========================================================================
+    'gemini-pro': {
+        ttft_p50_ms: 250,
+        ttft_p95_ms: 600,
+        tps_median: 60,
+        tps_peak: 100,
+    },
+    'gemini-1.5-pro': {
+        ttft_p50_ms: 300,
+        ttft_p95_ms: 800,
+        tps_median: 50,
+        tps_peak: 80,
+    },
+    'gemini-1.5-pro-latest': {
+        ttft_p50_ms: 300,
+        ttft_p95_ms: 800,
+        tps_median: 50,
+        tps_peak: 80,
+    },
+    'gemini-1.5-flash': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 400,
+        tps_median: 100,
+        tps_peak: 150,
+    },
+    'gemini-1.5-flash-latest': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 400,
+        tps_median: 100,
+        tps_peak: 150,
+    },
+    'gemini-2.0-flash': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 300,
+        tps_median: 120,
+        tps_peak: 180,
+    },
+    'gemini-2.0-flash-exp': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 300,
+        tps_median: 120,
+        tps_peak: 180,
+    },
+    'gemini-2.5-pro': {
+        ttft_p50_ms: 250,
+        ttft_p95_ms: 700,
+        tps_median: 60,
+        tps_peak: 90,
+    },
+    'gemini-2.5-flash': {
+        ttft_p50_ms: 100,
+        ttft_p95_ms: 250,
+        tps_median: 130,
+        tps_peak: 200,
+    },
+    // ==========================================================================
+    // Mistral Models
+    // ==========================================================================
+    'mistral-large': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 500,
+        tps_median: 70,
+        tps_peak: 110,
+    },
+    'mistral-large-latest': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 500,
+        tps_median: 70,
+        tps_peak: 110,
+    },
+    'mistral-medium': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 350,
+        tps_median: 90,
+        tps_peak: 140,
+    },
+    'mistral-small': {
+        ttft_p50_ms: 100,
+        ttft_p95_ms: 250,
+        tps_median: 110,
+        tps_peak: 160,
+    },
+    'mistral-small-latest': {
+        ttft_p50_ms: 100,
+        ttft_p95_ms: 250,
+        tps_median: 110,
+        tps_peak: 160,
+    },
+    'codestral': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 300,
+        tps_median: 100,
+        tps_peak: 150,
+    },
+    'codestral-latest': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 300,
+        tps_median: 100,
+        tps_peak: 150,
+    },
+    // ==========================================================================
+    // Together AI / Fireworks / Groq (Cloud-hosted open models)
+    // ==========================================================================
+    'mixtral-8x7b': {
+        ttft_p50_ms: 100,
+        ttft_p95_ms: 250,
+        tps_median: 60,
+        tps_peak: 100,
+    },
+    'mixtral-8x7b-instruct': {
+        ttft_p50_ms: 100,
+        ttft_p95_ms: 250,
+        tps_median: 60,
+        tps_peak: 100,
+    },
+    'mixtral-8x22b': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 400,
+        tps_median: 40,
+        tps_peak: 70,
+    },
+    'mixtral-8x22b-instruct': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 400,
+        tps_median: 40,
+        tps_peak: 70,
+    },
+    // ==========================================================================
+    // Llama 3 Models (Meta)
+    // ==========================================================================
+    'llama-3-70b': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 600,
+        tps_median: 35,
+        tps_peak: 60,
+    },
+    'llama-3-70b:vllm': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 400,
+        tps_median: 45,
+        tps_peak: 90,
+    },
+    'llama-3-70b:sglang': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 350,
+        tps_median: 55,
+        tps_peak: 100,
+    },
+    'llama-3-70b:tgi': {
+        ttft_p50_ms: 180,
+        ttft_p95_ms: 500,
+        tps_median: 35,
+        tps_peak: 70,
+    },
+    'meta-llama/Llama-3-70b-chat-hf': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 400,
+        tps_median: 45,
+        tps_peak: 90,
+    },
+    'llama-3-8b': {
+        ttft_p50_ms: 80,
+        ttft_p95_ms: 200,
+        tps_median: 80,
+        tps_peak: 150,
+    },
+    'llama-3-8b:vllm': {
+        ttft_p50_ms: 60,
+        ttft_p95_ms: 150,
+        tps_median: 100,
+        tps_peak: 180,
+    },
+    'meta-llama/Llama-3-8b-chat-hf': {
+        ttft_p50_ms: 60,
+        ttft_p95_ms: 150,
+        tps_median: 100,
+        tps_peak: 180,
+    },
+    // ==========================================================================
+    // Llama 3.1 Models
+    // ==========================================================================
+    'llama-3.1-405b': {
+        ttft_p50_ms: 400,
+        ttft_p95_ms: 1200,
+        tps_median: 20,
+        tps_peak: 35,
+    },
+    'llama-3.1-405b:vllm': {
+        ttft_p50_ms: 300,
+        ttft_p95_ms: 900,
+        tps_median: 25,
+        tps_peak: 45,
+    },
+    'llama-3.1-405b:sglang': {
+        ttft_p50_ms: 250,
+        ttft_p95_ms: 800,
+        tps_median: 30,
+        tps_peak: 50,
+    },
+    'meta-llama/Llama-3.1-405B-Instruct': {
+        ttft_p50_ms: 300,
+        ttft_p95_ms: 900,
+        tps_median: 25,
+        tps_peak: 45,
+    },
+    'llama-3.1-70b': {
+        ttft_p50_ms: 180,
+        ttft_p95_ms: 500,
+        tps_median: 40,
+        tps_peak: 70,
+    },
+    'llama-3.1-70b:vllm': {
+        ttft_p50_ms: 140,
+        ttft_p95_ms: 380,
+        tps_median: 50,
+        tps_peak: 95,
+    },
+    'llama-3.1-70b:sglang': {
+        ttft_p50_ms: 110,
+        ttft_p95_ms: 320,
+        tps_median: 60,
+        tps_peak: 110,
+    },
+    'meta-llama/Llama-3.1-70B-Instruct': {
+        ttft_p50_ms: 140,
+        ttft_p95_ms: 380,
+        tps_median: 50,
+        tps_peak: 95,
+    },
+    'llama-3.1-8b': {
+        ttft_p50_ms: 70,
+        ttft_p95_ms: 180,
+        tps_median: 90,
+        tps_peak: 160,
+    },
+    'llama-3.1-8b:vllm': {
+        ttft_p50_ms: 50,
+        ttft_p95_ms: 130,
+        tps_median: 110,
+        tps_peak: 200,
+    },
+    'meta-llama/Llama-3.1-8B-Instruct': {
+        ttft_p50_ms: 50,
+        ttft_p95_ms: 130,
+        tps_median: 110,
+        tps_peak: 200,
+    },
+    // ==========================================================================
+    // Llama 3.2 Models (Smaller/Edge)
+    // ==========================================================================
+    'llama-3.2-90b': {
+        ttft_p50_ms: 250,
+        ttft_p95_ms: 700,
+        tps_median: 30,
+        tps_peak: 55,
+    },
+    'llama-3.2-90b:vllm': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 550,
+        tps_median: 38,
+        tps_peak: 70,
+    },
+    'meta-llama/Llama-3.2-90B-Vision-Instruct': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 550,
+        tps_median: 38,
+        tps_peak: 70,
+    },
+    'llama-3.2-11b': {
+        ttft_p50_ms: 80,
+        ttft_p95_ms: 200,
+        tps_median: 75,
+        tps_peak: 140,
+    },
+    'llama-3.2-11b:vllm': {
+        ttft_p50_ms: 60,
+        ttft_p95_ms: 150,
+        tps_median: 95,
+        tps_peak: 170,
+    },
+    'meta-llama/Llama-3.2-11B-Vision-Instruct': {
+        ttft_p50_ms: 60,
+        ttft_p95_ms: 150,
+        tps_median: 95,
+        tps_peak: 170,
+    },
+    'llama-3.2-3b': {
+        ttft_p50_ms: 40,
+        ttft_p95_ms: 100,
+        tps_median: 140,
+        tps_peak: 250,
+    },
+    'llama-3.2-3b:vllm': {
+        ttft_p50_ms: 30,
+        ttft_p95_ms: 80,
+        tps_median: 170,
+        tps_peak: 300,
+    },
+    'llama-3.2-3b:ollama': {
+        ttft_p50_ms: 50,
+        ttft_p95_ms: 120,
+        tps_median: 120,
+        tps_peak: 200,
+    },
+    'meta-llama/Llama-3.2-3B-Instruct': {
+        ttft_p50_ms: 30,
+        ttft_p95_ms: 80,
+        tps_median: 170,
+        tps_peak: 300,
+    },
+    'llama-3.2-1b': {
+        ttft_p50_ms: 25,
+        ttft_p95_ms: 60,
+        tps_median: 200,
+        tps_peak: 350,
+    },
+    'llama-3.2-1b:ollama': {
+        ttft_p50_ms: 30,
+        ttft_p95_ms: 80,
+        tps_median: 180,
+        tps_peak: 300,
+    },
+    'meta-llama/Llama-3.2-1B-Instruct': {
+        ttft_p50_ms: 25,
+        ttft_p95_ms: 60,
+        tps_median: 200,
+        tps_peak: 350,
+    },
+    // ==========================================================================
+    // Llama 3.3 Models
+    // ==========================================================================
+    'llama-3.3-70b': {
+        ttft_p50_ms: 160,
+        ttft_p95_ms: 450,
+        tps_median: 45,
+        tps_peak: 80,
+    },
+    'llama-3.3-70b:vllm': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 350,
+        tps_median: 55,
+        tps_peak: 100,
+    },
+    'llama-3.3-70b:sglang': {
+        ttft_p50_ms: 100,
+        ttft_p95_ms: 300,
+        tps_median: 65,
+        tps_peak: 115,
+    },
+    'meta-llama/Llama-3.3-70B-Instruct': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 350,
+        tps_median: 55,
+        tps_peak: 100,
+    },
+    // ==========================================================================
+    // Qwen Models (Alibaba)
+    // ==========================================================================
+    'qwen-2.5-72b': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 550,
+        tps_median: 40,
+        tps_peak: 70,
+    },
+    'qwen-2.5-72b:vllm': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 420,
+        tps_median: 50,
+        tps_peak: 90,
+    },
+    'qwen-2.5-72b:sglang': {
+        ttft_p50_ms: 130,
+        ttft_p95_ms: 380,
+        tps_median: 55,
+        tps_peak: 100,
+    },
+    'Qwen/Qwen2.5-72B-Instruct': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 420,
+        tps_median: 50,
+        tps_peak: 90,
+    },
+    'qwen-2.5-32b': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 320,
+        tps_median: 60,
+        tps_peak: 100,
+    },
+    'qwen-2.5-32b:vllm': {
+        ttft_p50_ms: 90,
+        ttft_p95_ms: 250,
+        tps_median: 75,
+        tps_peak: 130,
+    },
+    'Qwen/Qwen2.5-32B-Instruct': {
+        ttft_p50_ms: 90,
+        ttft_p95_ms: 250,
+        tps_median: 75,
+        tps_peak: 130,
+    },
+    'qwen-2.5-14b': {
+        ttft_p50_ms: 80,
+        ttft_p95_ms: 220,
+        tps_median: 80,
+        tps_peak: 140,
+    },
+    'qwen-2.5-7b': {
+        ttft_p50_ms: 50,
+        ttft_p95_ms: 140,
+        tps_median: 110,
+        tps_peak: 190,
+    },
+    'qwen-2.5-3b': {
+        ttft_p50_ms: 35,
+        ttft_p95_ms: 90,
+        tps_median: 150,
+        tps_peak: 260,
+    },
+    'qwen-2.5-coder-32b': {
+        ttft_p50_ms: 120,
+        ttft_p95_ms: 320,
+        tps_median: 60,
+        tps_peak: 100,
+    },
+    'qwen-2.5-coder-7b': {
+        ttft_p50_ms: 50,
+        ttft_p95_ms: 140,
+        tps_median: 110,
+        tps_peak: 190,
+    },
+    // ==========================================================================
+    // DeepSeek Models
+    // ==========================================================================
+    'deepseek-v3': {
+        ttft_p50_ms: 300,
+        ttft_p95_ms: 800,
+        tps_median: 35,
+        tps_peak: 60,
+    },
+    'deepseek-v3:vllm': {
+        ttft_p50_ms: 220,
+        ttft_p95_ms: 600,
+        tps_median: 45,
+        tps_peak: 80,
+    },
+    'deepseek-chat': {
+        ttft_p50_ms: 300,
+        ttft_p95_ms: 800,
+        tps_median: 35,
+        tps_peak: 60,
+    },
+    'deepseek-coder': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 550,
+        tps_median: 50,
+        tps_peak: 85,
+    },
+    'deepseek-coder-v2': {
+        ttft_p50_ms: 180,
+        ttft_p95_ms: 500,
+        tps_median: 55,
+        tps_peak: 95,
+    },
+    'deepseek-r1': {
+        ttft_p50_ms: 1500,
+        ttft_p95_ms: 5000,
+        tps_median: 25,
+        tps_peak: 45,
+    },
+    'deepseek-r1-lite': {
+        ttft_p50_ms: 800,
+        ttft_p95_ms: 2500,
+        tps_median: 40,
+        tps_peak: 70,
+    },
+    // ==========================================================================
+    // Cohere Models
+    // ==========================================================================
+    'command-r': {
+        ttft_p50_ms: 200,
+        ttft_p95_ms: 500,
+        tps_median: 60,
+        tps_peak: 100,
+    },
+    'command-r-plus': {
+        ttft_p50_ms: 300,
+        ttft_p95_ms: 800,
+        tps_median: 45,
+        tps_peak: 75,
+    },
+    'command-light': {
+        ttft_p50_ms: 100,
+        ttft_p95_ms: 250,
+        tps_median: 100,
+        tps_peak: 150,
+    },
+    // ==========================================================================
+    // Local/Edge Models (Ollama, llama.cpp)
+    // ==========================================================================
+    'phi-3': {
+        ttft_p50_ms: 50,
+        ttft_p95_ms: 130,
+        tps_median: 120,
+        tps_peak: 200,
+    },
+    'phi-3:ollama': {
+        ttft_p50_ms: 60,
+        ttft_p95_ms: 150,
+        tps_median: 100,
+        tps_peak: 170,
+    },
+    'phi-3-mini': {
+        ttft_p50_ms: 30,
+        ttft_p95_ms: 80,
+        tps_median: 160,
+        tps_peak: 280,
+    },
+    'gemma-2-9b': {
+        ttft_p50_ms: 70,
+        ttft_p95_ms: 180,
+        tps_median: 85,
+        tps_peak: 150,
+    },
+    'gemma-2-9b:ollama': {
+        ttft_p50_ms: 80,
+        ttft_p95_ms: 200,
+        tps_median: 75,
+        tps_peak: 130,
+    },
+    'gemma-2-27b': {
+        ttft_p50_ms: 150,
+        ttft_p95_ms: 400,
+        tps_median: 45,
+        tps_peak: 80,
+    },
+};
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+/**
+ * Get performance envelope for a model.
+ *
+ * @param model - Model name (e.g., "gpt-4o", "llama-3-70b")
+ * @param runtime - Optional runtime (e.g., "vllm", "sglang")
+ * @returns Performance envelope or null if not found
+ */
+export function getEnvelope(model, runtime) {
+    // Try with runtime suffix first
+    if (runtime) {
+        const keyWithRuntime = `${model}:${runtime}`;
+        if (ENVELOPES[keyWithRuntime]) {
+            return ENVELOPES[keyWithRuntime];
+        }
+    }
+    // Try exact match
+    if (ENVELOPES[model]) {
+        return ENVELOPES[model];
+    }
+    // Try case-insensitive match
+    const lowerModel = model.toLowerCase();
+    for (const [key, envelope] of Object.entries(ENVELOPES)) {
+        if (key.toLowerCase() === lowerModel) {
+            return envelope;
+        }
+    }
+    // Try partial match (model name contains)
+    for (const [key, envelope] of Object.entries(ENVELOPES)) {
+        const lowerKey = key.toLowerCase();
+        if (lowerKey.includes(lowerModel) || lowerModel.includes(lowerKey.split(':')[0])) {
+            return envelope;
+        }
+    }
+    return null;
+}
+/**
+ * Calculate throughput percentage against reference envelope.
+ *
+ * @param model - Model name
+ * @param actualTps - Actual tokens per second observed
+ * @param runtime - Optional runtime
+ * @returns Percentage (0-100+) or null if no envelope found
+ */
+export function getThroughputPercent(model, actualTps, runtime) {
+    const envelope = getEnvelope(model, runtime);
+    if (!envelope)
+        return null;
+    return Math.round((actualTps / envelope.tps_median) * 100);
+}
+//# sourceMappingURL=envelopes.js.map