@tokcalc/mcp-server 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/README.md +86 -406
  2. package/dist/index.js +21285 -0
  3. package/package.json +32 -90
  4. package/.zscripts/build.sh +0 -175
  5. package/.zscripts/database-runtime-build.sh +0 -33
  6. package/.zscripts/dev.pid +0 -1
  7. package/.zscripts/dev.sh +0 -154
  8. package/.zscripts/mini-services-build.sh +0 -78
  9. package/.zscripts/mini-services-install.sh +0 -65
  10. package/.zscripts/mini-services-start.sh +0 -123
  11. package/.zscripts/python-runtime-build.sh +0 -120
  12. package/.zscripts/start.sh +0 -145
  13. package/CAPACITY_STUDY.md +0 -283
  14. package/CODE_OF_CONDUCT.md +0 -55
  15. package/CONTRIBUTING.md +0 -177
  16. package/Caddyfile +0 -23
  17. package/LICENSE +0 -204
  18. package/bun.lock +0 -1965
  19. package/components.json +0 -21
  20. package/db/custom.db +0 -0
  21. package/download/README.md +0 -1
  22. package/download/tokcalc-dark-calculator.png +0 -0
  23. package/download/tokcalc-dark-default.png +0 -0
  24. package/download/tokcalc-demo.webm +0 -0
  25. package/download/tokcalc-github-link.png +0 -0
  26. package/download/tokcalc-hydration-fixed.png +0 -0
  27. package/download/tokcalc-issue-resolved.png +0 -0
  28. package/download/tokcalc-light-mode.png +0 -0
  29. package/download/tokcalc-light-reference.png +0 -0
  30. package/download/tokcalc-long-context-qwen.png +0 -0
  31. package/download/tokcalc-long-context.png +0 -0
  32. package/download/tokcalc-og-image-preview.png +0 -0
  33. package/download/tokcalc-phase2-3.png +0 -0
  34. package/download/tokcalc-plain-english.png +0 -0
  35. package/download/tokcalc-preview.png +0 -0
  36. package/download/tokcalc-share-bvb.png +0 -0
  37. package/download/tokcalc-share-feature.png +0 -0
  38. package/download/tokcalc-tab-build-vs-buy.png +0 -0
  39. package/download/tokcalc-tab-calculator.png +0 -0
  40. package/download/tokcalc-tab-reference.png +0 -0
  41. package/eslint.config.mjs +0 -50
  42. package/examples/websocket/frontend.tsx +0 -196
  43. package/examples/websocket/server.ts +0 -138
  44. package/mini-services/.gitkeep +0 -0
  45. package/mini-services/mcp-server/README.md +0 -86
  46. package/mini-services/mcp-server/bun.lock +0 -202
  47. package/mini-services/mcp-server/index.ts +0 -504
  48. package/mini-services/mcp-server/package.json +0 -40
  49. package/next.config.ts +0 -12
  50. package/postcss.config.mjs +0 -5
  51. package/prisma/schema.prisma +0 -32
  52. package/public/google6f58ca6be85fa903.html +0 -1
  53. package/public/logo.svg +0 -29
  54. package/public/manifest.json +0 -51
  55. package/public/og-icon-256.png +0 -0
  56. package/public/og.png +0 -0
  57. package/public/robots.txt +0 -25
  58. package/public/sitemap.xml +0 -23
  59. package/public/tokcalc-demo.gif +0 -0
  60. package/scripts/og-template.html +0 -120
  61. package/scripts/render-og.mjs +0 -43
  62. package/server.json +0 -21
  63. package/src/app/api/pricing/aws/route.ts +0 -186
  64. package/src/app/api/pricing/azure/route.ts +0 -168
  65. package/src/app/api/pricing/gcp/route.ts +0 -230
  66. package/src/app/api/pricing/vast-ai/route.ts +0 -164
  67. package/src/app/api/route.ts +0 -5
  68. package/src/app/compare/h100-vs-h200/layout.tsx +0 -30
  69. package/src/app/compare/h100-vs-h200/page.tsx +0 -328
  70. package/src/app/globals.css +0 -122
  71. package/src/app/layout.tsx +0 -276
  72. package/src/app/page.tsx +0 -2670
  73. package/src/components/azure-live-pricing.tsx +0 -185
  74. package/src/components/benchmark-import.tsx +0 -340
  75. package/src/components/confidence-badge.tsx +0 -116
  76. package/src/components/live-pricing-comparison.tsx +0 -241
  77. package/src/components/theme-provider.tsx +0 -11
  78. package/src/components/theme-toggle.tsx +0 -55
  79. package/src/components/ui/accordion.tsx +0 -66
  80. package/src/components/ui/alert-dialog.tsx +0 -157
  81. package/src/components/ui/alert.tsx +0 -66
  82. package/src/components/ui/aspect-ratio.tsx +0 -11
  83. package/src/components/ui/avatar.tsx +0 -53
  84. package/src/components/ui/badge.tsx +0 -46
  85. package/src/components/ui/breadcrumb.tsx +0 -109
  86. package/src/components/ui/button.tsx +0 -59
  87. package/src/components/ui/calendar.tsx +0 -213
  88. package/src/components/ui/card.tsx +0 -92
  89. package/src/components/ui/carousel.tsx +0 -241
  90. package/src/components/ui/chart.tsx +0 -353
  91. package/src/components/ui/checkbox.tsx +0 -32
  92. package/src/components/ui/collapsible.tsx +0 -33
  93. package/src/components/ui/command.tsx +0 -184
  94. package/src/components/ui/context-menu.tsx +0 -252
  95. package/src/components/ui/dialog.tsx +0 -143
  96. package/src/components/ui/drawer.tsx +0 -135
  97. package/src/components/ui/dropdown-menu.tsx +0 -257
  98. package/src/components/ui/form.tsx +0 -167
  99. package/src/components/ui/hover-card.tsx +0 -44
  100. package/src/components/ui/input-otp.tsx +0 -77
  101. package/src/components/ui/input.tsx +0 -21
  102. package/src/components/ui/label.tsx +0 -24
  103. package/src/components/ui/menubar.tsx +0 -276
  104. package/src/components/ui/navigation-menu.tsx +0 -168
  105. package/src/components/ui/pagination.tsx +0 -127
  106. package/src/components/ui/popover.tsx +0 -48
  107. package/src/components/ui/progress.tsx +0 -31
  108. package/src/components/ui/radio-group.tsx +0 -45
  109. package/src/components/ui/resizable.tsx +0 -56
  110. package/src/components/ui/scroll-area.tsx +0 -58
  111. package/src/components/ui/select.tsx +0 -185
  112. package/src/components/ui/separator.tsx +0 -28
  113. package/src/components/ui/sheet.tsx +0 -139
  114. package/src/components/ui/sidebar.tsx +0 -726
  115. package/src/components/ui/skeleton.tsx +0 -13
  116. package/src/components/ui/slider.tsx +0 -63
  117. package/src/components/ui/sonner.tsx +0 -25
  118. package/src/components/ui/switch.tsx +0 -31
  119. package/src/components/ui/table.tsx +0 -116
  120. package/src/components/ui/tabs.tsx +0 -66
  121. package/src/components/ui/textarea.tsx +0 -18
  122. package/src/components/ui/toast.tsx +0 -129
  123. package/src/components/ui/toaster.tsx +0 -35
  124. package/src/components/ui/toggle-group.tsx +0 -73
  125. package/src/components/ui/toggle.tsx +0 -47
  126. package/src/components/ui/tooltip.tsx +0 -61
  127. package/src/components/vast-ai-live-pricing.tsx +0 -176
  128. package/src/hooks/use-mobile.ts +0 -19
  129. package/src/hooks/use-toast.ts +0 -194
  130. package/src/lib/benchmark-parser-sglang.ts +0 -150
  131. package/src/lib/benchmark-parser-tokcalc.ts +0 -247
  132. package/src/lib/benchmark-parser-trtllm.ts +0 -152
  133. package/src/lib/benchmark-parser-vllm.ts +0 -198
  134. package/src/lib/benchmark-schema.ts +0 -263
  135. package/src/lib/db.ts +0 -13
  136. package/src/lib/engine-presets.ts +0 -183
  137. package/src/lib/price-schema.ts +0 -141
  138. package/src/lib/token-calc.ts +0 -808
  139. package/src/lib/track.ts +0 -31
  140. package/src/lib/url-state.ts +0 -256
  141. package/src/lib/utils.ts +0 -6
  142. package/tailwind.config.ts +0 -64
  143. package/tests/database-runtime-build.sh +0 -75
  144. package/tests/python-runtime-build.sh +0 -64
  145. package/tests/python-runtime-container.sh +0 -31
  146. package/tool-results/bash_1789888171144_2c5381860539.txt +0 -161
  147. package/tool-results/bash_1789888175925_49c53ba3c61b.txt +0 -191
  148. package/tool-results/bash_1789888181202_49c53ba3c61b.txt +0 -191
  149. package/tool-results/bash_1789888195219_4a86a5c91411.txt +0 -200
  150. package/tool-results/bash_1789888203128_6cca13c71b47.txt +0 -199
  151. package/tool-results/bash_1789929256963_2a52aff0d0a8.txt +0 -160
  152. package/tool-results/read_1789888151021_69f58eec6a5b.txt +0 -653
  153. package/tool-results/read_1789888153837_1d3a8bfc2a94.txt +0 -653
  154. package/tool-results/read_1789888163087_ccc406d47505.txt +0 -122
  155. package/tool-results/read_1789888167347_67d1d7c9830a.txt +0 -122
  156. package/tool-results/read_1789929252529_d90e8f383a25.txt +0 -285
  157. package/tsconfig.json +0 -42
  158. package/upload/Pasted Content_1789887800864.txt +0 -652
  159. package/upload/Pasted Content_1789887909561.txt +0 -652
  160. package/upload/Pasted Content_1789887918428.txt +0 -652
  161. package/upload/Pasted Content_1789887959420.txt +0 -652
  162. package/upload/Pasted Content_1789888020485.txt +0 -652
  163. package/upload/Pasted Content_1789888058079.txt +0 -652
  164. package/upload/Pasted Content_1789888885033.txt +0 -686
  165. package/upload/Pasted Content_1789928912741.txt +0 -285
  166. package/upload/Pasted Content_1789928938402.txt +0 -285
  167. package/upload/Pasted Content_1789929160389.txt +0 -285
  168. package/upload/Pasted Content_1789929176660.txt +0 -285
  169. package/upload/issue_vision.json +0 -28
  170. package/upload/pasted_image_1789883175209.png +0 -0
  171. package/upload/pasted_image_1789899056690.png +0 -0
  172. package/upload/pasted_image_1789900371483.png +0 -0
  173. package/upload/pasted_image_1789900472823.png +0 -0
  174. package/upload/pasted_image_1789900490374.png +0 -0
  175. package/upload/pasted_image_1789900585552.png +0 -0
  176. package/upload/pasted_image_1789900606519.png +0 -0
  177. package/upload/pasted_image_1789901598705.png +0 -0
  178. package/upload/pasted_image_1789901613545.png +0 -0
  179. package/upload/pasted_image_1789978382674.png +0 -0
  180. package/upload/pasted_image_1789978392749.png +0 -0
  181. package/upload/pasted_image_1789978474879.png +0 -0
  182. package/upload/pasted_image_1789978523652.png +0 -0
  183. package/upload/pasted_image_1789984219089.png +0 -0
  184. package/upload/pasted_image_1789984491896.png +0 -0
  185. package/upload/pasted_image_1789985017950.png +0 -0
  186. package/upload/pasted_image_1789985036765.png +0 -0
  187. package/upload/pasted_image_1789985049848.png +0 -0
  188. package/upload/pasted_image_1790002427833.png +0 -0
  189. package/upload/pasted_image_1790002659944.png +0 -0
  190. package/upload/pasted_image_1790037038476.png +0 -0
  191. package/upload/screenshot_analysis.json +0 -28
  192. package/upload/vision_output.json +0 -28
package/src/app/page.tsx DELETED
@@ -1,2670 +0,0 @@
1
- "use client";
2
-
3
- import { useEffect, useMemo, useState } from "react";
4
- import {
5
- Calculator,
6
- Gauge,
7
- Clock,
8
- DollarSign,
9
- Cpu,
10
- MemoryStick,
11
- Database,
12
- Layers,
13
- Zap,
14
- AlertTriangle,
15
- CheckCircle2,
16
- Github,
17
- Info,
18
- BookOpen,
19
- HelpCircle,
20
- FileText,
21
- } from "lucide-react";
22
- import { ThemeToggle } from "@/components/theme-toggle";
23
- import { ConfidenceDot, ConfidenceBadge, ConfidenceLegend } from "@/components/confidence-badge";
24
- import { BenchmarkImport } from "@/components/benchmark-import";
25
- import { AzureLivePricing } from "@/components/azure-live-pricing";
26
- import { VastAiLivePricing } from "@/components/vast-ai-live-pricing";
27
- import { LivePricingComparison } from "@/components/live-pricing-comparison";
28
- import type { Confidence } from "@/lib/token-calc";
29
- import { track } from "@/lib/track";
30
- import { ENGINE_PRESETS, getEngine, getSupportedQuants, type EngineId } from "@/lib/engine-presets";
31
- import {
32
- Bar,
33
- BarChart,
34
- CartesianGrid,
35
- Cell,
36
- ResponsiveContainer,
37
- Tooltip,
38
- XAxis,
39
- YAxis,
40
- } from "recharts";
41
- import {
42
- Card,
43
- CardContent,
44
- CardDescription,
45
- CardHeader,
46
- CardTitle,
47
- } from "@/components/ui/card";
48
- import { Button } from "@/components/ui/button";
49
- import { Input } from "@/components/ui/input";
50
- import { Label } from "@/components/ui/label";
51
- import {
52
- Select,
53
- SelectContent,
54
- SelectGroup,
55
- SelectItem,
56
- SelectLabel,
57
- SelectTrigger,
58
- SelectValue,
59
- } from "@/components/ui/select";
60
- import { Switch } from "@/components/ui/switch";
61
- import { Slider } from "@/components/ui/slider";
62
- import { Badge } from "@/components/ui/badge";
63
- import { Separator } from "@/components/ui/separator";
64
- import {
65
- Tooltip as UITooltip,
66
- TooltipContent,
67
- TooltipProvider,
68
- TooltipTrigger,
69
- } from "@/components/ui/tooltip";
70
- import {
71
- Accordion,
72
- AccordionContent,
73
- AccordionItem,
74
- AccordionTrigger,
75
- } from "@/components/ui/accordion";
76
- import {
77
- calculate,
78
- fmtBytes,
79
- fmtMoney,
80
- fmtMs,
81
- fmtTokens,
82
- fmtContext,
83
- GPUS,
84
- MODELS,
85
- QUANTIZATIONS,
86
- DEFAULT_BATCHING_MULTIPLIER,
87
- computeMaxConcurrency,
88
- computeLongContextPrefillMs,
89
- recommendTopology,
90
- computeKVCacheGb,
91
- type CacheTTL,
92
- type CalcInput,
93
- type GpuCategory,
94
- type Quantization,
95
- } from "@/lib/token-calc";
96
- import {
97
- parseUrlHash,
98
- serializeCalcState,
99
- serializeBvbState,
100
- writeUrlHash,
101
- copyCurrentUrlToClipboard,
102
- saveCalcToStorage,
103
- loadCalcFromStorage,
104
- saveBvbToStorage,
105
- loadBvbFromStorage,
106
- type CalcTabState,
107
- } from "@/lib/url-state";
108
- import { useToast } from "@/hooks/use-toast";
109
- import { Share2, Check } from "lucide-react";
110
-
111
- const CATEGORY_LABEL: Record<GpuCategory, string> = {
112
- datacenter: "Datacenter",
113
- workstation: "Workstation",
114
- consumer: "Consumer",
115
- mac: "Mac unified",
116
- };
117
-
118
- /* ---------- PLAIN-ENGLISH GLOSSARY ---------- */
119
- /* Every technical term has a short, jargon-free explanation here. */
120
- const GLOSSARY = {
121
- tokens: "A 'token' is roughly 3/4 of an English word. So 1,000 tokens ≈ 750 words. The model reads and writes text in tokens, not characters.",
122
- decode: "'Decode' = the model WRITING new tokens, one at a time, after reading your prompt. This is the slow part where users wait.",
123
- prefill: "'Prefill' = the model READING your prompt before it starts answering. Happens once per request, in parallel.",
124
- tokensPerSec: "How many tokens the model writes per second. Higher = faster responses for users.",
125
- singleStream: "When you serve just 1 user at a time. This is the max speed a single user will see.",
126
- aggregate: "Total tokens/sec when serving many users at once (batching). Always higher than single-stream because the GPU is shared efficiently.",
127
- latency: "Total wait time from when a user sends a prompt to when they get the full answer back.",
128
- vram: "Video RAM — the GPU's own memory. The model + its scratch space (KV cache) must fit entirely inside this. If it doesn't fit, the model won't run on this GPU.",
129
- modelWeights: "The actual 'brain' of the model — billions of numbers stored in VRAM. Size = params × bytes per param. Bigger = smarter but slower and pricier.",
130
- kvCache: "Scratch memory the model keeps while generating, so it doesn't re-read your prompt every token. Grows with conversation length and number of users.",
131
- params: "Number of 'neurons' (actually weight values) in the model. More params = smarter but slower & uses more VRAM. 'B' = billion. So 7B = 7,000,000,000 weights.",
132
- active: "For MoE (mixture-of-experts) models, only some experts activate per token. 'Active params' = what actually gets used per token, vs total params = the whole model size on disk.",
133
- layers: "How many 'stacked blocks' the model has. Deeper = smarter reasoning, but each token must pass through all of them, so it's slower.",
134
- hbmBw: "How fast the GPU can read its own memory (in GB per second). This is the #1 factor for LLM speed — the GPU must load the entire model weights once per generated token.",
135
- flops: "Trillions of math operations per second (Tera-FLOPS). Determines how fast the model can process your prompt (prefill phase) and large batches.",
136
- tensorParallel: "Splitting one model across multiple GPUs (each GPU holds part of the model). Lets you run models too big for 1 GPU, with near-linear speedup.",
137
- quantization: "Shrinking the model's weights from 16-bit numbers to 8-bit or 4-bit. Smaller = faster + cheaper, but slightly less accurate. INT4 = 4-bit, FP16 = 16-bit.",
138
- batchSize: "How many user requests the GPU serves at once. 1 = one user; 8 = eight users sharing one forward pass. Higher batch = more total throughput, but each user waits the same time.",
139
- promptTokens: "How long your input is. A 1-paragraph question is ~100 tokens; a 10-page document is ~3,000 tokens. Longer prompts take longer to read (prefill).",
140
- outputTokens: "How long the model's answer should be. 200 tokens ≈ 150 words. Longer answers take proportionally more time.",
141
- speculative: "A trick where a small 'draft' model guesses the next several tokens, then the big model verifies in one pass. Can 2-3x speed if the draft model is accurate.",
142
- batchCrossover: "The batch size where the GPU switches from 'memory-limited' to 'math-limited'. Below this, more users = more speed for free. Above this, you've maxed out the math capacity.",
143
- costPerMTokens: "What you pay to generate 1 million output tokens. The standard pricing unit for LLM APIs (e.g. GPT-4 charges $/1M tokens).",
144
- moe: "Mixture-of-Experts: a model with many 'expert' sub-networks, where only a few activate per token. Total size on disk is large, but per-token work is small. Example: Mixtral 8x7B has 47B total but only ~13B active per token.",
145
- gqa: "Grouped-Query Attention: an optimization that shares the same 'memory scratch space' (KV cache) across multiple attention heads. Cuts VRAM use significantly.",
146
- continuousBatching: "A serving trick where the GPU keeps generating tokens for active requests while NEW requests join mid-flight. Big speedup (1.5-4x typical) for production traffic. Used by vLLM, TGI, SGLang.",
147
- ttft: "Time-to-first-token: how long the user waits before seeing the first word. Equals prefill time (the model reading your prompt). Lower = better UX.",
148
- itl: "Inter-token-latency: how fast the model writes each token AFTER the first. This is what users perceive as 'streaming speed'.",
149
- promptCaching: "Reuse the model's work from a repeated prompt prefix (system prompt, RAG docs, tool schemas). On cache hit, you skip the prefill computation for that prefix. Anthropic: 90% off cached tokens. OpenAI: 50% off.",
150
- reasoningTokens: "Hidden tokens the model generates internally before answering (e.g. OpenAI o1, DeepSeek R1, Claude thinking). Billed as output but invisible to the user. Important for cost estimates of reasoning models.",
151
- gguf: "A file format used by llama.cpp for local inference. Has many sub-variants (Q2_K through Q8_0) trading size for quality. Q4_K_M is the recommended sweet spot for local Llama/Mistral.",
152
- fp8: "8-bit floating-point format. Native on NVIDIA H100/H200 — actually FASTER than FP16 thanks to dedicated FP8 tensor cores. Same VRAM as INT8 but better quality.",
153
- nvfp4: "NVIDIA's 4-bit floating-point format with block scaling. Native on Blackwell (B200/B300). Roughly 2x faster than FP8 on Blackwell per cited benchmarks.",
154
- } as const;
155
-
156
- type GlossaryKey = keyof typeof GLOSSARY;
157
-
158
- type Tab = "calculator" | "build-vs-buy" | "reference";
159
-
160
- export default function Home() {
161
- // ---- Top-level tab state ----
162
- const [activeTab, setActiveTab] = useState<Tab>("calculator");
163
-
164
- // ---- Inputs (Calculator tab) ----
165
- const [modelId, setModelId] = useState("llama3-8b");
166
- const [gpuId, setGpuId] = useState("a100-80");
167
- const [quantization, setQuantization] = useState<Quantization>("fp16");
168
- const [numGpus, setNumGpus] = useState(1);
169
- const [batchSize, setBatchSize] = useState(1);
170
- const [promptTokens, setPromptTokens] = useState(500);
171
- const [outputTokens, setOutputTokens] = useState(200);
172
- const [gpuHourlyCost, setGpuHourlyCost] = useState<number | "">("");
173
- const [useSpeculative, setUseSpeculative] = useState(false);
174
- const [speculativeBoost, setSpeculativeBoost] = useState(2.0);
175
- // === Phase 2 state ===
176
- const [useContinuousBatching, setUseContinuousBatching] = useState(false);
177
- const [continuousBatchingMultiplier, setContinuousBatchingMultiplier] = useState(DEFAULT_BATCHING_MULTIPLIER);
178
- const [reasoningTokens, setReasoningTokens] = useState(0);
179
- const [cachePrefixTokens, setCachePrefixTokens] = useState(0);
180
- const [cacheHitRate, setCacheHitRate] = useState(0);
181
- const [cacheTTL, setCacheTTL] = useState<CacheTTL>("5m");
182
- const [cacheProvider, setCacheProvider] = useState<"self-hosted" | "anthropic" | "openai">("self-hosted");
183
- // === Engine selection ===
184
- const [engineId, setEngineId] = useState<EngineId>("generic");
185
- const selectedEngine = getEngine(engineId);
186
-
187
- // Auto-switch quantization if current one isn't supported by the selected engine
188
- useEffect(() => {
189
- if (engineId === "generic") return; // Generic supports everything
190
- const supported = getSupportedQuants(engineId);
191
- if (!supported.includes(quantization)) {
192
- // Switch to FP16 (supported by all engines) or first available
193
- if (supported.includes("fp16")) {
194
- setQuantization("fp16");
195
- } else {
196
- setQuantization(supported[0]);
197
- }
198
- }
199
- }, [engineId]); // eslint-disable-line react-hooks/exhaustive-deps
200
-
201
- // === URL-share state (Phase A) ===
202
- // Track whether we've restored state from URL — prevents the write effect
203
- // from clobbering the URL on first mount before we've read it.
204
- const [urlRestored, setUrlRestored] = useState(false);
205
- const [shareCopied, setShareCopied] = useState(false);
206
- const { toast } = useToast();
207
-
208
- // ---- One-time mount: read URL hash and restore state ----
209
- // Priority: URL hash > localStorage > defaults
210
- useEffect(() => {
211
- /* eslint-disable react-hooks/set-state-in-effect */
212
- if (typeof window === "undefined") return;
213
- const hash = window.location.hash;
214
- if (!hash || hash === "#") {
215
- // No URL hash — try localStorage for last-saved session
216
- const stored = loadCalcFromStorage();
217
- if (stored) {
218
- if (stored.modelId) setModelId(stored.modelId);
219
- if (stored.gpuId) setGpuId(stored.gpuId);
220
- if (stored.quantization) setQuantization(stored.quantization as Quantization);
221
- if (typeof stored.numGpus === "number") setNumGpus(stored.numGpus);
222
- if (typeof stored.batchSize === "number") setBatchSize(stored.batchSize);
223
- if (typeof stored.promptTokens === "number") setPromptTokens(stored.promptTokens);
224
- if (typeof stored.outputTokens === "number") setOutputTokens(stored.outputTokens);
225
- if (stored.gpuHourlyCost !== undefined) setGpuHourlyCost(stored.gpuHourlyCost);
226
- if (stored.useSpeculative) {
227
- setUseSpeculative(true);
228
- if (typeof stored.speculativeBoost === "number") setSpeculativeBoost(stored.speculativeBoost);
229
- }
230
- if (stored.useContinuousBatching) {
231
- setUseContinuousBatching(true);
232
- if (typeof stored.continuousBatchingMultiplier === "number") setContinuousBatchingMultiplier(stored.continuousBatchingMultiplier);
233
- }
234
- if (typeof stored.reasoningTokens === "number") setReasoningTokens(stored.reasoningTokens);
235
- if (typeof stored.cachePrefixTokens === "number") setCachePrefixTokens(stored.cachePrefixTokens);
236
- if (typeof stored.cacheHitRate === "number") setCacheHitRate(stored.cacheHitRate);
237
- if (stored.cacheTTL) setCacheTTL(stored.cacheTTL as CacheTTL);
238
- if (stored.cacheProvider) setCacheProvider(stored.cacheProvider as "self-hosted" | "anthropic" | "openai");
239
- if (stored.engineId) setEngineId(stored.engineId as EngineId);
240
- }
241
- setUrlRestored(true);
242
- return;
243
- }
244
- const parsed = parseUrlHash(hash);
245
- setActiveTab(parsed.tab);
246
- if (parsed.calc) {
247
- const c = parsed.calc;
248
- if (c.modelId) setModelId(c.modelId);
249
- if (c.gpuId) setGpuId(c.gpuId);
250
- if (c.quantization) setQuantization(c.quantization as Quantization);
251
- if (typeof c.numGpus === "number") setNumGpus(c.numGpus);
252
- if (typeof c.batchSize === "number") setBatchSize(c.batchSize);
253
- if (typeof c.promptTokens === "number") setPromptTokens(c.promptTokens);
254
- if (typeof c.outputTokens === "number") setOutputTokens(c.outputTokens);
255
- if (c.gpuHourlyCost !== undefined) setGpuHourlyCost(c.gpuHourlyCost);
256
- if (c.useSpeculative) {
257
- setUseSpeculative(true);
258
- if (typeof c.speculativeBoost === "number") setSpeculativeBoost(c.speculativeBoost);
259
- }
260
- if (c.useContinuousBatching) {
261
- setUseContinuousBatching(true);
262
- if (typeof c.continuousBatchingMultiplier === "number") setContinuousBatchingMultiplier(c.continuousBatchingMultiplier);
263
- }
264
- if (typeof c.reasoningTokens === "number") setReasoningTokens(c.reasoningTokens);
265
- if (typeof c.cachePrefixTokens === "number") setCachePrefixTokens(c.cachePrefixTokens);
266
- if (typeof c.cacheHitRate === "number") setCacheHitRate(c.cacheHitRate);
267
- if (c.cacheTTL) setCacheTTL(c.cacheTTL as CacheTTL);
268
- if (c.cacheProvider) setCacheProvider(c.cacheProvider as "self-hosted" | "anthropic" | "openai");
269
- if (c.engineId) setEngineId(c.engineId as EngineId);
270
- }
271
- setUrlRestored(true);
272
- /* eslint-enable react-hooks/set-state-in-effect */
273
- }, []);
274
-
275
- // ---- Continuous: when Calculator state changes, update URL hash + localStorage ----
276
- useEffect(() => {
277
- if (!urlRestored) return; // wait until we've read the initial URL
278
- const state: CalcTabState = {
279
- modelId, gpuId, quantization, numGpus, batchSize,
280
- promptTokens, outputTokens, gpuHourlyCost,
281
- useSpeculative, speculativeBoost,
282
- useContinuousBatching, continuousBatchingMultiplier,
283
- reasoningTokens, cachePrefixTokens, cacheHitRate, cacheTTL, cacheProvider,
284
- engineId,
285
- };
286
- if (activeTab === "calculator") {
287
- writeUrlHash(serializeCalcState(state));
288
- }
289
- saveCalcToStorage(state); // always persist to localStorage
290
- }, [
291
- urlRestored, activeTab,
292
- modelId, gpuId, quantization, numGpus, batchSize, promptTokens, outputTokens,
293
- gpuHourlyCost, useSpeculative, speculativeBoost, useContinuousBatching,
294
- continuousBatchingMultiplier, reasoningTokens, cachePrefixTokens, cacheHitRate,
295
- cacheTTL, cacheProvider,
296
- engineId,
297
- ]);
298
- // ---- Share button handler ----
299
- const handleShare = async () => {
300
- // Track: high-signal event — user found something worth sharing
301
- track("shared_scenario", { tab: activeTab });
302
-
303
- // For Calculator tab, force a fresh write to be safe.
304
- if (activeTab === "calculator") {
305
- const state: CalcTabState = {
306
- modelId, gpuId, quantization, numGpus, batchSize,
307
- promptTokens, outputTokens, gpuHourlyCost,
308
- useSpeculative, speculativeBoost,
309
- useContinuousBatching, continuousBatchingMultiplier,
310
- reasoningTokens, cachePrefixTokens, cacheHitRate, cacheTTL, cacheProvider,
311
- engineId,
312
- };
313
- writeUrlHash(serializeCalcState(state));
314
- }
315
- // MUST stay in the user-gesture call stack for clipboard permission.
316
- // (setTimeout would break the gesture chain — verified painful.)
317
- const ok = await copyCurrentUrlToClipboard();
318
- if (ok) {
319
- setShareCopied(true);
320
- toast({
321
- title: "Share URL copied",
322
- description: activeTab === "calculator"
323
- ? "Anyone who opens this link sees the exact same config."
324
- : activeTab === "build-vs-buy"
325
- ? "Anyone who opens this link sees the same build-vs-buy scenario."
326
- : "URL copied.",
327
- });
328
- setTimeout(() => setShareCopied(false), 2000);
329
- } else {
330
- // Clipboard write failed (e.g., headless browser or no permission).
331
- // Fallback: select the URL bar via document.execCommand('copy').
332
- try {
333
- const urlInput = document.createElement("input");
334
- urlInput.value = window.location.href;
335
- document.body.appendChild(urlInput);
336
- urlInput.select();
337
- document.execCommand("copy");
338
- document.body.removeChild(urlInput);
339
- setShareCopied(true);
340
- toast({ title: "Share URL copied", description: "Paste anywhere — Discord, Slack, GitHub issue, tweet." });
341
- setTimeout(() => setShareCopied(false), 2000);
342
- } catch {
343
- toast({ title: "Couldn't copy automatically", description: "Copy from the address bar manually." });
344
- }
345
- }
346
- };
347
-
348
- // ---- Copy as Markdown handler ----
349
- const handleCopyMarkdown = async () => {
350
- track("copied_markdown", { tab: activeTab });
351
- const md = generateMarkdown();
352
- try {
353
- await navigator.clipboard.writeText(md);
354
- toast({ title: "Markdown copied", description: "Paste into GitHub issues, Slack, Notion, or blog posts." });
355
- } catch {
356
- try {
357
- const el = document.createElement("textarea");
358
- el.value = md;
359
- document.body.appendChild(el);
360
- el.select();
361
- document.execCommand("copy");
362
- document.body.removeChild(el);
363
- toast({ title: "Markdown copied", description: "Paste into GitHub issues, Slack, Notion, or blog posts." });
364
- } catch {
365
- toast({ title: "Couldn't copy", description: "Copy manually from the address bar." });
366
- }
367
- }
368
- };
369
-
370
- // Generate markdown table of current results
371
- function generateMarkdown(): string {
372
- const lines: string[] = [];
373
- lines.push(`## tokcalc — ${selectedModel.name} on ${selectedGpu.name}`);
374
- lines.push("");
375
- lines.push(`| Metric | Value |`);
376
- lines.push(`|---|---|`);
377
- lines.push(`| **Model** | ${selectedModel.name} (${selectedModel.paramsB}B params, ${selectedModel.activeParamsB}B active) |`);
378
- lines.push(`| **GPU** | ${numGpus}× ${selectedGpu.name} (${selectedGpu.vramGb * numGpus} GB total) |`);
379
- lines.push(`| **Quantization** | ${selectedQuant.label} (${selectedQuant.bytesPerParam * 8}-bit) |`);
380
- lines.push(`| **Engine** | ${selectedEngine.name} |`);
381
- lines.push(`| **Context** | ${promptTokens + (cachePrefixTokens ?? 0)} tokens |`);
382
- lines.push(`| **Batch size** | ${batchSize} |`);
383
- if (useContinuousBatching) {
384
- lines.push(`| **Continuous batching** | ${continuousBatchingMultiplier.toFixed(1)}× multiplier |`);
385
- }
386
- if (reasoningTokens > 0) {
387
- lines.push(`| **Reasoning tokens** | ${reasoningTokens} (hidden) |`);
388
- }
389
- lines.push(`| **Generation speed** | ${fmtTokens(result.decodeTokensPerSec)} tok/s (1 user) |`);
390
- lines.push(`| **Total throughput** | ${fmtTokens(result.aggregateTokensPerSec)} tok/s (${batchSize} users) |`);
391
- lines.push(`| **TTFT** | ${fmtMs(result.ttftMs)} |`);
392
- lines.push(`| **ITL** | ${fmtMs(result.itlMs)} per token |`);
393
- lines.push(`| **Total latency** | ${fmtMs(result.totalLatencyMs)} |`);
394
- lines.push(`| **Model size** | ${fmtBytes(result.modelSizeGb)} |`);
395
- lines.push(`| **KV cache** | ${fmtBytes(result.kvCacheTotalGb)} |`);
396
- lines.push(`| **VRAM used** | ${((result.totalVramNeededGb / (selectedGpu.vramGb * numGpus)) * 100).toFixed(1)}% (${fmtBytes(result.totalVramNeededGb)} / ${selectedGpu.vramGb * numGpus} GB) |`);
397
- lines.push(`| **Topology** | ${topologyRec.topology} |`);
398
- lines.push(`| **GPU cost** | ${fmtMoney(result.costPerHour)}/hr |`);
399
- lines.push(`| **Cost per 1M tokens** | ${fmtMoney(result.costPerMillionOutputTokens)} |`);
400
- lines.push(`| **Cost per request** | ${fmtMoney(result.costPerRequest)} |`);
401
- lines.push("");
402
- lines.push(`> Generated by [tokcalc](https://tokcalc.vercel.app) — open-source LLM serving capacity planner.`);
403
- lines.push(`> [Try this configuration](${typeof window !== "undefined" ? window.location.href : "https://tokcalc.vercel.app"})`);
404
- return lines.join("\n");
405
- }
406
-
407
- const input: CalcInput = {
408
- modelId,
409
- gpuId,
410
- quantization,
411
- numGpus,
412
- batchSize,
413
- promptTokens,
414
- outputTokens,
415
- gpuHourlyCost: gpuHourlyCost === "" ? undefined : Number(gpuHourlyCost),
416
- useSpeculative,
417
- speculativeBoost,
418
- useContinuousBatching,
419
- continuousBatchingMultiplier,
420
- reasoningTokens,
421
- cachePrefixTokens,
422
- cacheHitRate,
423
- cacheTTL,
424
- cacheProvider,
425
- engineId,
426
- effMem: selectedEngine.etaMem,
427
- effCompute: selectedEngine.etaCompute,
428
- };
429
-
430
- const result = useMemo(() => calculate(input), [
431
- modelId, gpuId, quantization, numGpus, batchSize, promptTokens,
432
- outputTokens, gpuHourlyCost, useSpeculative, speculativeBoost,
433
- useContinuousBatching, continuousBatchingMultiplier, reasoningTokens,
434
- cachePrefixTokens, cacheHitRate, cacheTTL, cacheProvider,
435
- engineId, selectedEngine.etaMem, selectedEngine.etaCompute,
436
- ]);
437
-
438
- // Chart data — vary batch size from 1 to 32
439
- const chartData = useMemo(() => {
440
- return [1, 2, 4, 8, 16, 32, 64].map((b) => {
441
- const r = calculate({ ...input, batchSize: b });
442
- return {
443
- batch: `B=${b}`,
444
- tokens: Math.round(r.aggregateTokensPerSec),
445
- fits: r.vramFits,
446
- };
447
- });
448
- // eslint-disable-next-line react-hooks/exhaustive-deps
449
- }, [modelId, gpuId, quantization, numGpus, promptTokens, outputTokens, useSpeculative, speculativeBoost, gpuHourlyCost]);
450
-
451
- // Compare GPUs chart — show same model on different GPUs
452
- const gpuCompareData = useMemo(() => {
453
- const model = MODELS.find((m) => m.id === modelId)!;
454
- // only include GPUs that have enough VRAM for the model
455
- const candidates = GPUS.filter((g) => {
456
- const modelSizeGb = model.activeParamsB * QUANTIZATIONS.find((q) => q.id === quantization)!.bytesPerParam;
457
- return g.vramGb * numGpus >= modelSizeGb;
458
- });
459
- return candidates.map((g) => {
460
- const r = calculate({ ...input, gpuId: g.id });
461
- return {
462
- gpu: g.name.replace(/\s+\d+GB$/, "").replace(" SXM5", "").replace(" SXM4", ""),
463
- tokens: Math.round(r.decodeTokensPerSec),
464
- };
465
- });
466
- // eslint-disable-next-line react-hooks/exhaustive-deps
467
- }, [modelId, quantization, numGpus]);
468
-
469
- // Long-context sweep — show max concurrency across 4K → 1M context
470
- const longContextData = useMemo(() => {
471
- const model = MODELS.find((m) => m.id === modelId)!;
472
- const gpu = GPUS.find((g) => g.id === gpuId)!;
473
- const quant = QUANTIZATIONS.find((q) => q.id === quantization)!;
474
- const ctxSizes = [4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576];
475
- return ctxSizes
476
- .filter(ctx => ctx <= model.maxContext)
477
- .map(ctx => {
478
- const maxConcurrent = computeMaxConcurrency(model, gpu, numGpus, ctx, quant.bytesPerParam);
479
- const prefillMs = computeLongContextPrefillMs(model, result.effectiveFlopsTflops, ctx);
480
- const kvGb = computeKVCacheGb(model, ctx, 1);
481
- return {
482
- context: fmtContext(ctx),
483
- contextTokens: ctx,
484
- maxConcurrent,
485
- prefillMs,
486
- kvGb,
487
- };
488
- });
489
- // eslint-disable-next-line react-hooks/exhaustive-deps
490
- }, [modelId, gpuId, numGpus, quantization, result.effectiveFlopsTflops]);
491
-
492
- // Topology recommendation at current context
493
- const topologyRec = useMemo(() => {
494
- const model = MODELS.find((m) => m.id === modelId)!;
495
- const gpu = GPUS.find((g) => g.id === gpuId)!;
496
- const quant = QUANTIZATIONS.find((q) => q.id === quantization)!;
497
- const totalContext = promptTokens + (cachePrefixTokens ?? 0);
498
- return recommendTopology(model, gpu, totalContext, batchSize, quant.bytesPerParam);
499
- // eslint-disable-next-line react-hooks/exhaustive-deps
500
- }, [modelId, gpuId, numGpus, quantization, promptTokens, cachePrefixTokens, batchSize]);
501
-
502
- const selectedGpu = GPUS.find((g) => g.id === gpuId)!;
503
- const selectedModel = MODELS.find((m) => m.id === modelId)!;
504
- const selectedQuant = QUANTIZATIONS.find((q) => q.id === quantization)!;
505
-
506
- return (
507
- <TooltipProvider delayDuration={200}>
508
- <div className="min-h-screen flex flex-col bg-background text-foreground">
509
- {/* Header */}
510
- <header className="border-b border-border/60 backdrop-blur-sm sticky top-0 z-50 bg-background/80">
511
- <div className="max-w-7xl mx-auto px-4 sm:px-6 py-4 flex items-center justify-between">
512
- <div className="flex items-center gap-2.5">
513
- <div className="size-8 rounded-md bg-gradient-to-br from-emerald-400 to-emerald-600 flex items-center justify-center">
514
- <Gauge className="size-4.5 text-emerald-950" />
515
- </div>
516
- <div>
517
- <h1 className="text-base font-semibold tracking-tight leading-none">
518
- tokcalc
519
- </h1>
520
- <p className="text-[11px] text-muted-foreground mt-0.5 leading-none">
521
- LLM serving capacity planner
522
- </p>
523
- </div>
524
- </div>
525
- <div className="flex items-center gap-2">
526
- <Badge variant="outline" className="text-[11px] gap-1.5 hidden sm:inline-flex">
527
- <span className="size-1.5 rounded-full bg-emerald-500 animate-pulse" />
528
- live formulas
529
- </Badge>
530
- <a
531
- href="https://github.com/stevecrates489-commits/tokcalc"
532
- target="_blank"
533
- rel="noopener noreferrer"
534
- aria-label="View source on GitHub"
535
- className="hidden sm:inline-flex items-center gap-1.5 text-xs text-muted-foreground hover:text-foreground transition-colors"
536
- >
537
- <Github className="size-3.5" />
538
- source
539
- </a>
540
- <Button
541
- variant="outline"
542
- size="sm"
543
- onClick={handleShare}
544
- className="gap-1.5 h-8"
545
- aria-label="Copy shareable URL"
546
- >
547
- {shareCopied ? (
548
- <>
549
- <Check className="size-3.5 text-emerald-500" />
550
- <span className="text-xs">Copied</span>
551
- </>
552
- ) : (
553
- <>
554
- <Share2 className="size-3.5" />
555
- <span className="text-xs hidden sm:inline">Share</span>
556
- </>
557
- )}
558
- </Button>
559
- <Button
560
- variant="outline"
561
- size="sm"
562
- onClick={handleCopyMarkdown}
563
- className="gap-1.5 h-8"
564
- aria-label="Copy results as Markdown"
565
- >
566
- <FileText className="size-3.5" />
567
- <span className="text-xs hidden sm:inline">Markdown</span>
568
- </Button>
569
- <ThemeToggle />
570
- </div>
571
- </div>
572
- </header>
573
-
574
- {/* Main */}
575
- <main className="flex-1">
576
- <div className="max-w-7xl mx-auto px-4 sm:px-6 py-8 sm:py-12">
577
- {/* Hero */}
578
- <div className="mb-8 sm:mb-10 max-w-4xl">
579
- <h2 className="text-3xl sm:text-4xl font-bold tracking-tight leading-tight">
580
- Plan your LLM deployment
581
- <br />
582
- <span className="text-emerald-500">before you rent the GPUs.</span>
583
- </h2>
584
- <p className="mt-3 text-muted-foreground text-sm sm:text-base leading-relaxed">
585
- An open-source LLM serving capacity planner. Estimate model fit, KV-cache,
586
- prefill/decode throughput, continuous batching, latency, multi-GPU scaling,
587
- cloud cost, API cost, and self-hosting break-even — with transparent formulas
588
- and cited benchmarks.
589
- </p>
590
- <div className="mt-4 flex items-start gap-2 text-xs text-muted-foreground bg-muted/30 border border-border/60 rounded-md p-3">
591
- <HelpCircle className="size-3.5 shrink-0 mt-0.5 text-emerald-500" />
592
- <div className="leading-relaxed">
593
- <strong className="text-foreground">Not a GPU expert?</strong> Hover over
594
- any <Info className="inline size-3 text-emerald-500 align-text-bottom" /> icon
595
- to see a plain-English explanation. Or scroll to the
596
- <a href="#glossary" className="text-emerald-500 hover:underline ml-0.5">glossary</a>
597
- at the bottom for the full list.
598
- </div>
599
- </div>
600
- </div>
601
-
602
- {/* What can tokcalc answer? */}
603
- <div className="mb-8 grid grid-cols-1 md:grid-cols-2 gap-3">
604
- <div className="p-4 rounded-lg border border-border/60 bg-muted/20">
605
- <div className="text-[10px] uppercase tracking-wider text-emerald-500 font-semibold mb-2">
606
- Capacity planning questions tokcalc answers
607
- </div>
608
- <ul className="text-xs text-muted-foreground space-y-1.5 leading-relaxed">
609
- <li>→ Can I serve <strong className="text-foreground">Qwen 2.5 72B</strong> at 128K context on 2× H100 with 20 concurrent users?</li>
610
- <li>→ How many <strong className="text-foreground">H200s</strong> for 1,000 req/min with P95 TTFT &lt; 2s?</li>
611
- <li>→ Does <strong className="text-foreground">FP8 or AWQ</strong> save more money once quality + KV cache + engine support are included?</li>
612
- <li>→ At what daily volume does an H100 beat <strong className="text-foreground">GPT-4o</strong> pricing?</li>
613
- </ul>
614
- </div>
615
- <div className="p-4 rounded-lg border border-border/60 bg-muted/20">
616
- <div className="text-[10px] uppercase tracking-wider text-emerald-500 font-semibold mb-2">
617
- What makes tokcalc different
618
- </div>
619
- <ul className="text-xs text-muted-foreground space-y-1.5 leading-relaxed">
620
- <li>✓ <strong className="text-foreground">Transparent formulas</strong> — no black-box throughput assumptions</li>
621
- <li>✓ <strong className="text-foreground">Engine-aware</strong> — continuous batching, paged KV, prefix caching</li>
622
- <li>✓ <strong className="text-foreground">Cited benchmarks</strong> — every multiplier is workload-specific, never universal</li>
623
- <li>✓ <strong className="text-foreground">Provider-neutral pricing</strong> — RunPod / Lambda / Modal / AWS / GCP side-by-side</li>
624
- </ul>
625
- </div>
626
- </div>
627
-
628
- {/* Top-level tab switcher */}
629
- <div className="mb-6 border-b border-border/60">
630
- <div className="flex gap-1 -mb-px overflow-x-auto">
631
- {([
632
- { id: "calculator", label: "Calculator", icon: Calculator },
633
- { id: "build-vs-buy", label: "Build vs Buy", icon: DollarSign },
634
- { id: "reference", label: "Reference", icon: BookOpen },
635
- ] as const).map((tab) => {
636
- const Icon = tab.icon;
637
- const isActive = activeTab === tab.id;
638
- return (
639
- <button
640
- key={tab.id}
641
- onClick={() => {
642
- setActiveTab(tab.id);
643
- // Track: which tab users switch to (signals which workflow matters most)
644
- track("switched_tab", { tab: tab.id });
645
- }}
646
- className={`flex items-center gap-2 px-4 py-2.5 text-sm font-medium border-b-2 transition-colors whitespace-nowrap ${
647
- isActive
648
- ? "border-emerald-500 text-foreground"
649
- : "border-transparent text-muted-foreground hover:text-foreground"
650
- }`}
651
- >
652
- <Icon className={`size-4 ${isActive ? "text-emerald-500" : ""}`} />
653
- {tab.label}
654
- </button>
655
- );
656
- })}
657
- </div>
658
- </div>
659
-
660
- {/* ===== CALCULATOR TAB ===== */}
661
- {activeTab === "calculator" && (
662
- <>
663
- {/* Calculator grid */}
664
- <div className="grid grid-cols-1 lg:grid-cols-12 gap-4 lg:gap-6">
665
- {/* ===== INPUTS ===== */}
666
- <div className="lg:col-span-5 space-y-4">
667
- <Card className="border-border/60 shadow-sm">
668
- <CardHeader className="pb-3">
669
- <CardTitle className="text-sm flex items-center gap-2">
670
- <Layers className="size-4 text-emerald-500" />
671
- 1. Model
672
- <HelpIcon term="params" />
673
- </CardTitle>
674
- <CardDescription className="text-xs">
675
- Which LLM are you running? Bigger = smarter but slower.
676
- </CardDescription>
677
- </CardHeader>
678
- <CardContent className="space-y-3">
679
- <Select value={modelId} onValueChange={setModelId}>
680
- <SelectTrigger className="w-full">
681
- <SelectValue placeholder="Pick a model" />
682
- </SelectTrigger>
683
- <SelectContent className="max-h-80">
684
- {["Llama", "Mistral", "Qwen", "DeepSeek", "Gemma", "Phi", "EleutherAI"].map((fam) => {
685
- const items = MODELS.filter((m) => m.family === fam);
686
- if (items.length === 0) return null;
687
- return (
688
- <SelectGroup key={fam}>
689
- <SelectLabel className="text-[10px] uppercase tracking-wider text-muted-foreground">
690
- {fam}
691
- </SelectLabel>
692
- {items.map((m) => (
693
- <SelectItem key={m.id} value={m.id}>
694
- <span className="flex items-center gap-2">
695
- {m.name}
696
- {m.isMoE && (
697
- <Badge variant="secondary" className="text-[9px] px-1 py-0 h-3.5">
698
- MoE
699
- </Badge>
700
- )}
701
- </span>
702
- </SelectItem>
703
- ))}
704
- </SelectGroup>
705
- );
706
- })}
707
- </SelectContent>
708
- </Select>
709
- <div className="grid grid-cols-3 gap-2 text-[11px]">
710
- <Stat label="Total params" value={`${selectedModel.paramsB.toLocaleString()}B`} term="params" />
711
- <Stat label="Active params" value={`${selectedModel.activeParamsB.toLocaleString()}B`} term="active" />
712
- <Stat label="Layers" value={String(selectedModel.layers)} term="layers" />
713
- </div>
714
- </CardContent>
715
- </Card>
716
-
717
- <Card className="border-border/60 shadow-sm">
718
- <CardHeader className="pb-3">
719
- <CardTitle className="text-sm flex items-center gap-2">
720
- <Cpu className="size-4 text-emerald-500" />
721
- 2. GPU
722
- <HelpIcon term="hbmBw" />
723
- </CardTitle>
724
- <CardDescription className="text-xs">
725
- The chip that runs the model. Memory speed matters most for LLMs.
726
- </CardDescription>
727
- </CardHeader>
728
- <CardContent className="space-y-3">
729
- <Select value={gpuId} onValueChange={setGpuId}>
730
- <SelectTrigger className="w-full">
731
- <SelectValue placeholder="Pick a GPU" />
732
- </SelectTrigger>
733
- <SelectContent className="max-h-80">
734
- {(["datacenter", "workstation", "consumer", "mac"] as GpuCategory[]).map((cat) => {
735
- const items = GPUS.filter((g) => g.category === cat);
736
- if (items.length === 0) return null;
737
- return (
738
- <SelectGroup key={cat}>
739
- <SelectLabel className="text-[10px] uppercase tracking-wider text-muted-foreground">
740
- {CATEGORY_LABEL[cat]}
741
- </SelectLabel>
742
- {items.map((g) => (
743
- <SelectItem key={g.id} value={g.id}>
744
- <span className="flex items-center justify-between w-full">
745
- <span>{g.name}</span>
746
- <span className="text-[10px] text-muted-foreground ml-2">
747
- {g.memBandwidthGbps} GB/s · {g.vramGb}GB
748
- </span>
749
- </span>
750
- </SelectItem>
751
- ))}
752
- </SelectGroup>
753
- );
754
- })}
755
- </SelectContent>
756
- </Select>
757
- <div className="grid grid-cols-3 gap-2 text-[11px]">
758
- <Stat label="Memory speed" value={`${selectedGpu.memBandwidthGbps} GB/s`} term="hbmBw" />
759
- <Stat label="Math speed" value={`${selectedGpu.flopsTflops} TF`} term="flops" />
760
- <Stat label="Memory size" value={`${selectedGpu.vramGb} GB`} term="vram" />
761
- </div>
762
-
763
- {/* Multi-GPU */}
764
- <div className="pt-2">
765
- <div className="flex items-center justify-between mb-2">
766
- <Label className="text-xs flex items-center gap-1">
767
- GPUs to use
768
- <HelpIcon term="tensorParallel" />
769
- </Label>
770
- <Badge variant="outline" className="text-[10px] font-mono">{numGpus}×</Badge>
771
- </div>
772
- <Slider
773
- value={[numGpus]}
774
- min={1}
775
- max={8}
776
- step={1}
777
- onValueChange={(v) => setNumGpus(v[0])}
778
- className="w-full"
779
- />
780
- </div>
781
-
782
- {/* Engine selection */}
783
- <div className="pt-2">
784
- <Label className="text-xs">Inference engine</Label>
785
- <Select
786
- value={engineId}
787
- onValueChange={(v) => {
788
- setEngineId(v as EngineId);
789
- track("selected_engine", { engine: v });
790
- // Auto-enable continuous batching if engine has it on by default
791
- const engine = getEngine(v as EngineId);
792
- if (engine.batchingOnByDefault && !useContinuousBatching) {
793
- setUseContinuousBatching(true);
794
- setContinuousBatchingMultiplier(engine.defaultBatchingMultiplier);
795
- }
796
- if (!engine.batchingOnByDefault && useContinuousBatching && v !== "generic") {
797
- setUseContinuousBatching(false);
798
- }
799
- }}
800
- >
801
- <SelectTrigger className="w-full mt-1">
802
- <SelectValue />
803
- </SelectTrigger>
804
- <SelectContent>
805
- {ENGINE_PRESETS.map((e) => (
806
- <SelectItem key={e.id} value={e.id}>
807
- <span className="flex items-center gap-2">
808
- <span className="font-medium">{e.name}</span>
809
- <span className="text-[10px] text-muted-foreground">
810
- η_mem={e.etaMem} · η_comp={e.etaCompute}
811
- </span>
812
- </span>
813
- </SelectItem>
814
- ))}
815
- </SelectContent>
816
- </Select>
817
- {engineId !== "generic" && (
818
- <p className="text-[10px] text-muted-foreground mt-1.5 leading-relaxed">
819
- {selectedEngine.notes}
820
- </p>
821
- )}
822
- {engineId !== "generic" && selectedEngine.features.length > 0 && (
823
- <div className="flex flex-wrap gap-1 mt-1.5">
824
- {selectedEngine.features.slice(0, 4).map((f, i) => (
825
- <Badge key={i} variant="outline" className="text-[9px] px-1 py-0 h-3.5">
826
- {f}
827
- </Badge>
828
- ))}
829
- {selectedEngine.features.length > 4 && (
830
- <Badge variant="outline" className="text-[9px] px-1 py-0 h-3.5">
831
- +{selectedEngine.features.length - 4} more
832
- </Badge>
833
- )}
834
- </div>
835
- )}
836
- </div>
837
- </CardContent>
838
- </Card>
839
-
840
- <Card className="border-border/60 shadow-sm">
841
- <CardHeader className="pb-3">
842
- <CardTitle className="text-sm flex items-center gap-2">
843
- <MemoryStick className="size-4 text-emerald-500" />
844
- 3. Compression
845
- <HelpIcon term="quantization" />
846
- </CardTitle>
847
- <CardDescription className="text-xs">
848
- Shrink the model to fit & run faster. Small accuracy cost.
849
- </CardDescription>
850
- </CardHeader>
851
- <CardContent className="space-y-3">
852
- {/* Filter quantizations based on selected engine */}
853
- {(() => {
854
- const supportedQuants = getSupportedQuants(engineId);
855
- const filtered = QUANTIZATIONS.filter(q => supportedQuants.includes(q.id));
856
- const hiddenCount = QUANTIZATIONS.length - filtered.length;
857
- return (
858
- <>
859
- <div className="grid grid-cols-2 sm:grid-cols-3 gap-1.5">
860
- {filtered.map((q) => (
861
- <button
862
- key={q.id}
863
- onClick={() => setQuantization(q.id)}
864
- className={`text-xs px-2 py-1.5 rounded-md border transition-colors text-left ${
865
- quantization === q.id
866
- ? "border-emerald-500 bg-emerald-500/10 text-foreground"
867
- : "border-border hover:border-border/80 hover:bg-muted"
868
- }`}
869
- >
870
- <div className="font-medium">{q.label}</div>
871
- <div className="text-[10px] text-muted-foreground mt-0.5">
872
- {q.bytesPerParam * 8}-bit
873
- </div>
874
- </button>
875
- ))}
876
- </div>
877
- {engineId !== "generic" && hiddenCount > 0 && (
878
- <p className="text-[10px] text-muted-foreground flex items-center gap-1">
879
- <Info className="size-3 text-amber-500" />
880
- {hiddenCount} format{hiddenCount > 1 ? "s" : ""} hidden — not supported by {selectedEngine.name}.
881
- {" "}Select <button onClick={() => setEngineId("generic")} className="text-emerald-500 hover:underline">Generic</button> to see all.
882
- </p>
883
- )}
884
- </>
885
- );
886
- })()}
887
- <p className="text-[11px] text-muted-foreground leading-relaxed">
888
- {selectedQuant.description}
889
- </p>
890
- </CardContent>
891
- </Card>
892
-
893
- <Card className="border-border/60 shadow-sm">
894
- <CardHeader className="pb-3">
895
- <CardTitle className="text-sm flex items-center gap-2">
896
- <Database className="size-4 text-emerald-500" />
897
- 4. Workload
898
- </CardTitle>
899
- <CardDescription className="text-xs">
900
- How many users, and how long are the conversations?
901
- </CardDescription>
902
- </CardHeader>
903
- <CardContent className="space-y-3.5">
904
- <div>
905
- <div className="flex items-center justify-between mb-1.5">
906
- <Label className="text-xs flex items-center gap-1">
907
- Concurrent users
908
- <HelpIcon term="batchSize" />
909
- </Label>
910
- <Badge variant="outline" className="text-[10px] font-mono">{batchSize}</Badge>
911
- </div>
912
- <Slider
913
- value={[batchSize]}
914
- min={1}
915
- max={64}
916
- step={1}
917
- onValueChange={(v) => setBatchSize(v[0])}
918
- className="w-full"
919
- />
920
- </div>
921
-
922
- <div>
923
- <Label className="text-xs flex items-center gap-1">
924
- Input length (tokens)
925
- <HelpIcon term="promptTokens" />
926
- </Label>
927
- <Input
928
- type="number"
929
- value={promptTokens}
930
- onChange={(e) => setPromptTokens(Math.max(0, Number(e.target.value)))}
931
- className="mt-1"
932
- min={0}
933
- />
934
- </div>
935
-
936
- <div>
937
- <Label className="text-xs flex items-center gap-1">
938
- Answer length (tokens)
939
- <HelpIcon term="outputTokens" />
940
- </Label>
941
- <Input
942
- type="number"
943
- value={outputTokens}
944
- onChange={(e) => setOutputTokens(Math.max(0, Number(e.target.value)))}
945
- className="mt-1"
946
- min={0}
947
- />
948
- </div>
949
-
950
- <div>
951
- <Label className="text-xs flex items-center gap-1">
952
- GPU price ($/hour)
953
- <span className="text-[10px] text-muted-foreground font-normal ml-1">
954
- (blank = use default ${selectedGpu.usdPerHour}/hr)
955
- </span>
956
- </Label>
957
- <Input
958
- type="number"
959
- step="0.01"
960
- placeholder={`default $${selectedGpu.usdPerHour}`}
961
- value={gpuHourlyCost}
962
- onChange={(e) =>
963
- setGpuHourlyCost(e.target.value === "" ? "" : Number(e.target.value))
964
- }
965
- className="mt-1"
966
- />
967
- </div>
968
-
969
- <Separator />
970
-
971
- <div className="flex items-center justify-between">
972
- <div>
973
- <Label className="text-xs flex items-center gap-1">
974
- Speculative decoding
975
- <HelpIcon term="speculative" />
976
- </Label>
977
- <p className="text-[10px] text-muted-foreground mt-0.5">
978
- Use a small model to predict ahead → big speedup
979
- </p>
980
- </div>
981
- <Switch checked={useSpeculative} onCheckedChange={(v) => { setUseSpeculative(v); track("toggled_feature", { feature: "speculative_decoding", enabled: v }); }} />
982
- </div>
983
- {useSpeculative && (
984
- <div>
985
- <div className="flex items-center justify-between mb-1.5">
986
- <Label className="text-xs">Speed boost factor</Label>
987
- <Badge variant="outline" className="text-[10px] font-mono">
988
- {speculativeBoost.toFixed(1)}×
989
- </Badge>
990
- </div>
991
- <Slider
992
- value={[speculativeBoost]}
993
- min={1.2}
994
- max={4}
995
- step={0.1}
996
- onValueChange={(v) => setSpeculativeBoost(v[0])}
997
- className="w-full"
998
- />
999
- </div>
1000
- )}
1001
-
1002
- <Separator />
1003
-
1004
- {/* === Phase 2: Continuous batching === */}
1005
- <div className="flex items-center justify-between">
1006
- <div>
1007
- <Label className="text-xs flex items-center gap-1">
1008
- Continuous batching
1009
- <HelpIcon term="continuousBatching" />
1010
- </Label>
1011
- <p className="text-[10px] text-muted-foreground mt-0.5">
1012
- vLLM/TGI-style; serves requests mid-flight (1.5–4× typical)
1013
- </p>
1014
- </div>
1015
- <Switch checked={useContinuousBatching} onCheckedChange={(v) => { setUseContinuousBatching(v); track("toggled_feature", { feature: "continuous_batching", enabled: v }); }} />
1016
- </div>
1017
- {useContinuousBatching && (
1018
- <div>
1019
- <div className="flex items-center justify-between mb-1.5">
1020
- <Label className="text-xs">Throughput multiplier</Label>
1021
- <Badge variant="outline" className="text-[10px] font-mono">
1022
- {continuousBatchingMultiplier.toFixed(1)}×
1023
- </Badge>
1024
- </div>
1025
- <Slider
1026
- value={[continuousBatchingMultiplier]}
1027
- min={1.0}
1028
- max={4}
1029
- step={0.1}
1030
- onValueChange={(v) => setContinuousBatchingMultiplier(v[0])}
1031
- className="w-full"
1032
- />
1033
- <p className="text-[10px] text-muted-foreground mt-1.5 leading-relaxed">
1034
- Cited range 1.5–4× is workload-specific (SOSP paper).
1035
- Conservative default: 1.5×.
1036
- </p>
1037
- </div>
1038
- )}
1039
-
1040
- {/* === Phase 2: Reasoning tokens === */}
1041
- <div>
1042
- <Label className="text-xs flex items-center gap-1">
1043
- Hidden reasoning tokens
1044
- <HelpIcon term="reasoningTokens" />
1045
- </Label>
1046
- <Input
1047
- type="number"
1048
- value={reasoningTokens}
1049
- onChange={(e) => setReasoningTokens(Math.max(0, Number(e.target.value)))}
1050
- className="mt-1"
1051
- min={0}
1052
- placeholder="0 — only for o1/R1/Claude-thinking"
1053
- />
1054
- {reasoningTokens > 0 && (
1055
- <p className="text-[10px] text-amber-600 dark:text-amber-400 mt-1.5 leading-relaxed">
1056
- Billed output: {outputTokens + reasoningTokens} tok = {outputTokens} visible + {reasoningTokens} reasoning
1057
- </p>
1058
- )}
1059
- </div>
1060
-
1061
- <Separator />
1062
-
1063
- {/* === Phase 2: Prompt caching === */}
1064
- <div className="flex items-center justify-between">
1065
- <div>
1066
- <Label className="text-xs flex items-center gap-1">
1067
- Prompt caching
1068
- <HelpIcon term="promptCaching" />
1069
- </Label>
1070
- <p className="text-[10px] text-muted-foreground mt-0.5">
1071
- Reuse a shared prefix (system prompt, RAG docs)
1072
- </p>
1073
- </div>
1074
- <Switch
1075
- checked={cachePrefixTokens > 0}
1076
- onCheckedChange={(checked) => {
1077
- setCachePrefixTokens(checked ? 1000 : 0);
1078
- track("toggled_feature", { feature: "prompt_caching", enabled: checked });
1079
- }}
1080
- />
1081
- </div>
1082
- {cachePrefixTokens > 0 && (
1083
- <div className="space-y-3 p-3 rounded-md bg-muted/30 border border-border/40">
1084
- <div>
1085
- <Label className="text-xs">Cache prefix length (tokens)</Label>
1086
- <Input
1087
- type="number"
1088
- value={cachePrefixTokens}
1089
- onChange={(e) => setCachePrefixTokens(Math.max(0, Number(e.target.value)))}
1090
- className="mt-1"
1091
- min={0}
1092
- />
1093
- </div>
1094
- <div>
1095
- <div className="flex items-center justify-between mb-1.5">
1096
- <Label className="text-xs">Cache hit rate</Label>
1097
- <Badge variant="outline" className="text-[10px] font-mono">
1098
- {(cacheHitRate * 100).toFixed(0)}%
1099
- </Badge>
1100
- </div>
1101
- <Slider
1102
- value={[cacheHitRate * 100]}
1103
- min={0}
1104
- max={100}
1105
- step={5}
1106
- onValueChange={(v) => setCacheHitRate(v[0] / 100)}
1107
- className="w-full"
1108
- />
1109
- </div>
1110
- <div>
1111
- <Label className="text-xs">Cache provider</Label>
1112
- <Select value={cacheProvider} onValueChange={(v) => setCacheProvider(v as "self-hosted" | "anthropic" | "openai")}>
1113
- <SelectTrigger className="w-full mt-1">
1114
- <SelectValue />
1115
- </SelectTrigger>
1116
- <SelectContent>
1117
- <SelectItem value="self-hosted">Self-hosted (vLLM APC)</SelectItem>
1118
- <SelectItem value="anthropic">Anthropic (5m/1h TTL)</SelectItem>
1119
- <SelectItem value="openai">OpenAI (50% off cached)</SelectItem>
1120
- </SelectContent>
1121
- </Select>
1122
- </div>
1123
- {cacheProvider === "anthropic" && (
1124
- <div>
1125
- <Label className="text-xs">Cache TTL</Label>
1126
- <Select value={cacheTTL} onValueChange={(v) => setCacheTTL(v as CacheTTL)}>
1127
- <SelectTrigger className="w-full mt-1">
1128
- <SelectValue />
1129
- </SelectTrigger>
1130
- <SelectContent>
1131
- <SelectItem value="5m">5 min (1.25× write)</SelectItem>
1132
- <SelectItem value="1h">1 hour (2× write)</SelectItem>
1133
- </SelectContent>
1134
- </Select>
1135
- </div>
1136
- )}
1137
- </div>
1138
- )}
1139
- </CardContent>
1140
- </Card>
1141
- </div>
1142
-
1143
- {/* ===== RESULTS ===== */}
1144
- <div className="lg:col-span-7 space-y-4">
1145
- {/* Headline numbers */}
1146
- <div className="grid grid-cols-2 sm:grid-cols-4 gap-3">
1147
- <HeadlineCard
1148
- icon={<Gauge className="size-4" />}
1149
- label="Generation speed"
1150
- value={fmtTokens(result.decodeTokensPerSec)}
1151
- sub="tok/s — 1 user"
1152
- highlight
1153
- glossaryKey="decode"
1154
- confidence={result.confidence.decodeTokensPerSec}
1155
- />
1156
- <HeadlineCard
1157
- icon={<Zap className="size-4" />}
1158
- label="Total throughput"
1159
- value={fmtTokens(result.aggregateTokensPerSec)}
1160
- sub={`${batchSize} user${batchSize > 1 ? "s" : ""}${useContinuousBatching ? ` × ${continuousBatchingMultiplier.toFixed(1)}` : ""}`}
1161
- glossaryKey="aggregate"
1162
- confidence={result.confidence.aggregateTokensPerSec}
1163
- />
1164
- <HeadlineCard
1165
- icon={<Clock className="size-4" />}
1166
- label="Wait per request"
1167
- value={fmtMs(result.totalLatencyMs)}
1168
- sub={`${result.billedOutputTokens}-tok answer`}
1169
- glossaryKey="latency"
1170
- confidence={result.confidence.totalLatencyMs}
1171
- />
1172
- <HeadlineCard
1173
- icon={<Clock className="size-4" />}
1174
- label="Time to first token"
1175
- value={fmtMs(result.ttftMs)}
1176
- sub={`+ ${fmtMs(result.itlMs)} per token`}
1177
- glossaryKey="ttft"
1178
- confidence={result.confidence.ttftMs}
1179
- />
1180
- </div>
1181
-
1182
- {/* Long-context warning */}
1183
- {result.longContextWarning && (
1184
- <div className="flex items-start gap-3 p-3 rounded-lg border border-blue-500/30 bg-blue-500/5">
1185
- <Info className="size-4 text-blue-500 shrink-0 mt-0.5" />
1186
- <div className="text-xs">
1187
- <p className="font-medium text-blue-600 dark:text-blue-400">
1188
- Long-context caveat
1189
- </p>
1190
- <p className="text-muted-foreground mt-0.5 leading-relaxed">
1191
- {result.longContextWarning}
1192
- </p>
1193
- </div>
1194
- </div>
1195
- )}
1196
-
1197
- {/* Reasoning tokens callout */}
1198
- {reasoningTokens > 0 && (
1199
- <div className="flex items-start gap-3 p-3 rounded-lg border border-amber-500/30 bg-amber-500/5">
1200
- <AlertTriangle className="size-4 text-amber-500 shrink-0 mt-0.5" />
1201
- <div className="text-xs">
1202
- <p className="font-medium text-amber-600 dark:text-amber-400">
1203
- Reasoning model workload
1204
- </p>
1205
- <p className="text-muted-foreground mt-0.5">
1206
- Billed output: <strong className="text-foreground">{result.billedOutputTokens} tokens</strong> =
1207
- {outputTokens} visible + {reasoningTokens} hidden reasoning. Real latency/cost is
1208
- <strong className="text-amber-600 dark:text-amber-400"> {((result.billedOutputTokens / outputTokens - 1) * 100).toFixed(0)}% higher</strong> than visible answer alone.
1209
- </p>
1210
- </div>
1211
- </div>
1212
- )}
1213
-
1214
- {/* Cache savings callout */}
1215
- {cachePrefixTokens > 0 && result.cacheSavingsPct > 0 && (
1216
- <div className="flex items-start gap-3 p-3 rounded-lg border border-emerald-500/30 bg-emerald-500/5">
1217
- <Zap className="size-4 text-emerald-500 shrink-0 mt-0.5" />
1218
- <div className="text-xs">
1219
- <p className="font-medium text-emerald-600 dark:text-emerald-400">
1220
- Prompt cache savings
1221
- </p>
1222
- <p className="text-muted-foreground mt-0.5">
1223
- {cacheProvider === "self-hosted" && "Self-hosted (vLLM APC): "}
1224
- {cacheProvider === "anthropic" && "Anthropic: "}
1225
- {cacheProvider === "openai" && "OpenAI: "}
1226
- Saves <strong className="text-foreground">{result.prefillTokensAvoided.toLocaleString()} tokens</strong> of prefill on{" "}
1227
- {(cacheHitRate * 100).toFixed(0)}% of requests =
1228
- <strong className="text-emerald-600 dark:text-emerald-400"> {result.cacheSavingsPct.toFixed(0)}% prefill reduction</strong>
1229
- {cacheProvider === "anthropic" && ` (cache read at ${result.cacheReadCostMultiplier}× = 90% off cached tokens)`}
1230
- {cacheProvider === "openai" && ` (cached tokens at ${result.cacheReadCostMultiplier}× = 50% off)`}
1231
- </p>
1232
- </div>
1233
- </div>
1234
- )}
1235
-
1236
- {/* VRAM warning */}
1237
- {!result.vramFits && (
1238
- <div className="flex items-start gap-3 p-3 rounded-lg border border-amber-500/30 bg-amber-500/5">
1239
- <AlertTriangle className="size-4 text-amber-500 shrink-0 mt-0.5" />
1240
- <div className="text-xs">
1241
- <p className="font-medium text-amber-600 dark:text-amber-400">
1242
- Model doesn&apos;t fit in VRAM
1243
- </p>
1244
- <p className="text-muted-foreground mt-0.5">
1245
- Needs <strong>{fmtBytes(result.totalVramNeededGb)}</strong> but only{" "}
1246
- <strong>{selectedGpu.vramGb * numGpus} GB</strong> available. Reduce
1247
- quantization, use more GPUs, or shorter context.
1248
- </p>
1249
- </div>
1250
- </div>
1251
- )}
1252
-
1253
- {/* Memory & Compute breakdown */}
1254
- <Card className="border-border/60 shadow-sm">
1255
- <CardHeader className="pb-3">
1256
- <CardTitle className="text-sm flex items-center gap-2">
1257
- Memory budget
1258
- <HelpIcon term="vram" />
1259
- </CardTitle>
1260
- <CardDescription className="text-xs">
1261
- Will the model + conversation fit in the GPU&apos;s memory?
1262
- </CardDescription>
1263
- </CardHeader>
1264
- <CardContent className="space-y-3">
1265
- <div className="grid grid-cols-2 gap-3 text-sm">
1266
- <Metric
1267
- label="Model size"
1268
- value={fmtBytes(result.modelSizeGb)}
1269
- hint={`${selectedModel.activeParamsB}B × ${selectedQuant.bytesPerParam} bytes`}
1270
- glossaryKey="modelWeights"
1271
- confidence={result.confidence.modelSizeGb}
1272
- />
1273
- <Metric
1274
- label="Conversation memory"
1275
- value={fmtBytes(result.kvCacheTotalGb)}
1276
- hint={`${fmtBytes(result.kvCachePerTokenKb / 1024)}/tok × ${promptTokens} tok × ${batchSize}`}
1277
- glossaryKey="kvCache"
1278
- confidence={result.confidence.kvCacheTotalGb}
1279
- />
1280
- <Metric
1281
- label="Total memory needed"
1282
- value={fmtBytes(result.totalVramNeededGb)}
1283
- hint={result.vramFits ? "✓ fits in GPU memory" : "✗ over budget"}
1284
- warn={!result.vramFits}
1285
- />
1286
- <Metric
1287
- label="Available memory"
1288
- value={`${selectedGpu.vramGb * numGpus} GB`}
1289
- hint={`${selectedGpu.vramGb} GB × ${numGpus} GPU`}
1290
- />
1291
- </div>
1292
-
1293
- {/* VRAM usage bar */}
1294
- <div className="pt-1">
1295
- <div className="flex items-center justify-between text-[11px] mb-1.5">
1296
- <span className="text-muted-foreground">Memory used</span>
1297
- <span className="font-mono">
1298
- {((result.totalVramNeededGb / (selectedGpu.vramGb * numGpus)) * 100).toFixed(1)}%
1299
- </span>
1300
- </div>
1301
- <div className="h-2 rounded-full bg-muted overflow-hidden">
1302
- <div
1303
- className={`h-full transition-all ${
1304
- result.vramFits ? "bg-emerald-500" : "bg-amber-500"
1305
- }`}
1306
- style={{
1307
- width: `${Math.min(100, (result.totalVramNeededGb / (selectedGpu.vramGb * numGpus)) * 100)}%`,
1308
- }}
1309
- />
1310
- </div>
1311
- </div>
1312
- </CardContent>
1313
- </Card>
1314
-
1315
- {/* Throughput detail */}
1316
- <Card className="border-border/60 shadow-sm">
1317
- <CardHeader className="pb-3">
1318
- <CardTitle className="text-sm">Speed breakdown</CardTitle>
1319
- <CardDescription className="text-xs">
1320
- Throughput & latency detail
1321
- </CardDescription>
1322
- </CardHeader>
1323
- <CardContent className="grid grid-cols-2 gap-3 text-sm">
1324
- <Metric
1325
- label="Writing speed (1 user)"
1326
- value={`${fmtTokens(result.decodeTokensPerSec)} tok/s`}
1327
- hint={`${fmtMs(result.decodeTimePerTokenMs)} per token`}
1328
- glossaryKey="decode"
1329
- confidence={result.confidence.decodeTokensPerSec}
1330
- />
1331
- <Metric
1332
- label="Total speed (batched)"
1333
- value={`${fmtTokens(result.aggregateTokensPerSec)} tok/s`}
1334
- hint={`batch ${batchSize}${useContinuousBatching ? ` × ${continuousBatchingMultiplier.toFixed(1)}×` : ""}`}
1335
- glossaryKey="aggregate"
1336
- confidence={result.confidence.aggregateTokensPerSec}
1337
- />
1338
- <Metric
1339
- label="Reading speed (prefill)"
1340
- value={`${fmtTokens(result.prefillTokensPerSec)} tok/s`}
1341
- hint={cachePrefixTokens > 0 && result.cacheSavingsPct > 0
1342
- ? `prompt in ${fmtMs(result.prefillTimeWithCacheMs)} (${result.cacheSavingsPct.toFixed(0)}% cached)`
1343
- : `prompt in ${fmtMs(result.prefillTimeMs)}`}
1344
- glossaryKey="prefill"
1345
- confidence={result.confidence.prefillTokensPerSec}
1346
- />
1347
- <Metric
1348
- label="Batch sweet spot"
1349
- value={result.batchCrossover.toFixed(0)}
1350
- hint="users where adding more stops helping"
1351
- glossaryKey="batchCrossover"
1352
- />
1353
- {reasoningTokens > 0 && (
1354
- <Metric
1355
- label="Billed output"
1356
- value={`${result.billedOutputTokens} tok`}
1357
- hint={`${outputTokens} visible + ${reasoningTokens} hidden reasoning`}
1358
- warn
1359
- />
1360
- )}
1361
- {useSpeculative && (
1362
- <Metric
1363
- label="Speculative boost"
1364
- value={`${speculativeBoost.toFixed(1)}×`}
1365
- hint="assumes accurate draft model"
1366
- glossaryKey="speculative"
1367
- />
1368
- )}
1369
- </CardContent>
1370
- </Card>
1371
-
1372
- {/* Cost */}
1373
- <Card className="border-border/60 shadow-sm">
1374
- <CardHeader className="pb-3">
1375
- <CardTitle className="text-sm flex items-center gap-2">
1376
- <DollarSign className="size-4 text-emerald-500" />
1377
- Money
1378
- </CardTitle>
1379
- <CardDescription className="text-xs">
1380
- How much does it cost to run this?
1381
- </CardDescription>
1382
- </CardHeader>
1383
- <CardContent className="grid grid-cols-2 sm:grid-cols-3 gap-3 text-sm">
1384
- <Metric
1385
- label="GPU rent cost"
1386
- value={fmtMoney(result.costPerHour) + "/hr"}
1387
- hint={numGpus > 1 ? `${selectedGpu.usdPerHour} × ${numGpus} GPUs` : "per GPU"}
1388
- confidence={result.confidence.costPerHour}
1389
- />
1390
- <Metric
1391
- label="$ per 1M tokens"
1392
- value={fmtMoney(result.costPerMillionOutputTokens)}
1393
- hint="standard LLM pricing unit"
1394
- glossaryKey="costPerMTokens"
1395
- confidence={result.confidence.costPerMillionOutputTokens}
1396
- />
1397
- <Metric
1398
- label="$ per request"
1399
- value={fmtMoney(result.costPerRequest)}
1400
- hint={`${outputTokens}-token answer`}
1401
- />
1402
- </CardContent>
1403
- </Card>
1404
-
1405
- {/* Batch scaling chart */}
1406
- <Card className="border-border/60 shadow-sm">
1407
- <CardHeader className="pb-2">
1408
- <CardTitle className="text-sm">How throughput grows with more users</CardTitle>
1409
- <CardDescription className="text-xs">
1410
- More users = more total speed — until the GPU hits its math limit.
1411
- </CardDescription>
1412
- </CardHeader>
1413
- <CardContent>
1414
- <div className="h-[220px] w-full">
1415
- <ResponsiveContainer width="100%" height="100%">
1416
- <BarChart data={chartData} margin={{ top: 8, right: 8, bottom: 0, left: -16 }}>
1417
- <CartesianGrid strokeDasharray="3 3" stroke="var(--border)" opacity={0.4} />
1418
- <XAxis
1419
- dataKey="batch"
1420
- stroke="var(--muted-foreground)"
1421
- fontSize={10}
1422
- tickLine={false}
1423
- axisLine={false}
1424
- />
1425
- <YAxis
1426
- stroke="var(--muted-foreground)"
1427
- fontSize={10}
1428
- tickLine={false}
1429
- axisLine={false}
1430
- tickFormatter={(v) => v >= 1000 ? `${(v / 1000).toFixed(1)}k` : v}
1431
- />
1432
- <Tooltip
1433
- cursor={{ fill: "var(--muted)", opacity: 0.3 }}
1434
- contentStyle={{
1435
- backgroundColor: "var(--background)",
1436
- border: "1px solid var(--border)",
1437
- borderRadius: "6px",
1438
- fontSize: "11px",
1439
- }}
1440
- formatter={(v: number) => [`${v.toLocaleString()} tok/s`, "aggregate"]}
1441
- />
1442
- <Bar dataKey="tokens" radius={[3, 3, 0, 0]}>
1443
- {chartData.map((d, i) => (
1444
- <Cell
1445
- key={i}
1446
- fill={d.batch === `B=${batchSize}` ? "#10b981" : "var(--muted-foreground)"}
1447
- opacity={d.fits ? 1 : 0.3}
1448
- />
1449
- ))}
1450
- </Bar>
1451
- </BarChart>
1452
- </ResponsiveContainer>
1453
- </div>
1454
- </CardContent>
1455
- </Card>
1456
-
1457
- {/* GPU comparison chart */}
1458
- <Card className="border-border/60 shadow-sm">
1459
- <CardHeader className="pb-2">
1460
- <CardTitle className="text-sm">
1461
- Compare GPUs — {selectedModel.name} ({selectedQuant.label})
1462
- </CardTitle>
1463
- <CardDescription className="text-xs">
1464
- Speed of the same model on every GPU big enough to fit it. Green = your current pick.
1465
- </CardDescription>
1466
- </CardHeader>
1467
- <CardContent>
1468
- <div className="h-[260px] w-full">
1469
- <ResponsiveContainer width="100%" height="100%">
1470
- <BarChart
1471
- data={gpuCompareData}
1472
- layout="vertical"
1473
- margin={{ top: 0, right: 16, bottom: 0, left: 24 }}
1474
- >
1475
- <CartesianGrid strokeDasharray="3 3" stroke="var(--border)" opacity={0.4} horizontal={false} />
1476
- <XAxis
1477
- type="number"
1478
- stroke="var(--muted-foreground)"
1479
- fontSize={10}
1480
- tickLine={false}
1481
- axisLine={false}
1482
- tickFormatter={(v) => v >= 1000 ? `${(v / 1000).toFixed(1)}k` : v}
1483
- />
1484
- <YAxis
1485
- type="category"
1486
- dataKey="gpu"
1487
- stroke="var(--muted-foreground)"
1488
- fontSize={10}
1489
- tickLine={false}
1490
- axisLine={false}
1491
- width={84}
1492
- />
1493
- <Tooltip
1494
- cursor={{ fill: "var(--muted)", opacity: 0.3 }}
1495
- contentStyle={{
1496
- backgroundColor: "var(--background)",
1497
- border: "1px solid var(--border)",
1498
- borderRadius: "6px",
1499
- fontSize: "11px",
1500
- }}
1501
- formatter={(v: number) => [`${v.toLocaleString()} tok/s`, "decode"]}
1502
- />
1503
- <Bar dataKey="tokens" radius={[0, 3, 3, 0]}>
1504
- {gpuCompareData.map((d, i) => (
1505
- <Cell
1506
- key={i}
1507
- fill={d.gpu.includes(selectedGpu.name.split(" ")[0]) ? "#10b981" : "#475569"}
1508
- />
1509
- ))}
1510
- </Bar>
1511
- </BarChart>
1512
- </ResponsiveContainer>
1513
- </div>
1514
- </CardContent>
1515
- </Card>
1516
-
1517
- {/* ===== Long-Context Capacity Planner ===== */}
1518
- <Card className="border-border/60 shadow-sm">
1519
- <CardHeader className="pb-2">
1520
- <CardTitle className="text-sm flex items-center gap-2">
1521
- Long-context capacity
1522
- <HelpIcon term="kvCache" />
1523
- </CardTitle>
1524
- <CardDescription className="text-xs">
1525
- How many concurrent users fit as context grows? KV cache scales linearly with context — at 128K+ you&apos;ll likely need multiple GPUs.
1526
- </CardDescription>
1527
- </CardHeader>
1528
- <CardContent className="space-y-3">
1529
- {/* Topology recommendation callout */}
1530
- <div className={`flex items-start gap-3 p-3 rounded-md border ${
1531
- topologyRec.fits
1532
- ? "border-emerald-500/40 bg-emerald-500/5"
1533
- : "border-amber-500/40 bg-amber-500/5"
1534
- }`}>
1535
- <Cpu className={`size-4 shrink-0 mt-0.5 ${topologyRec.fits ? "text-emerald-500" : "text-amber-500"}`} />
1536
- <div className="text-xs">
1537
- <div className="font-medium text-foreground">
1538
- Recommended topology: <span className={topologyRec.fits ? "text-emerald-500" : "text-amber-500"}>{topologyRec.topology}</span>
1539
- </div>
1540
- <div className="text-muted-foreground mt-0.5 leading-relaxed">{topologyRec.reason}</div>
1541
- {topologyRec.hasContextParallel && (
1542
- <div className="text-[10px] text-amber-600 dark:text-amber-400 mt-1">
1543
- ⚠ Context Parallel / RingAttention is advanced — only vLLM/SGLang research branches support it today.
1544
- </div>
1545
- )}
1546
- </div>
1547
- </div>
1548
-
1549
- {/* Context sweep chart — max concurrent users at each context size */}
1550
- <div className="pt-1">
1551
- <div className="flex items-center justify-between text-[11px] mb-1.5">
1552
- <span className="text-muted-foreground">Max concurrent users at each context length</span>
1553
- <span className="font-mono text-[10px] text-muted-foreground">
1554
- {selectedGpu.vramGb * numGpus} GB total VRAM
1555
- </span>
1556
- </div>
1557
- <div className="h-[180px] w-full">
1558
- <ResponsiveContainer width="100%" height="100%">
1559
- <BarChart data={longContextData} margin={{ top: 8, right: 8, bottom: 0, left: -16 }}>
1560
- <CartesianGrid strokeDasharray="3 3" stroke="var(--border)" opacity={0.4} />
1561
- <XAxis
1562
- dataKey="context"
1563
- stroke="var(--muted-foreground)"
1564
- fontSize={10}
1565
- tickLine={false}
1566
- axisLine={false}
1567
- />
1568
- <YAxis
1569
- stroke="var(--muted-foreground)"
1570
- fontSize={10}
1571
- tickLine={false}
1572
- axisLine={false}
1573
- tickFormatter={(v) => v >= 1000 ? `${(v / 1000).toFixed(1)}k` : v}
1574
- />
1575
- <Tooltip
1576
- cursor={{ fill: "var(--muted)", opacity: 0.3 }}
1577
- contentStyle={{
1578
- backgroundColor: "var(--background)",
1579
- border: "1px solid var(--border)",
1580
- borderRadius: "6px",
1581
- fontSize: "11px",
1582
- }}
1583
- formatter={(_v: number, _n: string, props: { payload: { maxConcurrent: number; prefillMs: number; kvGb: number; context: string } }) => {
1584
- const p = props.payload;
1585
- return [
1586
- `${p.maxConcurrent.toLocaleString()} users · prefill ${fmtMs(p.prefillMs)} · KV ${fmtBytes(p.kvGb)}`,
1587
- p.context,
1588
- ];
1589
- }}
1590
- />
1591
- <Bar dataKey="maxConcurrent" radius={[3, 3, 0, 0]}>
1592
- {longContextData.map((d, i) => {
1593
- // Highlight current context (closest match by tokens)
1594
- const totalContext = promptTokens + (cachePrefixTokens ?? 0);
1595
- const isCurrent = d.contextTokens >= totalContext &&
1596
- (i === 0 || longContextData[i - 1].contextTokens < totalContext);
1597
- return (
1598
- <Cell
1599
- key={i}
1600
- fill={isCurrent ? "#10b981" : d.maxConcurrent === 0 ? "#ef4444" : "#475569"}
1601
- opacity={d.maxConcurrent === 0 ? 0.3 : 1}
1602
- />
1603
- );
1604
- })}
1605
- </Bar>
1606
- </BarChart>
1607
- </ResponsiveContainer>
1608
- </div>
1609
- <div className="text-[10px] text-muted-foreground mt-1.5 leading-relaxed">
1610
- Bars show how many concurrent users fit in VRAM at each context length.
1611
- <strong className="text-foreground"> Green</strong> = closest to your current context ({fmtContext(promptTokens + (cachePrefixTokens ?? 0))}).
1612
- <strong className="text-foreground"> Red/dim</strong> = doesn&apos;t fit even at batch=1.
1613
- </div>
1614
- </div>
1615
-
1616
- {/* Detailed metrics table */}
1617
- <div className="pt-1 overflow-x-auto">
1618
- <table className="w-full text-[11px]">
1619
- <thead className="text-[10px] uppercase tracking-wider text-muted-foreground">
1620
- <tr>
1621
- <th className="text-left py-1 pr-3">Context</th>
1622
- <th className="text-right py-1 pr-3">Max users</th>
1623
- <th className="text-right py-1 pr-3">KV/request</th>
1624
- <th className="text-right py-1 pr-3">Prefill time</th>
1625
- <th className="text-right py-1">Fits?</th>
1626
- </tr>
1627
- </thead>
1628
- <tbody>
1629
- {longContextData.map(d => {
1630
- const totalContext = promptTokens + (cachePrefixTokens ?? 0);
1631
- const isCurrent = d.contextTokens >= totalContext &&
1632
- (longContextData.indexOf(d) === 0 || longContextData[longContextData.indexOf(d) - 1].contextTokens < totalContext);
1633
- return (
1634
- <tr key={d.context} className={`border-t border-border/40 ${isCurrent ? "bg-emerald-500/5" : ""}`}>
1635
- <td className="py-1.5 pr-3 font-medium">{d.context}</td>
1636
- <td className="py-1.5 pr-3 text-right font-mono">
1637
- {d.maxConcurrent > 0 ? d.maxConcurrent.toLocaleString() : <span className="text-red-500">0 (no fit)</span>}
1638
- </td>
1639
- <td className="py-1.5 pr-3 text-right font-mono">{fmtBytes(d.kvGb)}</td>
1640
- <td className="py-1.5 pr-3 text-right font-mono">{fmtMs(d.prefillMs)}</td>
1641
- <td className="py-1.5 text-right">
1642
- {d.maxConcurrent > 0
1643
- ? <span className="text-emerald-500">✓</span>
1644
- : <span className="text-red-500">✗</span>}
1645
- </td>
1646
- </tr>
1647
- );
1648
- })}
1649
- </tbody>
1650
- </table>
1651
- </div>
1652
- <div className="text-[10px] text-muted-foreground leading-relaxed pt-1 border-t border-border/40">
1653
- <strong className="text-foreground">Math:</strong> KV/request = 2 × layers × kv_heads × head_dim × bytes/KV × context.
1654
- Prefill includes <strong className="text-foreground">superlinear attention cost</strong> at long context (O(N²) attention dominates beyond ~32K).
1655
- For RingAttention/blockwise attention, real prefill is lower than this estimate.
1656
- Refs: <a href="https://arxiv.org/abs/2310.01889" target="_blank" rel="noopener noreferrer" className="text-emerald-500 hover:underline">RingAttention paper</a>.
1657
- </div>
1658
- </CardContent>
1659
- </Card>
1660
-
1661
- {/* ===== Benchmark Import — calibrate the formula against real data ===== */}
1662
- <BenchmarkImport
1663
- estimate={result}
1664
- modelName={selectedModel.name}
1665
- gpuName={selectedGpu.name}
1666
- />
1667
- </div>
1668
- </div>
1669
- </>
1670
- )}
1671
-
1672
- {/* ===== BUILD-VS-BUY TAB ===== */}
1673
- {activeTab === "build-vs-buy" && (
1674
- <BuildVsBuyTab />
1675
- )}
1676
-
1677
- {/* ===== REFERENCE TAB ===== */}
1678
- {activeTab === "reference" && (
1679
- <ReferenceTab />
1680
- )}
1681
-
1682
- {/* Plain-English glossary — visible on all tabs */}
1683
- <div id="glossary" className="mt-10 scroll-mt-20">
1684
- <Card className="border-border/60 shadow-sm">
1685
- <CardHeader>
1686
- <CardTitle className="text-base flex items-center gap-2">
1687
- <BookOpen className="size-4 text-emerald-500" />
1688
- Plain-English glossary
1689
- </CardTitle>
1690
- <CardDescription className="text-sm">
1691
- What all these words mean, without jargon.
1692
- </CardDescription>
1693
- </CardHeader>
1694
- <CardContent>
1695
- <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-x-6 gap-y-3">
1696
- {Object.entries(GLOSSARY).map(([key, text]) => (
1697
- <div key={key} className="space-y-0.5">
1698
- <div className="text-xs font-semibold text-emerald-500 capitalize">
1699
- {key.replace(/([A-Z])/g, " $1").replace(/^./, (c) => c.toUpperCase())}
1700
- </div>
1701
- <div className="text-[11px] text-muted-foreground leading-relaxed">
1702
- {text}
1703
- </div>
1704
- </div>
1705
- ))}
1706
- </div>
1707
- </CardContent>
1708
- </Card>
1709
- </div>
1710
-
1711
- {/* Confidence legend */}
1712
- <div className="mt-10">
1713
- <ConfidenceLegend />
1714
- <p className="text-[10px] text-muted-foreground mt-2 leading-relaxed max-w-3xl">
1715
- Every metric on this page has a confidence dot (🟢 measured · 🟢 modeled · 🟡 inferred · ⚪ user-supplied).
1716
- <strong className="text-foreground"> Honesty is the moat</strong> — we&apos;d rather show you our uncertainty than
1717
- pretend at a universal &quot;tokens/sec&quot; number. Real-world performance depends on engine, model revision,
1718
- driver, and traffic distribution.
1719
- </p>
1720
- </div>
1721
-
1722
- {/* Formula reference */}
1723
- <div className="mt-10">
1724
- <Card className="border-border/60 shadow-sm">
1725
- <CardHeader>
1726
- <CardTitle className="text-base flex items-center gap-2">
1727
- <Calculator className="size-4 text-emerald-500" />
1728
- The math, transparently
1729
- </CardTitle>
1730
- <CardDescription className="text-sm">
1731
- Every number above comes from these formulas. No black boxes, no hidden
1732
- &quot;magic numbers.&quot; <span className="text-muted-foreground">Optional reading for engineers.</span>
1733
- </CardDescription>
1734
- </CardHeader>
1735
- <CardContent>
1736
- <Accordion type="single" collapsible className="w-full">
1737
- <AccordionItem value="decode">
1738
- <AccordionTrigger className="text-sm">
1739
- Decode tokens/sec (memory-bound)
1740
- </AccordionTrigger>
1741
- <AccordionContent className="text-xs space-y-2">
1742
- <FormulaBox>
1743
- decode_tokens/sec ≈ (HBM_BW × η_mem × quant_eff) / model_size
1744
- </FormulaBox>
1745
- <p className="text-muted-foreground">
1746
- Where <code className="text-emerald-500">η_mem = 0.65</code> (typical
1747
- real-world memory utilization), <code className="text-emerald-500">quant_eff</code> accounts
1748
- for INT4/INT8 dequant overhead, and <code className="text-emerald-500">model_size = params × bytes_per_param</code>.
1749
- For MoE, only <em>active</em> params are loaded per token.
1750
- </p>
1751
- <p className="text-muted-foreground">
1752
- Your numbers: <code>{selectedGpu.memBandwidthGbps}</code> GB/s × 0.65 × {result.quantEfficiency} ={" "}
1753
- <strong className="text-foreground">{result.effectiveBandwidthGbps.toFixed(0)} GB/s effective</strong>,
1754
- divided by model size <strong className="text-foreground">{result.modelSizeGb.toFixed(2)} GB</strong> ={" "}
1755
- <strong className="text-emerald-500">{fmtTokens(result.decodeTokensPerSec)} tok/s</strong>.
1756
- </p>
1757
- </AccordionContent>
1758
- </AccordionItem>
1759
-
1760
- <AccordionItem value="prefill">
1761
- <AccordionTrigger className="text-sm">
1762
- Prefill tokens/sec (compute-bound)
1763
- </AccordionTrigger>
1764
- <AccordionContent className="text-xs space-y-2">
1765
- <FormulaBox>
1766
- prefill_tokens/sec ≈ (GPU_FLOPS × η_compute) / (2 × active_params)
1767
- </FormulaBox>
1768
- <p className="text-muted-foreground">
1769
- Prefill is FLOPS-bound because the prompt tokens are processed in parallel.
1770
- Each token requires roughly <code className="text-emerald-500">2 × N</code> FLOPs (one
1771
- multiply + one add per parameter).
1772
- </p>
1773
- <p className="text-muted-foreground">
1774
- Your numbers: <code>{selectedGpu.flopsTflops * numGpus}</code> TFLOPS × 0.5 ={" "}
1775
- <strong className="text-foreground">{(result.effectiveFlopsTflops).toFixed(0)} TF effective</strong>,
1776
- divided by 2 × {selectedModel.activeParamsB}B params ={" "}
1777
- <strong className="text-emerald-500">{fmtTokens(result.prefillTokensPerSec)} tok/s</strong>.
1778
- </p>
1779
- </AccordionContent>
1780
- </AccordionItem>
1781
-
1782
- <AccordionItem value="batch">
1783
- <AccordionTrigger className="text-sm">
1784
- Batch crossover (memory → compute bound)
1785
- </AccordionTrigger>
1786
- <AccordionContent className="text-xs space-y-2">
1787
- <FormulaBox>
1788
- B_crossover ≈ (bytes_per_param × FLOPS × η_compute) / (2 × HBM_BW × η_mem × quant_eff)
1789
- </FormulaBox>
1790
- <p className="text-muted-foreground">
1791
- Below this batch size, throughput scales linearly with batch (memory-bound). Above it,
1792
- you become compute-bound and adding more batches doesn&apos;t help.
1793
- </p>
1794
- <p className="text-muted-foreground">
1795
- Your crossover point: <strong className="text-emerald-500">{result.batchCrossover.toFixed(1)}</strong> —
1796
- your current batch ({batchSize}) is{" "}
1797
- {batchSize < result.batchCrossover ? (
1798
- <span className="text-emerald-500">memory-bound (good)</span>
1799
- ) : (
1800
- <span className="text-amber-500">compute-bound</span>
1801
- )}.
1802
- </p>
1803
- </AccordionContent>
1804
- </AccordionItem>
1805
-
1806
- <AccordionItem value="kv">
1807
- <AccordionTrigger className="text-sm">
1808
- KV cache size
1809
- </AccordionTrigger>
1810
- <AccordionContent className="text-xs space-y-2">
1811
- <FormulaBox>
1812
- KV = 2 × layers × seq_len × kv_heads × head_dim × batch × dtype_bytes
1813
- </FormulaBox>
1814
- <p className="text-muted-foreground">
1815
- KV cache grows linearly with sequence length and batch size. With GQA
1816
- (grouped-query attention), only <code>kv_heads</code> caches are stored, not
1817
- all query heads.
1818
- </p>
1819
- <p className="text-muted-foreground">
1820
- Your numbers: 2 × {selectedModel.layers} × {promptTokens} × {selectedModel.kvHeads} ×{" "}
1821
- {selectedModel.headDim} × {batchSize} × 2 bytes ={" "}
1822
- <strong className="text-emerald-500">{fmtBytes(result.kvCacheTotalGb)}</strong>.
1823
- </p>
1824
- </AccordionContent>
1825
- </AccordionItem>
1826
-
1827
- <AccordionItem value="latency">
1828
- <AccordionTrigger className="text-sm">
1829
- End-to-end latency per request
1830
- </AccordionTrigger>
1831
- <AccordionContent className="text-xs space-y-2">
1832
- <FormulaBox>
1833
- total = prefill_time + output_tokens × decode_time + overhead
1834
- </FormulaBox>
1835
- <p className="text-muted-foreground">
1836
- Your breakdown: prefill = <strong>{fmtMs(result.prefillTimeMs)}</strong>,
1837
- decode = <strong>{fmtMs(result.totalDecodeTimeMs)}</strong> ({outputTokens} × {fmtMs(result.decodeTimePerTokenMs)}),
1838
- overhead ≈ 30ms → total{" "}
1839
- <strong className="text-emerald-500">{fmtMs(result.totalLatencyMs)}</strong>.
1840
- </p>
1841
- </AccordionContent>
1842
- </AccordionItem>
1843
-
1844
- <AccordionItem value="cost">
1845
- <AccordionTrigger className="text-sm">
1846
- Cost per million output tokens
1847
- </AccordionTrigger>
1848
- <AccordionContent className="text-xs space-y-2">
1849
- <FormulaBox>
1850
- $/M = (gpu_$/hr × num_gpus / 3600) / aggregate_tokens_per_sec × 1,000,000
1851
- </FormulaBox>
1852
- <p className="text-muted-foreground">
1853
- Your numbers: {fmtMoney(result.costPerHour)}/hr ÷ {fmtTokens(result.aggregateTokensPerSec)} tok/s × 1M ={" "}
1854
- <strong className="text-emerald-500">{fmtMoney(result.costPerMillionOutputTokens)}/M tokens</strong>.
1855
- </p>
1856
- </AccordionContent>
1857
- </AccordionItem>
1858
-
1859
- <AccordionItem value="disclaimer">
1860
- <AccordionTrigger className="text-sm">
1861
- Caveats & real-world accuracy
1862
- </AccordionTrigger>
1863
- <AccordionContent className="text-xs space-y-2 text-muted-foreground">
1864
- <p>
1865
- These formulas give theoretical maxima. Real-world throughput is typically
1866
- <strong className="text-foreground"> 70–90% of the decode number</strong> due to:
1867
- </p>
1868
- <ul className="list-disc list-inside space-y-1 ml-2">
1869
- <li>Attention cost (grows quadratically with context length beyond ~4K)</li>
1870
- <li>Kernel launch overhead and Python GIL contention</li>
1871
- <li>Sampling / tokenization / detokenization</li>
1872
- <li>Sampling with rejection (in nucleus sampling)</li>
1873
- <li>Network I/O for multi-node or API-serving</li>
1874
- <li>Continuous batching in modern servers (vLLM, TGI) — can <em>exceed</em> these numbers</li>
1875
- </ul>
1876
- <p className="flex items-start gap-2 mt-2 pt-2 border-t border-border/60">
1877
- <Info className="size-3.5 shrink-0 mt-0.5 text-emerald-500" />
1878
- <span>
1879
- For benchmark-grade numbers, run <code>vllm --model &lt;model&gt; --gpu &lt;gpu&gt;</code>
1880
- {" "}and use <code>--max-num-batched-tokens</code> to find the sweet spot.
1881
- This tool is for first-pass planning.
1882
- </span>
1883
- </p>
1884
- </AccordionContent>
1885
- </AccordionItem>
1886
- </Accordion>
1887
- </CardContent>
1888
- </Card>
1889
- </div>
1890
-
1891
- {/* Tech badges */}
1892
- <div className="mt-8 flex flex-wrap items-center gap-2 text-[11px] text-muted-foreground">
1893
- <CheckCircle2 className="size-3.5 text-emerald-500" />
1894
- <span>Formulas verified against:</span>
1895
- {["vLLM docs", "NVIDIA cuBLAS benchmarks", "llama.cpp discussions", "GPU spec sheets", "artificialanalysis.ai"].map((s) => (
1896
- <Badge key={s} variant="outline" className="text-[10px] font-normal">
1897
- {s}
1898
- </Badge>
1899
- ))}
1900
- </div>
1901
- </div>
1902
- </main>
1903
-
1904
- {/* Footer (sticky) */}
1905
- <footer className="mt-auto border-t border-border/60 py-6">
1906
- <div className="max-w-7xl mx-auto px-4 sm:px-6 flex flex-col sm:flex-row items-center justify-between gap-3 text-xs text-muted-foreground">
1907
- <div className="flex items-center gap-2">
1908
- <span className="font-medium">tokcalc</span>
1909
- <span>·</span>
1910
- <span>open-source LLM throughput estimator</span>
1911
- </div>
1912
- <div className="flex items-center gap-3">
1913
- <span>built with Next.js 16 · Tailwind · Recharts</span>
1914
- <span>·</span>
1915
- <span>not affiliated with any GPU vendor</span>
1916
- </div>
1917
- </div>
1918
- </footer>
1919
- </div>
1920
- </TooltipProvider>
1921
- );
1922
- }
1923
-
1924
- /* ---------- small subcomponents ---------- */
1925
-
1926
- function HelpIcon({ term }: { term: GlossaryKey }) {
1927
- return (
1928
- <UITooltip>
1929
- <TooltipTrigger asChild>
1930
- <button
1931
- type="button"
1932
- aria-label={`What does ${term} mean?`}
1933
- className="inline-flex items-center justify-center text-muted-foreground hover:text-emerald-500 transition-colors"
1934
- >
1935
- <Info className="size-3.5" />
1936
- </button>
1937
- </TooltipTrigger>
1938
- <TooltipContent
1939
- side="top"
1940
- className="max-w-[260px] text-xs leading-relaxed"
1941
- >
1942
- {GLOSSARY[term]}
1943
- </TooltipContent>
1944
- </UITooltip>
1945
- );
1946
- }
1947
-
1948
- function Stat({ label, value, term }: { label: string; value: string; term?: GlossaryKey }) {
1949
- return (
1950
- <div className="flex flex-col gap-0.5 px-2 py-1.5 rounded-md bg-muted/40">
1951
- <span className="text-[10px] uppercase tracking-wider text-muted-foreground flex items-center gap-1">
1952
- {label}
1953
- {term && <HelpIcon term={term} />}
1954
- </span>
1955
- <span className="font-mono text-xs">{value}</span>
1956
- </div>
1957
- );
1958
- }
1959
-
1960
- function HeadlineCard({
1961
- icon,
1962
- label,
1963
- value,
1964
- sub,
1965
- highlight,
1966
- glossaryKey,
1967
- confidence,
1968
- }: {
1969
- icon: React.ReactNode;
1970
- label: string;
1971
- value: string;
1972
- sub?: string;
1973
- highlight?: boolean;
1974
- glossaryKey?: GlossaryKey;
1975
- confidence?: Confidence;
1976
- }) {
1977
- return (
1978
- <Card
1979
- className={`border-border/60 shadow-sm overflow-hidden ${
1980
- highlight ? "border-emerald-500/40 bg-emerald-500/5" : ""
1981
- }`}
1982
- >
1983
- <CardContent className="p-3 sm:p-4">
1984
- <div className="flex items-center gap-1.5 text-muted-foreground mb-1.5">
1985
- <span className={highlight ? "text-emerald-500" : ""}>{icon}</span>
1986
- <span className="text-[11px] font-medium">{label}</span>
1987
- {glossaryKey && <HelpIcon term={glossaryKey} />}
1988
- {confidence && <ConfidenceDot confidence={confidence} />}
1989
- </div>
1990
- <div
1991
- className={`text-2xl sm:text-3xl font-bold font-mono tabular-nums ${
1992
- highlight ? "text-emerald-500" : ""
1993
- }`}
1994
- >
1995
- {value}
1996
- </div>
1997
- {sub && (
1998
- <div className="text-[10px] text-muted-foreground mt-0.5">{sub}</div>
1999
- )}
2000
- </CardContent>
2001
- </Card>
2002
- );
2003
- }
2004
-
2005
- function Metric({
2006
- label,
2007
- value,
2008
- hint,
2009
- warn,
2010
- glossaryKey,
2011
- confidence,
2012
- }: {
2013
- label: string;
2014
- value: string;
2015
- hint?: string;
2016
- warn?: boolean;
2017
- glossaryKey?: GlossaryKey;
2018
- confidence?: Confidence;
2019
- }) {
2020
- return (
2021
- <div className="space-y-0.5">
2022
- <div className="text-[10px] uppercase tracking-wider text-muted-foreground flex items-center gap-1">
2023
- {label}
2024
- {glossaryKey && <HelpIcon term={glossaryKey} />}
2025
- {confidence && <ConfidenceBadge confidence={confidence} />}
2026
- </div>
2027
- <div className={`font-mono text-sm font-medium ${warn ? "text-amber-500" : ""}`}>{value}</div>
2028
- {hint && <div className="text-[10px] text-muted-foreground">{hint}</div>}
2029
- </div>
2030
- );
2031
- }
2032
-
2033
- function FormulaBox({ children }: { children: React.ReactNode }) {
2034
- return (
2035
- <div className="bg-muted/40 border border-border/60 rounded-md px-3 py-2 font-mono text-[11px] text-foreground">
2036
- {children}
2037
- </div>
2038
- );
2039
- }
2040
-
2041
- /* ============================================================
2042
- BUILD-VS-BUY TAB (independent state)
2043
- ============================================================ */
2044
- function BuildVsBuyTab() {
2045
- // Self-host config
2046
- const [shModel, setShModel] = useState("llama3-8b");
2047
- const [shGpu, setShGpu] = useState("a100-80");
2048
- const [shQuant, setShQuant] = useState<Quantization>("fp16");
2049
- const [shNumGpus, setShNumGpus] = useState(1);
2050
- const [shGpuPrice, setShGpuPrice] = useState<number | "">("");
2051
- const [utilization, setUtilization] = useState(50); // %
2052
- const [batchSize, setBatchSize] = useState(8);
2053
-
2054
- // Workload
2055
- const [inputTokens, setInputTokens] = useState(500);
2056
- const [outputTokens, setOutputTokens] = useState(200);
2057
- const [reqsPerDay, setReqsPerDay] = useState(1000);
2058
-
2059
- // API config
2060
- const [apiProvider, setApiProvider] = useState<"openai" | "anthropic" | "google" | "groq" | "deepseek" | "mistral" | "together">("openai");
2061
- const [apiModel, setApiModel] = useState("gpt-4o-mini");
2062
-
2063
- // === URL-share state ===
2064
- const [urlRestored, setUrlRestored] = useState(false);
2065
-
2066
- // One-time mount: restore state from URL hash (priority: URL > localStorage > defaults)
2067
- useEffect(() => {
2068
- /* eslint-disable react-hooks/set-state-in-effect */
2069
- if (typeof window === "undefined") return;
2070
- const hash = window.location.hash;
2071
- if (!hash || hash === "#") {
2072
- // No URL hash — try localStorage
2073
- const stored = loadBvbFromStorage();
2074
- if (stored) {
2075
- if (stored.shModel) setShModel(stored.shModel);
2076
- if (stored.shGpu) setShGpu(stored.shGpu);
2077
- if (stored.shQuant) setShQuant(stored.shQuant as Quantization);
2078
- if (typeof stored.shNumGpus === "number") setShNumGpus(stored.shNumGpus);
2079
- if (stored.shGpuPrice !== undefined) setShGpuPrice(stored.shGpuPrice);
2080
- if (typeof stored.utilization === "number") setUtilization(stored.utilization);
2081
- if (typeof stored.batchSize === "number") setBatchSize(stored.batchSize);
2082
- if (typeof stored.inputTokens === "number") setInputTokens(stored.inputTokens);
2083
- if (typeof stored.outputTokens === "number") setOutputTokens(stored.outputTokens);
2084
- if (typeof stored.reqsPerDay === "number") setReqsPerDay(stored.reqsPerDay);
2085
- if (stored.apiProvider) setApiProvider(stored.apiProvider as typeof apiProvider);
2086
- if (stored.apiModel) setApiModel(stored.apiModel);
2087
- }
2088
- setUrlRestored(true);
2089
- return;
2090
- }
2091
- const parsed = parseUrlHash(hash);
2092
- if (parsed.tab !== "build-vs-buy" || !parsed.bvb) {
2093
- setUrlRestored(true);
2094
- return;
2095
- }
2096
- const b = parsed.bvb;
2097
- if (b.shModel) setShModel(b.shModel);
2098
- if (b.shGpu) setShGpu(b.shGpu);
2099
- if (b.shQuant) setShQuant(b.shQuant as Quantization);
2100
- if (typeof b.shNumGpus === "number") setShNumGpus(b.shNumGpus);
2101
- if (b.shGpuPrice !== undefined) setShGpuPrice(b.shGpuPrice);
2102
- if (typeof b.utilization === "number") setUtilization(b.utilization);
2103
- if (typeof b.batchSize === "number") setBatchSize(b.batchSize);
2104
- if (typeof b.inputTokens === "number") setInputTokens(b.inputTokens);
2105
- if (typeof b.outputTokens === "number") setOutputTokens(b.outputTokens);
2106
- if (typeof b.reqsPerDay === "number") setReqsPerDay(b.reqsPerDay);
2107
- if (b.apiProvider) setApiProvider(b.apiProvider as typeof apiProvider);
2108
- if (b.apiModel) setApiModel(b.apiModel);
2109
- setUrlRestored(true);
2110
- /* eslint-enable react-hooks/set-state-in-effect */
2111
- }, []);
2112
-
2113
- // Continuous: when state changes, update URL hash + localStorage
2114
- useEffect(() => {
2115
- if (!urlRestored) return;
2116
- const state = {
2117
- shModel, shGpu, shQuant, shNumGpus, shGpuPrice,
2118
- utilization, batchSize, inputTokens, outputTokens, reqsPerDay,
2119
- apiProvider, apiModel,
2120
- };
2121
- writeUrlHash(serializeBvbState(state));
2122
- saveBvbToStorage(state);
2123
- }, [
2124
- urlRestored, shModel, shGpu, shQuant, shNumGpus, shGpuPrice,
2125
- utilization, batchSize, inputTokens, outputTokens, reqsPerDay,
2126
- apiProvider, apiModel,
2127
- ]);
2128
-
2129
- const shGpuSpec = GPUS.find((g) => g.id === shGpu)!;
2130
- const shModelSpec = MODELS.find((m) => m.id === shModel)!;
2131
- const shQuantSpec = QUANTIZATIONS.find((q) => q.id === shQuant)!;
2132
-
2133
- // Run calc on self-host
2134
- const sh = useMemo(() => calculate({
2135
- modelId: shModel,
2136
- gpuId: shGpu,
2137
- quantization: shQuant,
2138
- numGpus: shNumGpus,
2139
- batchSize,
2140
- promptTokens: inputTokens,
2141
- outputTokens,
2142
- gpuHourlyCost: shGpuPrice === "" ? undefined : Number(shGpuPrice),
2143
- }), [shModel, shGpu, shQuant, shNumGpus, batchSize, inputTokens, outputTokens, shGpuPrice]);
2144
-
2145
- // Self-host cost per 1M tokens, accounting for utilization
2146
- const effGpuPrice = (shGpuPrice === "" ? (shGpuSpec.usdPerHour ?? 0) : Number(shGpuPrice)) * shNumGpus;
2147
- const effAggregateTokens = sh.aggregateTokensPerSec * (utilization / 100);
2148
- const selfHostCostPerM = effAggregateTokens > 0
2149
- ? (effGpuPrice / 3600 / effAggregateTokens) * 1e6
2150
- : Infinity;
2151
- const selfHostMonthly = effGpuPrice * 730; // 730 hours/month
2152
- const tokensPerDay = effAggregateTokens * 3600 * (utilization / 100) * 24;
2153
-
2154
- // API pricing (cited 2025-2026 from research)
2155
- const API_PRICES: Record<string, { in: number; out: number; cached: number | null; status: string }> = {
2156
- "gpt-4o": { in: 2.50, out: 10.00, cached: 1.25, status: "current" },
2157
- "gpt-4o-mini": { in: 0.15, out: 0.60, cached: 0.075, status: "current" },
2158
- "o1": { in: 15.00, out: 60.00, cached: 7.50, status: "current" },
2159
- "o3-mini": { in: 1.10, out: 4.40, cached: 0.55, status: "current" },
2160
- "claude-3.5-sonnet": { in: 3.00, out: 15.00, cached: 0.30, status: "retired Oct 2025" },
2161
- "claude-3.5-haiku": { in: 0.80, out: 4.00, cached: 0.08, status: "retired Feb 2026" },
2162
- "gemini-2.0-flash": { in: 0.10, out: 0.40, cached: null, status: "shut down Jun 2026" },
2163
- "gemini-1.5-flash": { in: 0.075, out: 0.30, cached: 0.01875, status: "shut down Sep 2025" },
2164
- "llama-3.1-70b-together": { in: 0.88, out: 0.88, cached: null, status: "current" },
2165
- "llama-3.3-70b-groq": { in: 0.59, out: 0.79, cached: null, status: "current" },
2166
- "deepseek-v4.1-flash": { in: 0.15, out: 0.60, cached: 0.003, status: "current" },
2167
- "mistral-large-3": { in: 0.50, out: 1.50, cached: 0.05, status: "current" },
2168
- "codestral": { in: 0.30, out: 0.90, cached: 0.03, status: "current" },
2169
- };
2170
-
2171
- const apiPricing = API_PRICES[apiModel];
2172
- // Per-request API cost: input tokens + output tokens (no caching assumed in this basic version)
2173
- const apiCostPerRequest =
2174
- (inputTokens / 1e6) * apiPricing.in + (outputTokens / 1e6) * apiPricing.out;
2175
- const apiCostPerDay = apiCostPerRequest * reqsPerDay;
2176
- const apiMonthlyCost = apiCostPerDay * 30;
2177
- const apiCostPerMOut = apiPricing.out; // $/M output tokens (at face value)
2178
-
2179
- // Break-even: when monthly self-host = monthly API
2180
- // self_host_monthly = api_monthly → reqs_per_day_break_even = self_host_monthly / (30 * api_cost_per_request)
2181
- const breakEvenReqsPerDay = apiCostPerRequest > 0
2182
- ? selfHostMonthly / (30 * apiCostPerRequest)
2183
- : Infinity;
2184
-
2185
- // Verdict
2186
- const cheaperThanApi = selfHostCostPerM < apiCostPerMOut;
2187
- const meetsVolume = reqsPerDay > breakEvenReqsPerDay;
2188
-
2189
- return (
2190
- <div className="space-y-4">
2191
- <div className="mb-4 max-w-3xl">
2192
- <h2 className="text-xl font-semibold tracking-tight mb-2">Should you self-host or use an API?</h2>
2193
- <p className="text-sm text-muted-foreground leading-relaxed">
2194
- Tell us your workload and compare the cost of running on your own GPU vs paying per-token to an API.
2195
- The break-even point tells you when self-hosting becomes cheaper.
2196
- </p>
2197
- </div>
2198
-
2199
- <div className="grid grid-cols-1 lg:grid-cols-2 gap-4">
2200
- {/* Self-host side */}
2201
- <Card className="border-border/60 shadow-sm">
2202
- <CardHeader className="pb-3">
2203
- <CardTitle className="text-sm flex items-center gap-2">
2204
- <Cpu className="size-4 text-emerald-500" />
2205
- Self-host (rent GPUs)
2206
- </CardTitle>
2207
- </CardHeader>
2208
- <CardContent className="space-y-3">
2209
- <div>
2210
- <Label className="text-xs">Model</Label>
2211
- <Select value={shModel} onValueChange={setShModel}>
2212
- <SelectTrigger className="w-full mt-1"><SelectValue /></SelectTrigger>
2213
- <SelectContent className="max-h-80">
2214
- {MODELS.filter(m => m.category === "text" || m.category === "code" || m.category === "reasoning").map(m => (
2215
- <SelectItem key={m.id} value={m.id}>{m.name}</SelectItem>
2216
- ))}
2217
- </SelectContent>
2218
- </Select>
2219
- </div>
2220
- <div>
2221
- <Label className="text-xs">GPU</Label>
2222
- <Select value={shGpu} onValueChange={setShGpu}>
2223
- <SelectTrigger className="w-full mt-1"><SelectValue /></SelectTrigger>
2224
- <SelectContent className="max-h-80">
2225
- {(["datacenter", "workstation", "consumer", "tpu", "lpu", "wse"] as GpuCategory[]).map(cat => {
2226
- const items = GPUS.filter(g => g.category === cat);
2227
- if (items.length === 0) return null;
2228
- return (
2229
- <SelectGroup key={cat}>
2230
- <SelectLabel className="text-[10px] uppercase text-muted-foreground">{cat}</SelectLabel>
2231
- {items.map(g => <SelectItem key={g.id} value={g.id}>{g.name}</SelectItem>)}
2232
- </SelectGroup>
2233
- );
2234
- })}
2235
- </SelectContent>
2236
- </Select>
2237
- </div>
2238
- <div>
2239
- <Label className="text-xs">Quantization</Label>
2240
- <Select value={shQuant} onValueChange={(v) => setShQuant(v as Quantization)}>
2241
- <SelectTrigger className="w-full mt-1"><SelectValue /></SelectTrigger>
2242
- <SelectContent className="max-h-80">
2243
- {QUANTIZATIONS.map(q => <SelectItem key={q.id} value={q.id}>{q.label}</SelectItem>)}
2244
- </SelectContent>
2245
- </Select>
2246
- </div>
2247
- <div>
2248
- <div className="flex items-center justify-between mb-1.5">
2249
- <Label className="text-xs">GPUs</Label>
2250
- <Badge variant="outline" className="text-[10px] font-mono">{shNumGpus}×</Badge>
2251
- </div>
2252
- <Slider value={[shNumGpus]} min={1} max={8} step={1} onValueChange={(v) => setShNumGpus(v[0])} className="w-full" />
2253
- </div>
2254
- <div>
2255
- <div className="flex items-center justify-between mb-1.5">
2256
- <Label className="text-xs">GPU utilization</Label>
2257
- <Badge variant="outline" className="text-[10px] font-mono">{utilization}%</Badge>
2258
- </div>
2259
- <Slider value={[utilization]} min={5} max={100} step={5} onValueChange={(v) => setUtilization(v[0])} className="w-full" />
2260
- <p className="text-[10px] text-muted-foreground mt-1">Effective utilization is the decisive factor — not peak throughput.</p>
2261
- </div>
2262
- <div>
2263
- <div className="flex items-center justify-between mb-1.5">
2264
- <Label className="text-xs">Batch size</Label>
2265
- <Badge variant="outline" className="text-[10px] font-mono">{batchSize}</Badge>
2266
- </div>
2267
- <Slider value={[batchSize]} min={1} max={64} step={1} onValueChange={(v) => setBatchSize(v[0])} className="w-full" />
2268
- </div>
2269
- <div>
2270
- <Label className="text-xs">GPU $/hr (blank = default ${(shGpuSpec.usdPerHour ?? "—")})</Label>
2271
- <Input type="number" step="0.01" value={shGpuPrice} onChange={e => setShGpuPrice(e.target.value === "" ? "" : Number(e.target.value))} className="mt-1" />
2272
- </div>
2273
- </CardContent>
2274
- </Card>
2275
-
2276
- {/* Workload + API side */}
2277
- <Card className="border-border/60 shadow-sm">
2278
- <CardHeader className="pb-3">
2279
- <CardTitle className="text-sm flex items-center gap-2">
2280
- <DollarSign className="size-4 text-emerald-500" />
2281
- API alternative
2282
- </CardTitle>
2283
- </CardHeader>
2284
- <CardContent className="space-y-3">
2285
- <div className="grid grid-cols-2 gap-3">
2286
- <div>
2287
- <Label className="text-xs">Input tokens/req</Label>
2288
- <Input type="number" value={inputTokens} onChange={e => setInputTokens(Math.max(0, Number(e.target.value)))} className="mt-1" />
2289
- </div>
2290
- <div>
2291
- <Label className="text-xs">Output tokens/req</Label>
2292
- <Input type="number" value={outputTokens} onChange={e => setOutputTokens(Math.max(0, Number(e.target.value)))} className="mt-1" />
2293
- </div>
2294
- </div>
2295
- <div>
2296
- <Label className="text-xs">Requests per day</Label>
2297
- <Input type="number" value={reqsPerDay} onChange={e => setReqsPerDay(Math.max(0, Number(e.target.value)))} className="mt-1" />
2298
- </div>
2299
- <Separator />
2300
- <div>
2301
- <Label className="text-xs">API provider</Label>
2302
- <Select value={apiProvider} onValueChange={(v) => {
2303
- setApiProvider(v as typeof apiProvider);
2304
- // Set default model per provider
2305
- const defaults: Record<string, string> = {
2306
- openai: "gpt-4o-mini",
2307
- anthropic: "claude-3.5-sonnet",
2308
- google: "gemini-2.0-flash",
2309
- groq: "llama-3.3-70b-groq",
2310
- deepseek: "deepseek-v4.1-flash",
2311
- mistral: "mistral-large-3",
2312
- together: "llama-3.1-70b-together",
2313
- };
2314
- setApiModel(defaults[v]);
2315
- }}>
2316
- <SelectTrigger className="w-full mt-1"><SelectValue /></SelectTrigger>
2317
- <SelectContent>
2318
- <SelectItem value="openai">OpenAI</SelectItem>
2319
- <SelectItem value="anthropic">Anthropic</SelectItem>
2320
- <SelectItem value="google">Google Gemini</SelectItem>
2321
- <SelectItem value="groq">Groq</SelectItem>
2322
- <SelectItem value="deepseek">DeepSeek</SelectItem>
2323
- <SelectItem value="mistral">Mistral</SelectItem>
2324
- <SelectItem value="together">Together AI</SelectItem>
2325
- </SelectContent>
2326
- </Select>
2327
- </div>
2328
- <div>
2329
- <Label className="text-xs">Model</Label>
2330
- <Select value={apiModel} onValueChange={setApiModel}>
2331
- <SelectTrigger className="w-full mt-1"><SelectValue /></SelectTrigger>
2332
- <SelectContent>
2333
- {Object.entries(API_PRICES).map(([key, p]) => (
2334
- <SelectItem key={key} value={key}>
2335
- {key} — ${p.in}/M in, ${p.out}/M out
2336
- </SelectItem>
2337
- ))}
2338
- </SelectContent>
2339
- </Select>
2340
- <p className="text-[10px] text-amber-600 dark:text-amber-400 mt-1">
2341
- Status: {apiPricing.status}
2342
- </p>
2343
- </div>
2344
- </CardContent>
2345
- </Card>
2346
- </div>
2347
-
2348
- {/* Verdict */}
2349
- <Card className={`border-2 shadow-sm ${cheaperThanApi && meetsVolume ? "border-emerald-500" : "border-amber-500/40"}`}>
2350
- <CardHeader>
2351
- <CardTitle className="text-base flex items-center gap-2">
2352
- {cheaperThanApi && meetsVolume ? (
2353
- <CheckCircle2 className="size-4 text-emerald-500" />
2354
- ) : (
2355
- <AlertTriangle className="size-4 text-amber-500" />
2356
- )}
2357
- Verdict
2358
- </CardTitle>
2359
- </CardHeader>
2360
- <CardContent>
2361
- {cheaperThanApi && meetsVolume ? (
2362
- <p className="text-sm">
2363
- <strong className="text-emerald-500">Self-hosting is cheaper</strong> at your workload
2364
- ({reqsPerDay.toLocaleString()} req/day). You pay{" "}
2365
- <strong className="text-foreground">{fmtMoney(selfHostCostPerM)}/M output tokens</strong> vs the API&apos;s{" "}
2366
- <strong className="text-foreground">{fmtMoney(apiCostPerMOut)}/M</strong>. Break-even was at{" "}
2367
- {Math.round(breakEvenReqsPerDay).toLocaleString()} req/day.
2368
- </p>
2369
- ) : meetsVolume ? (
2370
- <p className="text-sm">
2371
- <strong className="text-amber-500">API is cheaper at this volume.</strong> You&apos;d pay{" "}
2372
- <strong className="text-foreground">{fmtMoney(selfHostCostPerM)}/M</strong> self-hosting vs{" "}
2373
- <strong className="text-foreground">{fmtMoney(apiCostPerMOut)}/M</strong> via API.
2374
- Self-host becomes cheaper only above {Math.round(breakEvenReqsPerDay).toLocaleString()} req/day
2375
- (you&apos;re at {reqsPerDay.toLocaleString()}).
2376
- </p>
2377
- ) : (
2378
- <p className="text-sm">
2379
- <strong className="text-amber-500">Not enough volume to justify self-hosting.</strong>{" "}
2380
- You&apos;d need {Math.round(breakEvenReqsPerDay).toLocaleString()} req/day to break even;
2381
- you&apos;re at {reqsPerDay.toLocaleString()}. Stick with the API.
2382
- </p>
2383
- )}
2384
- <div className="mt-4 grid grid-cols-2 sm:grid-cols-4 gap-3 text-sm">
2385
- <Metric label="Self-host $/M tokens" value={fmtMoney(selfHostCostPerM)} hint={`at ${utilization}% util`} />
2386
- <Metric label="API $/M output" value={fmtMoney(apiCostPerMOut)} hint={apiModel} />
2387
- <Metric label="Self-host monthly" value={fmtMoney(selfHostMonthly)} hint="730 hrs × $/hr" />
2388
- <Metric label="API monthly" value={fmtMoney(apiMonthlyCost)} hint={`${reqsPerDay.toLocaleString()} req/day × 30`} />
2389
- </div>
2390
- </CardContent>
2391
- </Card>
2392
- <p className="text-[10px] text-muted-foreground leading-relaxed">
2393
- Pricing is timestamped to research brief (Sept 2026). Providers retire models — Claude 3.5, Gemini 1.5/2.0, Mistral Large 2 etc. are already retired.
2394
- Always verify at the provider&apos;s official pricing page before making purchase decisions.
2395
- </p>
2396
- </div>
2397
- );
2398
- }
2399
-
2400
- /* ============================================================
2401
- REFERENCE TAB — catalog + pricing tables
2402
- ============================================================ */
2403
- function ReferenceTab() {
2404
- const [subview, setSubview] = useState<"models" | "gpus" | "quants" | "api" | "cloud">("models");
2405
-
2406
- return (
2407
- <div className="space-y-4">
2408
- <div className="mb-2 max-w-3xl">
2409
- <h2 className="text-xl font-semibold tracking-tight mb-1">Reference catalog</h2>
2410
- <p className="text-sm text-muted-foreground leading-relaxed">
2411
- All models, GPUs, quantizations, and live API/cloud pricing used by tokcalc.
2412
- Every record has a source link where available.
2413
- </p>
2414
- </div>
2415
-
2416
- {/* Sub-tabs */}
2417
- <div className="border-b border-border/60">
2418
- <div className="flex gap-1 -mb-px overflow-x-auto">
2419
- {([
2420
- { id: "models", label: "Models" },
2421
- { id: "gpus", label: "GPUs" },
2422
- { id: "quants", label: "Quantization" },
2423
- { id: "api", label: "API pricing" },
2424
- { id: "cloud", label: "Cloud GPU pricing" },
2425
- ] as const).map(t => (
2426
- <button
2427
- key={t.id}
2428
- onClick={() => setSubview(t.id)}
2429
- className={`px-3 py-2 text-xs font-medium border-b-2 transition-colors whitespace-nowrap ${
2430
- subview === t.id
2431
- ? "border-emerald-500 text-foreground"
2432
- : "border-transparent text-muted-foreground hover:text-foreground"
2433
- }`}
2434
- >
2435
- {t.label}
2436
- </button>
2437
- ))}
2438
- </div>
2439
- </div>
2440
-
2441
- {subview === "models" && (
2442
- <Card>
2443
- <CardContent className="p-0">
2444
- <div className="overflow-x-auto">
2445
- <table className="w-full text-xs">
2446
- <thead className="bg-muted/40 text-[10px] uppercase">
2447
- <tr>
2448
- <th className="text-left px-3 py-2">Model</th>
2449
- <th className="text-left px-3 py-2">Family</th>
2450
- <th className="text-left px-3 py-2">Cat</th>
2451
- <th className="text-right px-3 py-2">Params (total)</th>
2452
- <th className="text-right px-3 py-2">Active</th>
2453
- <th className="text-right px-3 py-2">Layers</th>
2454
- <th className="text-right px-3 py-2">Hidden</th>
2455
- <th className="text-right px-3 py-2">Q/KV</th>
2456
- <th className="text-right px-3 py-2">Ctx</th>
2457
- <th className="text-center px-3 py-2">MoE</th>
2458
- </tr>
2459
- </thead>
2460
- <tbody>
2461
- {MODELS.map(m => (
2462
- <tr key={m.id} className="border-t border-border/40 hover:bg-muted/20">
2463
- <td className="px-3 py-1.5 font-medium">{m.name}</td>
2464
- <td className="px-3 py-1.5 text-muted-foreground">{m.family}</td>
2465
- <td className="px-3 py-1.5 text-muted-foreground">{m.category}</td>
2466
- <td className="px-3 py-1.5 text-right font-mono">{m.paramsB}B</td>
2467
- <td className="px-3 py-1.5 text-right font-mono">{m.activeParamsB}B</td>
2468
- <td className="px-3 py-1.5 text-right font-mono">{m.layers}</td>
2469
- <td className="px-3 py-1.5 text-right font-mono">{m.hiddenDim}</td>
2470
- <td className="px-3 py-1.5 text-right font-mono">{m.qHeads}/{m.kvHeads}</td>
2471
- <td className="px-3 py-1.5 text-right font-mono">{(m.maxContext / 1000).toFixed(0)}K</td>
2472
- <td className="px-3 py-1.5 text-center">{m.isMoE ? "✓" : ""}</td>
2473
- </tr>
2474
- ))}
2475
- </tbody>
2476
- </table>
2477
- </div>
2478
- </CardContent>
2479
- </Card>
2480
- )}
2481
-
2482
- {subview === "gpus" && (
2483
- <Card>
2484
- <CardContent className="p-0">
2485
- <div className="overflow-x-auto">
2486
- <table className="w-full text-xs">
2487
- <thead className="bg-muted/40 text-[10px] uppercase">
2488
- <tr>
2489
- <th className="text-left px-3 py-2">GPU</th>
2490
- <th className="text-left px-3 py-2">Vendor</th>
2491
- <th className="text-left px-3 py-2">Category</th>
2492
- <th className="text-right px-3 py-2">Mem BW</th>
2493
- <th className="text-right px-3 py-2">FP16 TF</th>
2494
- <th className="text-right px-3 py-2">VRAM</th>
2495
- <th className="text-right px-3 py-2">NVLink</th>
2496
- <th className="text-right px-3 py-2">$/hr</th>
2497
- <th className="text-right px-3 py-2">Year</th>
2498
- </tr>
2499
- </thead>
2500
- <tbody>
2501
- {GPUS.map(g => (
2502
- <tr key={g.id} className="border-t border-border/40 hover:bg-muted/20">
2503
- <td className="px-3 py-1.5 font-medium">
2504
- {g.name}
2505
- {g.note && <span className="ml-2 text-[10px] text-amber-600 dark:text-amber-400">⚠</span>}
2506
- </td>
2507
- <td className="px-3 py-1.5 text-muted-foreground">{g.vendor}</td>
2508
- <td className="px-3 py-1.5 text-muted-foreground">{g.category}</td>
2509
- <td className="px-3 py-1.5 text-right font-mono">{g.memBandwidthGbps}</td>
2510
- <td className="px-3 py-1.5 text-right font-mono">{g.flopsTflops ?? "—"}</td>
2511
- <td className="px-3 py-1.5 text-right font-mono">{g.vramGb}</td>
2512
- <td className="px-3 py-1.5 text-right font-mono">{g.nvlinkGbps || "—"}</td>
2513
- <td className="px-3 py-1.5 text-right font-mono">{g.usdPerHour ?? "quote"}</td>
2514
- <td className="px-3 py-1.5 text-right font-mono">{g.year}</td>
2515
- </tr>
2516
- ))}
2517
- </tbody>
2518
- </table>
2519
- </div>
2520
- {GPUS.some(g => g.note) && (
2521
- <div className="p-3 text-[10px] text-muted-foreground">
2522
- ⚠ = special note. Hover over the GPU name in the Calculator tab dropdown to see details.
2523
- </div>
2524
- )}
2525
- </CardContent>
2526
- </Card>
2527
- )}
2528
-
2529
- {subview === "quants" && (
2530
- <Card>
2531
- <CardContent className="p-0">
2532
- <div className="overflow-x-auto">
2533
- <table className="w-full text-xs">
2534
- <thead className="bg-muted/40 text-[10px] uppercase">
2535
- <tr>
2536
- <th className="text-left px-3 py-2">Format</th>
2537
- <th className="text-left px-3 py-2">Family</th>
2538
- <th className="text-right px-3 py-2">Bytes/param</th>
2539
- <th className="text-right px-3 py-2">Eff. bits</th>
2540
- <th className="text-right px-3 py-2">Efficiency</th>
2541
- <th className="text-left px-3 py-2">Use case</th>
2542
- <th className="text-left px-3 py-2">Description</th>
2543
- </tr>
2544
- </thead>
2545
- <tbody>
2546
- {QUANTIZATIONS.map(q => (
2547
- <tr key={q.id} className="border-t border-border/40 hover:bg-muted/20">
2548
- <td className="px-3 py-1.5 font-medium">{q.label}</td>
2549
- <td className="px-3 py-1.5 text-muted-foreground">{q.family}</td>
2550
- <td className="px-3 py-1.5 text-right font-mono">{q.bytesPerParam}</td>
2551
- <td className="px-3 py-1.5 text-right font-mono">{(q.bytesPerParam * 8).toFixed(2)}</td>
2552
- <td className="px-3 py-1.5 text-right font-mono">{q.efficiency.toFixed(2)}×</td>
2553
- <td className="px-3 py-1.5 text-muted-foreground text-[10px]">{q.useCase ?? "—"}</td>
2554
- <td className="px-3 py-1.5 text-muted-foreground text-[10px]">{q.description}</td>
2555
- </tr>
2556
- ))}
2557
- </tbody>
2558
- </table>
2559
- </div>
2560
- </CardContent>
2561
- </Card>
2562
- )}
2563
-
2564
- {subview === "api" && (
2565
- <Card>
2566
- <CardHeader>
2567
- <CardTitle className="text-sm">LLM API pricing ($/M tokens)</CardTitle>
2568
- <CardDescription className="text-xs">
2569
- Timestamped to research brief (Sept 2026). Verify at provider&apos;s official pricing page before purchase.
2570
- </CardDescription>
2571
- </CardHeader>
2572
- <CardContent className="p-0">
2573
- <div className="overflow-x-auto">
2574
- <table className="w-full text-xs">
2575
- <thead className="bg-muted/40 text-[10px] uppercase">
2576
- <tr>
2577
- <th className="text-left px-3 py-2">Model</th>
2578
- <th className="text-right px-3 py-2">Input $/M</th>
2579
- <th className="text-right px-3 py-2">Cached $/M</th>
2580
- <th className="text-right px-3 py-2">Output $/M</th>
2581
- <th className="text-left px-3 py-2">Status</th>
2582
- </tr>
2583
- </thead>
2584
- <tbody>
2585
- {[
2586
- ["gpt-4o", 2.50, 1.25, 10.00, "current"],
2587
- ["gpt-4o-mini", 0.15, 0.075, 0.60, "current"],
2588
- ["o1", 15.00, 7.50, 60.00, "current"],
2589
- ["o3-mini", 1.10, 0.55, 4.40, "current"],
2590
- ["claude-3.5-sonnet", 3.00, 0.30, 15.00, "retired Oct 2025"],
2591
- ["claude-3.5-haiku", 0.80, 0.08, 4.00, "retired Feb 2026"],
2592
- ["gemini-2.0-flash", 0.10, null, 0.40, "shut down Jun 2026"],
2593
- ["gemini-1.5-flash", 0.075, 0.01875, 0.30, "shut down Sep 2025"],
2594
- ["deepseek-v4.1-flash", 0.15, 0.003, 0.60, "current"],
2595
- ["mistral-large-3", 0.50, 0.05, 1.50, "current"],
2596
- ["codestral", 0.30, 0.03, 0.90, "current"],
2597
- ["llama-3.1-70b-together", 0.88, null, 0.88, "current"],
2598
- ["llama-3.3-70b-groq", 0.59, null, 0.79, "current"],
2599
- ].map(([m, i, c, o, s]) => (
2600
- <tr key={m as string} className="border-t border-border/40 hover:bg-muted/20">
2601
- <td className="px-3 py-1.5 font-medium">{m}</td>
2602
- <td className="px-3 py-1.5 text-right font-mono">${i}</td>
2603
- <td className="px-3 py-1.5 text-right font-mono">{c === null ? "—" : `$${c}`}</td>
2604
- <td className="px-3 py-1.5 text-right font-mono">${o}</td>
2605
- <td className="px-3 py-1.5 text-muted-foreground text-[10px]">{s}</td>
2606
- </tr>
2607
- ))}
2608
- </tbody>
2609
- </table>
2610
- </div>
2611
- </CardContent>
2612
- </Card>
2613
- )}
2614
-
2615
- {subview === "cloud" && (
2616
- <>
2617
- {/* Unified live comparison across all providers */}
2618
- <LivePricingComparison />
2619
-
2620
- {/* Detail: Azure live prices */}
2621
- <AzureLivePricing />
2622
-
2623
- {/* Detail: Vast.ai marketplace spot prices */}
2624
- <VastAiLivePricing />
2625
-
2626
- {/* Static estimates from tokcalc catalog */}
2627
- <Card>
2628
- <CardHeader>
2629
- <CardTitle className="text-sm">Static estimates (from research catalog)</CardTitle>
2630
- <CardDescription className="text-xs">
2631
- Approximate on-demand rates from research brief. Prices vary by region, commitment, and availability.
2632
- Live Azure prices shown above.
2633
- </CardDescription>
2634
- </CardHeader>
2635
- <CardContent className="p-0">
2636
- <div className="overflow-x-auto">
2637
- <table className="w-full text-xs">
2638
- <thead className="bg-muted/40 text-[10px] uppercase">
2639
- <tr>
2640
- <th className="text-left px-3 py-2">GPU</th>
2641
- <th className="text-left px-3 py-2">Vendor</th>
2642
- <th className="text-right px-3 py-2">$/hr (default)</th>
2643
- <th className="text-left px-3 py-2">Typical providers</th>
2644
- </tr>
2645
- </thead>
2646
- <tbody>
2647
- {GPUS.filter(g => g.usdPerHour !== null && g.usdPerHour > 0).map(g => (
2648
- <tr key={g.id} className="border-t border-border/40 hover:bg-muted/20">
2649
- <td className="px-3 py-1.5 font-medium">{g.name}</td>
2650
- <td className="px-3 py-1.5 text-muted-foreground">{g.vendor}</td>
2651
- <td className="px-3 py-1.5 text-right font-mono">${g.usdPerHour}</td>
2652
- <td className="px-3 py-1.5 text-muted-foreground text-[10px]">
2653
- {g.category === "datacenter" && "RunPod · Lambda · CoreWeave · AWS · GCP"}
2654
- {g.category === "workstation" && "TensorDock · RunPod"}
2655
- {g.category === "consumer" && "Vast.ai · TensorDock (spot)"}
2656
- {g.category === "tpu" && "Google Cloud TPU"}
2657
- {g.category === "legacy" && "Secondary market"}
2658
- </td>
2659
- </tr>
2660
- ))}
2661
- </tbody>
2662
- </table>
2663
- </div>
2664
- </CardContent>
2665
- </Card>
2666
- </>
2667
- )}
2668
- </div>
2669
- );
2670
- }