@tokcalc/mcp-server 0.1.3 → 0.2.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +86 -406
  2. package/dist/http.js +23504 -0
  3. package/dist/index.js +21290 -0
  4. package/package.json +36 -92
  5. package/.zscripts/build.sh +0 -175
  6. package/.zscripts/database-runtime-build.sh +0 -33
  7. package/.zscripts/dev.pid +0 -1
  8. package/.zscripts/dev.sh +0 -154
  9. package/.zscripts/mini-services-build.sh +0 -78
  10. package/.zscripts/mini-services-install.sh +0 -65
  11. package/.zscripts/mini-services-start.sh +0 -123
  12. package/.zscripts/python-runtime-build.sh +0 -120
  13. package/.zscripts/start.sh +0 -145
  14. package/CAPACITY_STUDY.md +0 -283
  15. package/CODE_OF_CONDUCT.md +0 -55
  16. package/CONTRIBUTING.md +0 -177
  17. package/Caddyfile +0 -23
  18. package/LICENSE +0 -204
  19. package/bun.lock +0 -1965
  20. package/components.json +0 -21
  21. package/db/custom.db +0 -0
  22. package/download/README.md +0 -1
  23. package/download/tokcalc-dark-calculator.png +0 -0
  24. package/download/tokcalc-dark-default.png +0 -0
  25. package/download/tokcalc-demo.webm +0 -0
  26. package/download/tokcalc-github-link.png +0 -0
  27. package/download/tokcalc-hydration-fixed.png +0 -0
  28. package/download/tokcalc-issue-resolved.png +0 -0
  29. package/download/tokcalc-light-mode.png +0 -0
  30. package/download/tokcalc-light-reference.png +0 -0
  31. package/download/tokcalc-long-context-qwen.png +0 -0
  32. package/download/tokcalc-long-context.png +0 -0
  33. package/download/tokcalc-og-image-preview.png +0 -0
  34. package/download/tokcalc-phase2-3.png +0 -0
  35. package/download/tokcalc-plain-english.png +0 -0
  36. package/download/tokcalc-preview.png +0 -0
  37. package/download/tokcalc-share-bvb.png +0 -0
  38. package/download/tokcalc-share-feature.png +0 -0
  39. package/download/tokcalc-tab-build-vs-buy.png +0 -0
  40. package/download/tokcalc-tab-calculator.png +0 -0
  41. package/download/tokcalc-tab-reference.png +0 -0
  42. package/eslint.config.mjs +0 -50
  43. package/examples/websocket/frontend.tsx +0 -196
  44. package/examples/websocket/server.ts +0 -138
  45. package/mini-services/.gitkeep +0 -0
  46. package/mini-services/mcp-server/README.md +0 -86
  47. package/mini-services/mcp-server/bun.lock +0 -202
  48. package/mini-services/mcp-server/index.ts +0 -504
  49. package/mini-services/mcp-server/package.json +0 -40
  50. package/next.config.ts +0 -12
  51. package/postcss.config.mjs +0 -5
  52. package/prisma/schema.prisma +0 -32
  53. package/public/google6f58ca6be85fa903.html +0 -1
  54. package/public/logo.svg +0 -29
  55. package/public/manifest.json +0 -51
  56. package/public/og-icon-256.png +0 -0
  57. package/public/og.png +0 -0
  58. package/public/robots.txt +0 -25
  59. package/public/sitemap.xml +0 -23
  60. package/public/tokcalc-demo.gif +0 -0
  61. package/scripts/og-template.html +0 -120
  62. package/scripts/render-og.mjs +0 -43
  63. package/server.json +0 -21
  64. package/src/app/api/pricing/aws/route.ts +0 -186
  65. package/src/app/api/pricing/azure/route.ts +0 -168
  66. package/src/app/api/pricing/gcp/route.ts +0 -230
  67. package/src/app/api/pricing/vast-ai/route.ts +0 -164
  68. package/src/app/api/route.ts +0 -5
  69. package/src/app/compare/h100-vs-h200/layout.tsx +0 -30
  70. package/src/app/compare/h100-vs-h200/page.tsx +0 -328
  71. package/src/app/globals.css +0 -122
  72. package/src/app/layout.tsx +0 -276
  73. package/src/app/page.tsx +0 -2670
  74. package/src/components/azure-live-pricing.tsx +0 -185
  75. package/src/components/benchmark-import.tsx +0 -340
  76. package/src/components/confidence-badge.tsx +0 -116
  77. package/src/components/live-pricing-comparison.tsx +0 -241
  78. package/src/components/theme-provider.tsx +0 -11
  79. package/src/components/theme-toggle.tsx +0 -55
  80. package/src/components/ui/accordion.tsx +0 -66
  81. package/src/components/ui/alert-dialog.tsx +0 -157
  82. package/src/components/ui/alert.tsx +0 -66
  83. package/src/components/ui/aspect-ratio.tsx +0 -11
  84. package/src/components/ui/avatar.tsx +0 -53
  85. package/src/components/ui/badge.tsx +0 -46
  86. package/src/components/ui/breadcrumb.tsx +0 -109
  87. package/src/components/ui/button.tsx +0 -59
  88. package/src/components/ui/calendar.tsx +0 -213
  89. package/src/components/ui/card.tsx +0 -92
  90. package/src/components/ui/carousel.tsx +0 -241
  91. package/src/components/ui/chart.tsx +0 -353
  92. package/src/components/ui/checkbox.tsx +0 -32
  93. package/src/components/ui/collapsible.tsx +0 -33
  94. package/src/components/ui/command.tsx +0 -184
  95. package/src/components/ui/context-menu.tsx +0 -252
  96. package/src/components/ui/dialog.tsx +0 -143
  97. package/src/components/ui/drawer.tsx +0 -135
  98. package/src/components/ui/dropdown-menu.tsx +0 -257
  99. package/src/components/ui/form.tsx +0 -167
  100. package/src/components/ui/hover-card.tsx +0 -44
  101. package/src/components/ui/input-otp.tsx +0 -77
  102. package/src/components/ui/input.tsx +0 -21
  103. package/src/components/ui/label.tsx +0 -24
  104. package/src/components/ui/menubar.tsx +0 -276
  105. package/src/components/ui/navigation-menu.tsx +0 -168
  106. package/src/components/ui/pagination.tsx +0 -127
  107. package/src/components/ui/popover.tsx +0 -48
  108. package/src/components/ui/progress.tsx +0 -31
  109. package/src/components/ui/radio-group.tsx +0 -45
  110. package/src/components/ui/resizable.tsx +0 -56
  111. package/src/components/ui/scroll-area.tsx +0 -58
  112. package/src/components/ui/select.tsx +0 -185
  113. package/src/components/ui/separator.tsx +0 -28
  114. package/src/components/ui/sheet.tsx +0 -139
  115. package/src/components/ui/sidebar.tsx +0 -726
  116. package/src/components/ui/skeleton.tsx +0 -13
  117. package/src/components/ui/slider.tsx +0 -63
  118. package/src/components/ui/sonner.tsx +0 -25
  119. package/src/components/ui/switch.tsx +0 -31
  120. package/src/components/ui/table.tsx +0 -116
  121. package/src/components/ui/tabs.tsx +0 -66
  122. package/src/components/ui/textarea.tsx +0 -18
  123. package/src/components/ui/toast.tsx +0 -129
  124. package/src/components/ui/toaster.tsx +0 -35
  125. package/src/components/ui/toggle-group.tsx +0 -73
  126. package/src/components/ui/toggle.tsx +0 -47
  127. package/src/components/ui/tooltip.tsx +0 -61
  128. package/src/components/vast-ai-live-pricing.tsx +0 -176
  129. package/src/hooks/use-mobile.ts +0 -19
  130. package/src/hooks/use-toast.ts +0 -194
  131. package/src/lib/benchmark-parser-sglang.ts +0 -150
  132. package/src/lib/benchmark-parser-tokcalc.ts +0 -247
  133. package/src/lib/benchmark-parser-trtllm.ts +0 -152
  134. package/src/lib/benchmark-parser-vllm.ts +0 -198
  135. package/src/lib/benchmark-schema.ts +0 -263
  136. package/src/lib/db.ts +0 -13
  137. package/src/lib/engine-presets.ts +0 -183
  138. package/src/lib/price-schema.ts +0 -141
  139. package/src/lib/token-calc.ts +0 -808
  140. package/src/lib/track.ts +0 -31
  141. package/src/lib/url-state.ts +0 -256
  142. package/src/lib/utils.ts +0 -6
  143. package/tailwind.config.ts +0 -64
  144. package/tests/database-runtime-build.sh +0 -75
  145. package/tests/python-runtime-build.sh +0 -64
  146. package/tests/python-runtime-container.sh +0 -31
  147. package/tool-results/bash_1789888171144_2c5381860539.txt +0 -161
  148. package/tool-results/bash_1789888175925_49c53ba3c61b.txt +0 -191
  149. package/tool-results/bash_1789888181202_49c53ba3c61b.txt +0 -191
  150. package/tool-results/bash_1789888195219_4a86a5c91411.txt +0 -200
  151. package/tool-results/bash_1789888203128_6cca13c71b47.txt +0 -199
  152. package/tool-results/bash_1789929256963_2a52aff0d0a8.txt +0 -160
  153. package/tool-results/read_1789888151021_69f58eec6a5b.txt +0 -653
  154. package/tool-results/read_1789888153837_1d3a8bfc2a94.txt +0 -653
  155. package/tool-results/read_1789888163087_ccc406d47505.txt +0 -122
  156. package/tool-results/read_1789888167347_67d1d7c9830a.txt +0 -122
  157. package/tool-results/read_1789929252529_d90e8f383a25.txt +0 -285
  158. package/tsconfig.json +0 -42
  159. package/upload/Pasted Content_1789887800864.txt +0 -652
  160. package/upload/Pasted Content_1789887909561.txt +0 -652
  161. package/upload/Pasted Content_1789887918428.txt +0 -652
  162. package/upload/Pasted Content_1789887959420.txt +0 -652
  163. package/upload/Pasted Content_1789888020485.txt +0 -652
  164. package/upload/Pasted Content_1789888058079.txt +0 -652
  165. package/upload/Pasted Content_1789888885033.txt +0 -686
  166. package/upload/Pasted Content_1789928912741.txt +0 -285
  167. package/upload/Pasted Content_1789928938402.txt +0 -285
  168. package/upload/Pasted Content_1789929160389.txt +0 -285
  169. package/upload/Pasted Content_1789929176660.txt +0 -285
  170. package/upload/issue_vision.json +0 -28
  171. package/upload/pasted_image_1789883175209.png +0 -0
  172. package/upload/pasted_image_1789899056690.png +0 -0
  173. package/upload/pasted_image_1789900371483.png +0 -0
  174. package/upload/pasted_image_1789900472823.png +0 -0
  175. package/upload/pasted_image_1789900490374.png +0 -0
  176. package/upload/pasted_image_1789900585552.png +0 -0
  177. package/upload/pasted_image_1789900606519.png +0 -0
  178. package/upload/pasted_image_1789901598705.png +0 -0
  179. package/upload/pasted_image_1789901613545.png +0 -0
  180. package/upload/pasted_image_1789978382674.png +0 -0
  181. package/upload/pasted_image_1789978392749.png +0 -0
  182. package/upload/pasted_image_1789978474879.png +0 -0
  183. package/upload/pasted_image_1789978523652.png +0 -0
  184. package/upload/pasted_image_1789984219089.png +0 -0
  185. package/upload/pasted_image_1789984491896.png +0 -0
  186. package/upload/pasted_image_1789985017950.png +0 -0
  187. package/upload/pasted_image_1789985036765.png +0 -0
  188. package/upload/pasted_image_1789985049848.png +0 -0
  189. package/upload/pasted_image_1790002427833.png +0 -0
  190. package/upload/pasted_image_1790002659944.png +0 -0
  191. package/upload/pasted_image_1790037038476.png +0 -0
  192. package/upload/screenshot_analysis.json +0 -28
  193. package/upload/vision_output.json +0 -28
@@ -1,185 +0,0 @@
1
- "use client";
2
-
3
- import { useEffect, useState } from "react";
4
- import {
5
- Card,
6
- CardContent,
7
- CardDescription,
8
- CardHeader,
9
- CardTitle,
10
- } from "@/components/ui/card";
11
- import { Button } from "@/components/ui/button";
12
- import { Badge } from "@/components/ui/badge";
13
- import { RefreshCw, Zap, AlertTriangle } from "lucide-react";
14
- import { track } from "@/lib/track";
15
-
16
- interface AzurePrice {
17
- gpu: string;
18
- sku: string;
19
- price: number;
20
- region: string;
21
- retrievedAt: string;
22
- }
23
-
24
- interface AzureApiResponse {
25
- source: string;
26
- cached: boolean;
27
- retrievedAt?: string;
28
- count?: number;
29
- prices?: AzurePrice[];
30
- error?: string;
31
- }
32
-
33
- export function AzureLivePricing() {
34
- const [data, setData] = useState<AzureApiResponse | null>(null);
35
- const [loading, setLoading] = useState(false);
36
- const [error, setError] = useState<string | null>(null);
37
-
38
- const fetchPrices = async () => {
39
- setLoading(true);
40
- setError(null);
41
- try {
42
- const res = await fetch("/api/pricing/azure");
43
- if (!res.ok) {
44
- const errData = await res.json().catch(() => ({}));
45
- throw new Error(errData.error || `HTTP ${res.status}`);
46
- }
47
- const json = (await res.json()) as AzureApiResponse;
48
- setData(json);
49
- track("fetched_live_pricing", { provider: "azure", count: json.count });
50
- } catch (e) {
51
- setError(e instanceof Error ? e.message : "Failed to fetch Azure pricing");
52
- } finally {
53
- setLoading(false);
54
- }
55
- };
56
-
57
- // Auto-fetch on mount
58
- useEffect(() => {
59
- fetchPrices();
60
- }, []);
61
-
62
- // Find cheapest price per GPU
63
- const cheapestPerGpu = (() => {
64
- if (!data?.prices) return [];
65
- const seen = new Map<string, AzurePrice>();
66
- for (const p of data.prices) {
67
- if (!seen.has(p.gpu) || seen.get(p.gpu)!.price > p.price) {
68
- seen.set(p.gpu, p);
69
- }
70
- }
71
- return Array.from(seen.values()).sort((a, b) => a.price - b.price);
72
- })();
73
-
74
- // Calculate "as of" label
75
- const asOfLabel = (() => {
76
- if (!data?.retrievedAt) return null;
77
- const ageMs = Date.now() - new Date(data.retrievedAt).getTime();
78
- if (ageMs < 60000) return "just now";
79
- if (ageMs < 3600000) return `${Math.floor(ageMs / 60000)}m ago`;
80
- return `${Math.floor(ageMs / 3600000)}h ago`;
81
- })();
82
-
83
- return (
84
- <Card className="border-emerald-500/30 shadow-sm">
85
- <CardHeader className="pb-3">
86
- <div className="flex items-center justify-between">
87
- <div>
88
- <CardTitle className="text-sm flex items-center gap-2">
89
- <Zap className="size-4 text-emerald-500" />
90
- Live Azure GPU pricing
91
- <Badge variant="outline" className="text-[9px] gap-1 ml-1">
92
- <span className="size-1.5 rounded-full bg-emerald-500 animate-pulse" />
93
- LIVE
94
- </Badge>
95
- </CardTitle>
96
- <CardDescription className="text-xs mt-1">
97
- Fetched from Azure Retail Prices API (public, unauthenticated).
98
- {asOfLabel && ` Last updated ${asOfLabel}.`}
99
- </CardDescription>
100
- </div>
101
- <Button
102
- variant="ghost"
103
- size="sm"
104
- onClick={fetchPrices}
105
- disabled={loading}
106
- className="gap-1.5 h-8"
107
- >
108
- <RefreshCw className={`size-3.5 ${loading ? "animate-spin" : ""}`} />
109
- <span className="text-xs">Refresh</span>
110
- </Button>
111
- </div>
112
- </CardHeader>
113
- <CardContent>
114
- {loading && !data && (
115
- <div className="flex items-center justify-center py-8 text-xs text-muted-foreground">
116
- <RefreshCw className="size-4 animate-spin mr-2" />
117
- Fetching live Azure prices...
118
- </div>
119
- )}
120
-
121
- {error && (
122
- <div className="flex items-start gap-2 p-3 rounded-md border border-amber-500/30 bg-amber-500/5 text-xs">
123
- <AlertTriangle className="size-3.5 text-amber-500 shrink-0 mt-0.5" />
124
- <div>
125
- <p className="font-medium text-amber-600 dark:text-amber-400">
126
- Couldn&apos;t fetch live prices
127
- </p>
128
- <p className="text-muted-foreground mt-0.5">{error}</p>
129
- <p className="text-muted-foreground mt-1">
130
- Static estimates from tokcalc&apos;s catalog are shown below.
131
- </p>
132
- </div>
133
- </div>
134
- )}
135
-
136
- {data?.error && (
137
- <div className="flex items-start gap-2 p-3 rounded-md border border-amber-500/30 bg-amber-500/5 text-xs">
138
- <AlertTriangle className="size-3.5 text-amber-500 shrink-0 mt-0.5" />
139
- <p className="text-muted-foreground">{data.error}</p>
140
- </div>
141
- )}
142
-
143
- {cheapestPerGpu.length > 0 && (
144
- <div className="overflow-x-auto">
145
- <table className="w-full text-xs">
146
- <thead className="bg-muted/40 text-[10px] uppercase">
147
- <tr>
148
- <th className="text-left px-3 py-1.5">GPU</th>
149
- <th className="text-left px-3 py-1.5">Azure SKU</th>
150
- <th className="text-right px-3 py-1.5">$/GPU-hr (cheapest)</th>
151
- <th className="text-left px-3 py-1.5">Region</th>
152
- <th className="text-left px-3 py-1.5">As of</th>
153
- </tr>
154
- </thead>
155
- <tbody>
156
- {cheapestPerGpu.map((p, i) => (
157
- <tr key={i} className="border-t border-border/40 hover:bg-muted/20">
158
- <td className="px-3 py-1.5 font-medium">{p.gpu}</td>
159
- <td className="px-3 py-1.5 text-muted-foreground font-mono text-[10px]">{p.sku}</td>
160
- <td className="px-3 py-1.5 text-right font-mono text-emerald-500 font-semibold">
161
- ${p.price}/hr
162
- </td>
163
- <td className="px-3 py-1.5 text-muted-foreground">{p.region}</td>
164
- <td className="px-3 py-1.5 text-muted-foreground text-[10px]">{asOfLabel}</td>
165
- </tr>
166
- ))}
167
- </tbody>
168
- </table>
169
- <p className="text-[10px] text-muted-foreground mt-2 leading-relaxed">
170
- Per-GPU price derived from Azure instance price ÷ GPU count.
171
- On-demand rates only (spot/reserved excluded).
172
- Prices may vary by region, availability zone, and commitment.
173
- Source: <a
174
- href="https://prices.azure.com/api/retail/prices"
175
- target="_blank"
176
- rel="noopener noreferrer"
177
- className="text-emerald-500 hover:underline"
178
- >Azure Retail Prices API ↗</a>
179
- </p>
180
- </div>
181
- )}
182
- </CardContent>
183
- </Card>
184
- );
185
- }
@@ -1,340 +0,0 @@
1
- "use client";
2
-
3
- import { useState } from "react";
4
- import {
5
- Card,
6
- CardContent,
7
- CardDescription,
8
- CardHeader,
9
- CardTitle,
10
- } from "@/components/ui/card";
11
- import { Button } from "@/components/ui/button";
12
- import { Textarea } from "@/components/ui/textarea";
13
- import { Badge } from "@/components/ui/badge";
14
- import { AlertTriangle, CheckCircle2, Upload, Beaker, Download } from "lucide-react";
15
- import {
16
- vllmParser,
17
- } from "@/lib/benchmark-parser-vllm";
18
- import { sglangParser } from "@/lib/benchmark-parser-sglang";
19
- import { trtllmParser } from "@/lib/benchmark-parser-trtllm";
20
- import { tokcalcStandardParser, BENCHMARK_TEMPLATE } from "@/lib/benchmark-parser-tokcalc";
21
- import type {
22
- BenchmarkRecord,
23
- BenchmarkParser,
24
- CalibrationResult,
25
- } from "@/lib/benchmark-schema";
26
- import {
27
- formatCalibrationVerdict,
28
- generateBenchmarkId,
29
- } from "@/lib/benchmark-schema";
30
- import type { CalcResult } from "@/lib/token-calc";
31
- import { fmtTokens, fmtMs } from "@/lib/token-calc";
32
- import { track } from "@/lib/track";
33
-
34
- interface BenchmarkImportProps {
35
- /** The theoretical estimate from tokcalc's formulas */
36
- estimate: CalcResult;
37
- /** Current model name (for matching) */
38
- modelName: string;
39
- /** Current GPU name (for matching) */
40
- gpuName: string;
41
- }
42
-
43
- export function BenchmarkImport({ estimate, modelName, gpuName }: BenchmarkImportProps) {
44
- const [rawText, setRawText] = useState("");
45
- const [record, setRecord] = useState<BenchmarkRecord | null>(null);
46
- const [error, setError] = useState<string | null>(null);
47
- const [expanded, setExpanded] = useState(false);
48
-
49
- const handleParse = () => {
50
- setError(null);
51
- if (!rawText.trim()) {
52
- setError("Paste a benchmark JSON output first.");
53
- return;
54
- }
55
-
56
- // Try parsers — order matters: most-specific first, most-generic last.
57
- // TRT-LLM has unique field names (first_token_latency_avg, inter_token_latency_avg)
58
- // SGLang has unique field names (total_throughput, ttft_avg, itl_avg)
59
- // vLLM is most generic (output_throughput, mean_ttft_ms) — try last
60
- const parsers: BenchmarkParser[] = [trtllmParser, sglangParser, vllmParser, tokcalcStandardParser];
61
- for (const parser of parsers) {
62
- if (parser.detect(rawText)) {
63
- const parsed = parser.parse(rawText);
64
- if (parsed) {
65
- setRecord(parsed);
66
- track("imported_benchmark", {
67
- engine: parsed.serving_engine,
68
- model: parsed.model_display_name,
69
- throughput: parsed.output_token_throughput_tps,
70
- parser: parser.name,
71
- });
72
- setExpanded(true);
73
- return;
74
- }
75
- }
76
- }
77
-
78
- setError(
79
- "Couldn't parse this as vLLM, SGLang, or TensorRT-LLM benchmark JSON. " +
80
- "Make sure you're pasting the raw JSON output from the benchmark script. " +
81
- "Supported formats: vLLM benchmark_serving.py, SGLang benchmark, TRT-LLM benchmark_serving.py."
82
- );
83
- };
84
-
85
- // Compute calibration if we have both estimate and observed data
86
- const calibration: CalibrationResult | null = record
87
- ? computeCalibration(estimate, record)
88
- : null;
89
-
90
- const verdict = calibration ? formatCalibrationVerdict(calibration.verdict) : null;
91
-
92
- return (
93
- <Card className="border-border/60 shadow-sm">
94
- <CardHeader className="pb-3">
95
- <CardTitle className="text-sm flex items-center gap-2">
96
- <Beaker className="size-4 text-emerald-500" />
97
- Observed benchmark — calibrate the formula
98
- </CardTitle>
99
- <CardDescription className="text-xs">
100
- Paste a vLLM benchmark JSON output to see how tokcalc&apos;s theoretical estimates compare to real-world measured performance.
101
- The calibration verdict shows whether the formula is accurate, too conservative, or too optimistic.
102
- </CardDescription>
103
- </CardHeader>
104
- <CardContent className="space-y-3">
105
- {!record && !expanded && (
106
- <div className="flex flex-col sm:flex-row gap-2">
107
- <Button
108
- variant="outline"
109
- size="sm"
110
- onClick={() => setExpanded(true)}
111
- className="flex-1 gap-1.5"
112
- >
113
- <Upload className="size-3.5" />
114
- Import a benchmark result
115
- </Button>
116
- <Button
117
- variant="ghost"
118
- size="sm"
119
- onClick={() => {
120
- const blob = new Blob([BENCHMARK_TEMPLATE], { type: "application/json" });
121
- const url = URL.createObjectURL(blob);
122
- const a = document.createElement("a");
123
- a.href = url;
124
- a.download = "tokcalc-benchmark-template.json";
125
- document.body.appendChild(a);
126
- a.click();
127
- document.body.removeChild(a);
128
- URL.revokeObjectURL(url);
129
- track("downloaded_benchmark_template");
130
- }}
131
- className="flex-1 gap-1.5"
132
- >
133
- <Download className="size-3.5" />
134
- Download template
135
- </Button>
136
- </div>
137
- )}
138
-
139
- {expanded && !record && (
140
- <>
141
- <div>
142
- <Textarea
143
- placeholder={`Paste JSON from:\n- vllm benchmark_serving.py output\n- SGLang benchmark output\n- TensorRT-LLM benchmark_serving.py output\n- tokcalc community template (download above)\n\nOr click "Download template" to get a structured JSON you can fill in manually.`}
144
- value={rawText}
145
- onChange={(e) => setRawText(e.target.value)}
146
- className="font-mono text-xs min-h-[160px] resize-y"
147
- />
148
- </div>
149
- {error && (
150
- <div className="text-[11px] text-amber-600 dark:text-amber-400 flex items-start gap-1.5">
151
- <AlertTriangle className="size-3.5 shrink-0 mt-0.5" />
152
- <span>{error}</span>
153
- </div>
154
- )}
155
- <div className="flex gap-2">
156
- <Button size="sm" onClick={handleParse} className="gap-1.5">
157
- <Beaker className="size-3.5" />
158
- Parse &amp; calibrate
159
- </Button>
160
- <Button
161
- size="sm"
162
- variant="ghost"
163
- onClick={() => { setExpanded(false); setRawText(""); setError(null); }}
164
- >
165
- Cancel
166
- </Button>
167
- </div>
168
- <p className="text-[10px] text-muted-foreground leading-relaxed">
169
- Your benchmark data stays in your browser — nothing is sent to a server.
170
- SGLang and TensorRT-LLM parsers are on the roadmap.
171
- </p>
172
- </>
173
- )}
174
-
175
- {record && calibration && verdict && (
176
- <div className="space-y-3">
177
- {/* Calibration verdict */}
178
- <div className={`flex items-start gap-3 p-3 rounded-md border ${
179
- calibration.verdict === "validated"
180
- ? "border-emerald-500/40 bg-emerald-500/5"
181
- : calibration.verdict === "insufficient_data"
182
- ? "border-border/60 bg-muted/20"
183
- : "border-amber-500/40 bg-amber-500/5"
184
- }`}>
185
- {calibration.verdict === "validated" ? (
186
- <CheckCircle2 className="size-4 text-emerald-500 shrink-0 mt-0.5" />
187
- ) : (
188
- <AlertTriangle className="size-4 text-amber-500 shrink-0 mt-0.5" />
189
- )}
190
- <div className="text-xs">
191
- <div className={`font-medium ${verdict.color}`}>
192
- {verdict.label}
193
- </div>
194
- <div className="text-muted-foreground mt-0.5 leading-relaxed">
195
- {verdict.description}
196
- </div>
197
- {calibration.calibrationErrors.throughputErrorPct !== undefined && (
198
- <div className="mt-1.5 font-mono text-[11px]">
199
- Throughput: estimated <strong className="text-foreground">{fmtTokens(estimate.decodeTokensPerSec)} tok/s</strong> vs observed <strong className="text-foreground">{fmtTokens(record.output_token_throughput_tps)} tok/s</strong>
200
- {" "}→ <span className={Math.abs(calibration.calibrationErrors.throughputErrorPct) < 20 ? "text-emerald-500" : "text-amber-500"}>
201
- {calibration.calibrationErrors.throughputErrorPct > 0 ? "+" : ""}{calibration.calibrationErrors.throughputErrorPct.toFixed(1)}% error
202
- </span>
203
- </div>
204
- )}
205
- {calibration.calibrationErrors.ttftErrorPct !== undefined && (
206
- <div className="mt-1 font-mono text-[11px]">
207
- TTFT: estimated <strong className="text-foreground">{fmtMs(estimate.ttftMs)}</strong> vs observed <strong className="text-foreground">{fmtMs(record.ttft_mean_ms || 0)}</strong>
208
- {" "}→ <span className={Math.abs(calibration.calibrationErrors.ttftErrorPct) < 20 ? "text-emerald-500" : "text-amber-500"}>
209
- {calibration.calibrationErrors.ttftErrorPct > 0 ? "+" : ""}{calibration.calibrationErrors.ttftErrorPct.toFixed(1)}% error
210
- </span>
211
- </div>
212
- )}
213
- {calibration.calibrationErrors.itlErrorPct !== undefined && (
214
- <div className="mt-1 font-mono text-[11px]">
215
- ITL: estimated <strong className="text-foreground">{fmtMs(estimate.itlMs)}</strong> vs observed <strong className="text-foreground">{fmtMs(record.itl_mean_ms || 0)}</strong>
216
- {" "}→ <span className={Math.abs(calibration.calibrationErrors.itlErrorPct) < 20 ? "text-emerald-500" : "text-amber-500"}>
217
- {calibration.calibrationErrors.itlErrorPct > 0 ? "+" : ""}{calibration.calibrationErrors.itlErrorPct.toFixed(1)}% error
218
- </span>
219
- </div>
220
- )}
221
- </div>
222
- </div>
223
-
224
- {/* Observed metrics detail */}
225
- <div className="grid grid-cols-2 sm:grid-cols-4 gap-3 text-sm">
226
- <div className="space-y-0.5">
227
- <div className="text-[10px] uppercase tracking-wider text-muted-foreground">Observed throughput</div>
228
- <div className="font-mono text-sm font-medium">{fmtTokens(record.output_token_throughput_tps)} tok/s</div>
229
- <div className="text-[10px] text-muted-foreground">{record.request_throughput_rps?.toFixed(1) || "—"} req/s</div>
230
- </div>
231
- <div className="space-y-0.5">
232
- <div className="text-[10px] uppercase tracking-wider text-muted-foreground">TTFT (mean / p99)</div>
233
- <div className="font-mono text-sm font-medium">{fmtMs(record.ttft_mean_ms || 0)}</div>
234
- <div className="text-[10px] text-muted-foreground">p99: {fmtMs(record.ttft_p99_ms || 0)}</div>
235
- </div>
236
- <div className="space-y-0.5">
237
- <div className="text-[10px] uppercase tracking-wider text-muted-foreground">ITL (mean / p99)</div>
238
- <div className="font-mono text-sm font-medium">{fmtMs(record.itl_mean_ms || 0)}</div>
239
- <div className="text-[10px] text-muted-foreground">p99: {fmtMs(record.itl_p99_ms || 0)}</div>
240
- </div>
241
- <div className="space-y-0.5">
242
- <div className="text-[10px] uppercase tracking-wider text-muted-foreground">Config</div>
243
- <div className="font-mono text-[11px] font-medium">{record.gpu_count}×{record.accelerator_model.split(" ")[0]}</div>
244
- <div className="text-[10px] text-muted-foreground">{record.serving_engine} v{record.engine_version.slice(0, 6)}</div>
245
- </div>
246
- </div>
247
-
248
- {/* Provenance */}
249
- <div className="text-[10px] text-muted-foreground border-t border-border/40 pt-2">
250
- <strong className="text-foreground">Source:</strong> {record.source_name} ({record.confidence_tier.replace(/_/g, " ")}) ·
251
- {" "}{record.request_count.toLocaleString()} requests · {record.input_tokens_mean} in / {record.output_tokens_mean} out tokens ·
252
- {" "}concurrency {record.concurrency} ·
253
- {" "}{record.model_display_name} · {record.quantization_format} ·
254
- {" "}<a href={record.source_url} target="_blank" rel="noopener noreferrer" className="text-emerald-500 hover:underline">source ↗</a>
255
- </div>
256
-
257
- <Button
258
- size="sm"
259
- variant="ghost"
260
- onClick={() => { setRecord(null); setRawText(""); setExpanded(false); }}
261
- className="text-xs"
262
- >
263
- Import a different benchmark
264
- </Button>
265
- </div>
266
- )}
267
- </CardContent>
268
- </Card>
269
- );
270
- }
271
-
272
- /* ---------- Calibration computation ---------- */
273
-
274
- function computeCalibration(
275
- estimate: CalcResult,
276
- observed: BenchmarkRecord,
277
- ): CalibrationResult {
278
- const errors: CalibrationResult["calibrationErrors"] = {};
279
-
280
- // Throughput calibration (if observed has throughput)
281
- if (observed.output_token_throughput_tps && estimate.decodeTokensPerSec > 0) {
282
- errors.throughputErrorPct =
283
- ((observed.output_token_throughput_tps - estimate.decodeTokensPerSec) / estimate.decodeTokensPerSec) * 100;
284
- }
285
-
286
- // TTFT calibration (if observed has mean TTFT)
287
- if (observed.ttft_mean_ms && estimate.ttftMs > 0) {
288
- errors.ttftErrorPct =
289
- ((observed.ttft_mean_ms - estimate.ttftMs) / estimate.ttftMs) * 100;
290
- }
291
-
292
- // ITL calibration (if observed has mean ITL)
293
- if (observed.itl_mean_ms && estimate.itlMs > 0) {
294
- errors.itlErrorPct =
295
- ((observed.itl_mean_ms - estimate.itlMs) / estimate.itlMs) * 100;
296
- }
297
-
298
- // Memory calibration (if observed has peak GPU memory)
299
- if (observed.gpu_memory_peak_gb && estimate.totalVramNeededGb > 0) {
300
- errors.memoryErrorPct =
301
- ((observed.gpu_memory_peak_gb - estimate.totalVramNeededGb) / estimate.totalVramNeededGb) * 100;
302
- }
303
-
304
- // Determine verdict
305
- let verdict: CalibrationResult["verdict"] = "insufficient_data";
306
- const errorValues = Object.values(errors).filter((v) => v !== undefined) as number[];
307
-
308
- if (errorValues.length >= 2) {
309
- const maxAbsError = Math.max(...errorValues.map((v) => Math.abs(v)));
310
- if (maxAbsError < 20) {
311
- verdict = "validated";
312
- } else if (maxAbsError > 0) {
313
- // If most errors are positive (observed > estimated), we underestimated
314
- // If most are negative (observed < estimated), we overestimated
315
- const avgError = errorValues.reduce((a, b) => a + b, 0) / errorValues.length;
316
- verdict = avgError > 0 ? "underestimated" : "overestimated";
317
- }
318
- }
319
-
320
- return {
321
- estimated: {
322
- decodeTokensPerSec: estimate.decodeTokensPerSec,
323
- aggregateTokensPerSec: estimate.aggregateTokensPerSec,
324
- prefillTokensPerSec: estimate.prefillTokensPerSec,
325
- ttftMs: estimate.ttftMs,
326
- itlMs: estimate.itlMs,
327
- modelSizeGb: estimate.modelSizeGb,
328
- },
329
- observed: {
330
- outputTokenThroughputTps: observed.output_token_throughput_tps,
331
- ttftMeanMs: observed.ttft_mean_ms,
332
- ttftP95Ms: observed.ttft_p95_ms,
333
- itlMeanMs: observed.itl_mean_ms,
334
- itlP95Ms: observed.itl_p95_ms,
335
- gpuMemoryPeakGb: observed.gpu_memory_peak_gb,
336
- },
337
- calibrationErrors: errors,
338
- verdict,
339
- };
340
- }
@@ -1,116 +0,0 @@
1
- "use client";
2
-
3
- import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
4
- import type { Confidence } from "@/lib/token-calc";
5
-
6
- const CONFIDENCE_CONFIG: Record<
7
- Confidence,
8
- { color: string; label: string; description: string }
9
- > = {
10
- measured: {
11
- color: "bg-emerald-500",
12
- label: "Measured",
13
- description:
14
- "Sourced directly from official specs, model cards, API pricing pages, or HuggingFace config.json. Highest confidence — verify at the cited source URL.",
15
- },
16
- modeled: {
17
- color: "bg-emerald-400",
18
- label: "Modeled",
19
- description:
20
- "Derived from a physics-based formula with cited sources (e.g., decode tok/s = HBM_BW × η_mem × quant_eff / model_size). Conditional on assumptions — see formula docs.",
21
- },
22
- inferred: {
23
- color: "bg-amber-500",
24
- label: "Inferred",
25
- description:
26
- "Derived from a heuristic with known error bars (e.g., continuous batching multiplier default 1.5×, long-context attention O(N²) correction). Treat as ballpark, not exact.",
27
- },
28
- "user-supplied": {
29
- color: "bg-zinc-400",
30
- label: "User-supplied",
31
- description: "You provided this value directly. We trust your input.",
32
- },
33
- };
34
-
35
- export function ConfidenceBadge({
36
- confidence,
37
- className,
38
- }: {
39
- confidence: Confidence;
40
- className?: string;
41
- }) {
42
- const config = CONFIDENCE_CONFIG[confidence];
43
- return (
44
- <Tooltip>
45
- <TooltipTrigger asChild>
46
- <button
47
- type="button"
48
- aria-label={`Confidence: ${config.label}`}
49
- className={`inline-flex items-center gap-1 text-[9px] font-medium uppercase tracking-wider text-muted-foreground hover:text-foreground transition-colors ${className ?? ""}`}
50
- >
51
- <span className={`size-1.5 rounded-full ${config.color}`} />
52
- {config.label}
53
- </button>
54
- </TooltipTrigger>
55
- <TooltipContent side="top" className="max-w-[280px] text-xs leading-relaxed">
56
- <div className="font-semibold mb-1">{config.label}</div>
57
- <div className="text-muted-foreground">{config.description}</div>
58
- </TooltipContent>
59
- </Tooltip>
60
- );
61
- }
62
-
63
- /** Compact version — just the dot, no label. Use in tight spaces like headline cards. */
64
- export function ConfidenceDot({
65
- confidence,
66
- className,
67
- }: {
68
- confidence: Confidence;
69
- className?: string;
70
- }) {
71
- const config = CONFIDENCE_CONFIG[confidence];
72
- return (
73
- <Tooltip>
74
- <TooltipTrigger asChild>
75
- <button
76
- type="button"
77
- aria-label={`Confidence: ${config.label}`}
78
- className={`inline-flex items-center justify-center ${className ?? ""}`}
79
- >
80
- <span className={`size-1.5 rounded-full ${config.color} hover:scale-125 transition-transform`} />
81
- </button>
82
- </TooltipTrigger>
83
- <TooltipContent side="top" className="max-w-[280px] text-xs leading-relaxed">
84
- <div className="font-semibold mb-1">{config.label}</div>
85
- <div className="text-muted-foreground">{config.description}</div>
86
- </TooltipContent>
87
- </Tooltip>
88
- );
89
- }
90
-
91
- /** Legend block — show at the bottom of the Calculator tab + in the glossary. */
92
- export function ConfidenceLegend() {
93
- return (
94
- <div className="rounded-md border border-border/60 bg-muted/20 p-3">
95
- <div className="text-[10px] uppercase tracking-wider text-muted-foreground font-semibold mb-2">
96
- Confidence levels
97
- </div>
98
- <div className="grid grid-cols-2 sm:grid-cols-4 gap-2 text-[11px]">
99
- {(Object.keys(CONFIDENCE_CONFIG) as Confidence[]).map((key) => {
100
- const config = CONFIDENCE_CONFIG[key];
101
- return (
102
- <div key={key} className="flex items-start gap-1.5">
103
- <span className={`size-1.5 rounded-full ${config.color} mt-1 shrink-0`} />
104
- <div>
105
- <div className="font-medium text-foreground">{config.label}</div>
106
- <div className="text-muted-foreground text-[10px] leading-snug mt-0.5">
107
- {config.description.split(". ")[0]}.
108
- </div>
109
- </div>
110
- </div>
111
- );
112
- })}
113
- </div>
114
- </div>
115
- );
116
- }