@tokcalc/mcp-server 0.1.3 → 0.2.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +86 -406
  2. package/dist/http.js +23504 -0
  3. package/dist/index.js +21290 -0
  4. package/package.json +36 -92
  5. package/.zscripts/build.sh +0 -175
  6. package/.zscripts/database-runtime-build.sh +0 -33
  7. package/.zscripts/dev.pid +0 -1
  8. package/.zscripts/dev.sh +0 -154
  9. package/.zscripts/mini-services-build.sh +0 -78
  10. package/.zscripts/mini-services-install.sh +0 -65
  11. package/.zscripts/mini-services-start.sh +0 -123
  12. package/.zscripts/python-runtime-build.sh +0 -120
  13. package/.zscripts/start.sh +0 -145
  14. package/CAPACITY_STUDY.md +0 -283
  15. package/CODE_OF_CONDUCT.md +0 -55
  16. package/CONTRIBUTING.md +0 -177
  17. package/Caddyfile +0 -23
  18. package/LICENSE +0 -204
  19. package/bun.lock +0 -1965
  20. package/components.json +0 -21
  21. package/db/custom.db +0 -0
  22. package/download/README.md +0 -1
  23. package/download/tokcalc-dark-calculator.png +0 -0
  24. package/download/tokcalc-dark-default.png +0 -0
  25. package/download/tokcalc-demo.webm +0 -0
  26. package/download/tokcalc-github-link.png +0 -0
  27. package/download/tokcalc-hydration-fixed.png +0 -0
  28. package/download/tokcalc-issue-resolved.png +0 -0
  29. package/download/tokcalc-light-mode.png +0 -0
  30. package/download/tokcalc-light-reference.png +0 -0
  31. package/download/tokcalc-long-context-qwen.png +0 -0
  32. package/download/tokcalc-long-context.png +0 -0
  33. package/download/tokcalc-og-image-preview.png +0 -0
  34. package/download/tokcalc-phase2-3.png +0 -0
  35. package/download/tokcalc-plain-english.png +0 -0
  36. package/download/tokcalc-preview.png +0 -0
  37. package/download/tokcalc-share-bvb.png +0 -0
  38. package/download/tokcalc-share-feature.png +0 -0
  39. package/download/tokcalc-tab-build-vs-buy.png +0 -0
  40. package/download/tokcalc-tab-calculator.png +0 -0
  41. package/download/tokcalc-tab-reference.png +0 -0
  42. package/eslint.config.mjs +0 -50
  43. package/examples/websocket/frontend.tsx +0 -196
  44. package/examples/websocket/server.ts +0 -138
  45. package/mini-services/.gitkeep +0 -0
  46. package/mini-services/mcp-server/README.md +0 -86
  47. package/mini-services/mcp-server/bun.lock +0 -202
  48. package/mini-services/mcp-server/index.ts +0 -504
  49. package/mini-services/mcp-server/package.json +0 -40
  50. package/next.config.ts +0 -12
  51. package/postcss.config.mjs +0 -5
  52. package/prisma/schema.prisma +0 -32
  53. package/public/google6f58ca6be85fa903.html +0 -1
  54. package/public/logo.svg +0 -29
  55. package/public/manifest.json +0 -51
  56. package/public/og-icon-256.png +0 -0
  57. package/public/og.png +0 -0
  58. package/public/robots.txt +0 -25
  59. package/public/sitemap.xml +0 -23
  60. package/public/tokcalc-demo.gif +0 -0
  61. package/scripts/og-template.html +0 -120
  62. package/scripts/render-og.mjs +0 -43
  63. package/server.json +0 -21
  64. package/src/app/api/pricing/aws/route.ts +0 -186
  65. package/src/app/api/pricing/azure/route.ts +0 -168
  66. package/src/app/api/pricing/gcp/route.ts +0 -230
  67. package/src/app/api/pricing/vast-ai/route.ts +0 -164
  68. package/src/app/api/route.ts +0 -5
  69. package/src/app/compare/h100-vs-h200/layout.tsx +0 -30
  70. package/src/app/compare/h100-vs-h200/page.tsx +0 -328
  71. package/src/app/globals.css +0 -122
  72. package/src/app/layout.tsx +0 -276
  73. package/src/app/page.tsx +0 -2670
  74. package/src/components/azure-live-pricing.tsx +0 -185
  75. package/src/components/benchmark-import.tsx +0 -340
  76. package/src/components/confidence-badge.tsx +0 -116
  77. package/src/components/live-pricing-comparison.tsx +0 -241
  78. package/src/components/theme-provider.tsx +0 -11
  79. package/src/components/theme-toggle.tsx +0 -55
  80. package/src/components/ui/accordion.tsx +0 -66
  81. package/src/components/ui/alert-dialog.tsx +0 -157
  82. package/src/components/ui/alert.tsx +0 -66
  83. package/src/components/ui/aspect-ratio.tsx +0 -11
  84. package/src/components/ui/avatar.tsx +0 -53
  85. package/src/components/ui/badge.tsx +0 -46
  86. package/src/components/ui/breadcrumb.tsx +0 -109
  87. package/src/components/ui/button.tsx +0 -59
  88. package/src/components/ui/calendar.tsx +0 -213
  89. package/src/components/ui/card.tsx +0 -92
  90. package/src/components/ui/carousel.tsx +0 -241
  91. package/src/components/ui/chart.tsx +0 -353
  92. package/src/components/ui/checkbox.tsx +0 -32
  93. package/src/components/ui/collapsible.tsx +0 -33
  94. package/src/components/ui/command.tsx +0 -184
  95. package/src/components/ui/context-menu.tsx +0 -252
  96. package/src/components/ui/dialog.tsx +0 -143
  97. package/src/components/ui/drawer.tsx +0 -135
  98. package/src/components/ui/dropdown-menu.tsx +0 -257
  99. package/src/components/ui/form.tsx +0 -167
  100. package/src/components/ui/hover-card.tsx +0 -44
  101. package/src/components/ui/input-otp.tsx +0 -77
  102. package/src/components/ui/input.tsx +0 -21
  103. package/src/components/ui/label.tsx +0 -24
  104. package/src/components/ui/menubar.tsx +0 -276
  105. package/src/components/ui/navigation-menu.tsx +0 -168
  106. package/src/components/ui/pagination.tsx +0 -127
  107. package/src/components/ui/popover.tsx +0 -48
  108. package/src/components/ui/progress.tsx +0 -31
  109. package/src/components/ui/radio-group.tsx +0 -45
  110. package/src/components/ui/resizable.tsx +0 -56
  111. package/src/components/ui/scroll-area.tsx +0 -58
  112. package/src/components/ui/select.tsx +0 -185
  113. package/src/components/ui/separator.tsx +0 -28
  114. package/src/components/ui/sheet.tsx +0 -139
  115. package/src/components/ui/sidebar.tsx +0 -726
  116. package/src/components/ui/skeleton.tsx +0 -13
  117. package/src/components/ui/slider.tsx +0 -63
  118. package/src/components/ui/sonner.tsx +0 -25
  119. package/src/components/ui/switch.tsx +0 -31
  120. package/src/components/ui/table.tsx +0 -116
  121. package/src/components/ui/tabs.tsx +0 -66
  122. package/src/components/ui/textarea.tsx +0 -18
  123. package/src/components/ui/toast.tsx +0 -129
  124. package/src/components/ui/toaster.tsx +0 -35
  125. package/src/components/ui/toggle-group.tsx +0 -73
  126. package/src/components/ui/toggle.tsx +0 -47
  127. package/src/components/ui/tooltip.tsx +0 -61
  128. package/src/components/vast-ai-live-pricing.tsx +0 -176
  129. package/src/hooks/use-mobile.ts +0 -19
  130. package/src/hooks/use-toast.ts +0 -194
  131. package/src/lib/benchmark-parser-sglang.ts +0 -150
  132. package/src/lib/benchmark-parser-tokcalc.ts +0 -247
  133. package/src/lib/benchmark-parser-trtllm.ts +0 -152
  134. package/src/lib/benchmark-parser-vllm.ts +0 -198
  135. package/src/lib/benchmark-schema.ts +0 -263
  136. package/src/lib/db.ts +0 -13
  137. package/src/lib/engine-presets.ts +0 -183
  138. package/src/lib/price-schema.ts +0 -141
  139. package/src/lib/token-calc.ts +0 -808
  140. package/src/lib/track.ts +0 -31
  141. package/src/lib/url-state.ts +0 -256
  142. package/src/lib/utils.ts +0 -6
  143. package/tailwind.config.ts +0 -64
  144. package/tests/database-runtime-build.sh +0 -75
  145. package/tests/python-runtime-build.sh +0 -64
  146. package/tests/python-runtime-container.sh +0 -31
  147. package/tool-results/bash_1789888171144_2c5381860539.txt +0 -161
  148. package/tool-results/bash_1789888175925_49c53ba3c61b.txt +0 -191
  149. package/tool-results/bash_1789888181202_49c53ba3c61b.txt +0 -191
  150. package/tool-results/bash_1789888195219_4a86a5c91411.txt +0 -200
  151. package/tool-results/bash_1789888203128_6cca13c71b47.txt +0 -199
  152. package/tool-results/bash_1789929256963_2a52aff0d0a8.txt +0 -160
  153. package/tool-results/read_1789888151021_69f58eec6a5b.txt +0 -653
  154. package/tool-results/read_1789888153837_1d3a8bfc2a94.txt +0 -653
  155. package/tool-results/read_1789888163087_ccc406d47505.txt +0 -122
  156. package/tool-results/read_1789888167347_67d1d7c9830a.txt +0 -122
  157. package/tool-results/read_1789929252529_d90e8f383a25.txt +0 -285
  158. package/tsconfig.json +0 -42
  159. package/upload/Pasted Content_1789887800864.txt +0 -652
  160. package/upload/Pasted Content_1789887909561.txt +0 -652
  161. package/upload/Pasted Content_1789887918428.txt +0 -652
  162. package/upload/Pasted Content_1789887959420.txt +0 -652
  163. package/upload/Pasted Content_1789888020485.txt +0 -652
  164. package/upload/Pasted Content_1789888058079.txt +0 -652
  165. package/upload/Pasted Content_1789888885033.txt +0 -686
  166. package/upload/Pasted Content_1789928912741.txt +0 -285
  167. package/upload/Pasted Content_1789928938402.txt +0 -285
  168. package/upload/Pasted Content_1789929160389.txt +0 -285
  169. package/upload/Pasted Content_1789929176660.txt +0 -285
  170. package/upload/issue_vision.json +0 -28
  171. package/upload/pasted_image_1789883175209.png +0 -0
  172. package/upload/pasted_image_1789899056690.png +0 -0
  173. package/upload/pasted_image_1789900371483.png +0 -0
  174. package/upload/pasted_image_1789900472823.png +0 -0
  175. package/upload/pasted_image_1789900490374.png +0 -0
  176. package/upload/pasted_image_1789900585552.png +0 -0
  177. package/upload/pasted_image_1789900606519.png +0 -0
  178. package/upload/pasted_image_1789901598705.png +0 -0
  179. package/upload/pasted_image_1789901613545.png +0 -0
  180. package/upload/pasted_image_1789978382674.png +0 -0
  181. package/upload/pasted_image_1789978392749.png +0 -0
  182. package/upload/pasted_image_1789978474879.png +0 -0
  183. package/upload/pasted_image_1789978523652.png +0 -0
  184. package/upload/pasted_image_1789984219089.png +0 -0
  185. package/upload/pasted_image_1789984491896.png +0 -0
  186. package/upload/pasted_image_1789985017950.png +0 -0
  187. package/upload/pasted_image_1789985036765.png +0 -0
  188. package/upload/pasted_image_1789985049848.png +0 -0
  189. package/upload/pasted_image_1790002427833.png +0 -0
  190. package/upload/pasted_image_1790002659944.png +0 -0
  191. package/upload/pasted_image_1790037038476.png +0 -0
  192. package/upload/screenshot_analysis.json +0 -28
  193. package/upload/vision_output.json +0 -28
@@ -1,247 +0,0 @@
1
- /**
2
- * Community benchmark submission template.
3
- *
4
- * Users download this JSON file, fill in their benchmark results,
5
- * and paste it into tokcalc's BenchmarkImport textarea.
6
- *
7
- * The tokcalc-standard parser recognizes this format and creates
8
- * a BenchmarkRecord with confidence_tier = "community_reproducible"
9
- * (if all required fields are filled) or "community_unverified" (if some are missing).
10
- *
11
- * This is the "tier 4" ingestion path from the Perplexity research:
12
- * "A tokcalc-standard JSON schema for community uploads."
13
- */
14
-
15
- export const BENCHMARK_TEMPLATE = `{
16
- "format": "tokcalc-standard",
17
- "version": "1.0",
18
-
19
- "source": {
20
- "submitter": "your-github-username",
21
- "submit_date": "2026-09-24",
22
- "raw_artifact_url": "https://github.com/your-username/your-repo/blob/main/benchmark-results.json",
23
- "notes": "Describe how you ran this benchmark. Include the exact command you used."
24
- },
25
-
26
- "model": {
27
- "model_id": "meta-llama/Llama-3.3-70B-Instruct",
28
- "model_revision": "commit hash or release date",
29
- "quantization_format": "FP8",
30
- "weight_dtype": "float8_e4m3",
31
- "kv_cache_dtype": "float16"
32
- },
33
-
34
- "hardware": {
35
- "accelerator_vendor": "NVIDIA",
36
- "accelerator_model": "H100 SXM5 80GB",
37
- "gpu_count": 1,
38
- "memory_per_gpu_gb": 80,
39
- "interconnect": "NVLink",
40
- "cloud_provider": "RunPod",
41
- "instance_type": "H100 80GB SXM",
42
- "region": "us-east-1",
43
- "driver_version": "550.54.15",
44
- "cuda_version": "12.4"
45
- },
46
-
47
- "engine": {
48
- "serving_engine": "vLLM",
49
- "engine_version": "0.29.0",
50
- "tensor_parallel_size": 1,
51
- "max_model_len": 32768,
52
- "max_num_seqs": 256,
53
- "max_num_batched_tokens": 8192,
54
- "prefix_cache_enabled": true,
55
- "chunked_prefill_enabled": true,
56
- "attention_backend": "FlashInfer",
57
- "command": "vllm serve meta-llama/Llama-3.3-70B-Instruct --quantization fp8 --tensor-parallel-size 1 --max-model-len 32768 --max-num-seqs 256"
58
- },
59
-
60
- "workload": {
61
- "workload_type": "online_poisson",
62
- "dataset_name": "ShareGPT",
63
- "request_count": 1000,
64
- "concurrency": 64,
65
- "input_tokens_mean": 1024,
66
- "input_tokens_p95": 4096,
67
- "output_tokens_mean": 256,
68
- "output_tokens_p95": 1024,
69
- "streaming_enabled": true,
70
- "sampling_config": {
71
- "temperature": 0.7,
72
- "top_p": 0.95,
73
- "max_tokens": 512
74
- }
75
- },
76
-
77
- "metrics": {
78
- "successful_requests": 1000,
79
- "failed_requests": 0,
80
- "output_token_throughput_tps": 2450.5,
81
- "total_token_throughput_tps": 11000.0,
82
- "request_throughput_rps": 3.91,
83
- "ttft_mean_ms": 112.3,
84
- "ttft_p50_ms": 108.5,
85
- "ttft_p95_ms": 145.2,
86
- "ttft_p99_ms": 189.7,
87
- "itl_mean_ms": 5.67,
88
- "itl_p50_ms": 5.45,
89
- "itl_p95_ms": 7.23,
90
- "itl_p99_ms": 8.91,
91
- "gpu_memory_peak_gb": 78.5,
92
- "gpu_utilization_mean_pct": 92.5
93
- }
94
- }`;
95
-
96
- /**
97
- * Parse a tokcalc-standard benchmark JSON.
98
- * This is the community submission format — users fill in the template
99
- * and paste it into the BenchmarkImport textarea.
100
- */
101
- import type {
102
- BenchmarkParser,
103
- BenchmarkRecord,
104
- ConfidenceTier,
105
- } from "./benchmark-schema";
106
- import { generateBenchmarkId } from "./benchmark-schema";
107
-
108
- export const tokcalcStandardParser: BenchmarkParser = {
109
- name: "tokcalc-standard",
110
- version: "v1",
111
- formatLabel: "tokcalc community benchmark",
112
-
113
- detect(rawText: string): boolean {
114
- try {
115
- const data = JSON.parse(rawText);
116
- return data?.format === "tokcalc-standard" || !!data?.metrics?.output_token_throughput_tps;
117
- } catch {
118
- return false;
119
- }
120
- },
121
-
122
- parse(rawText: string): BenchmarkRecord | null {
123
- try {
124
- const data = JSON.parse(rawText);
125
- if (!data) return null;
126
-
127
- const model = data.model || {};
128
- const hardware = data.hardware || {};
129
- const engine = data.engine || {};
130
- const workload = data.workload || {};
131
- const metrics = data.metrics || {};
132
- const source = data.source || {};
133
-
134
- const modelId = model.model_id || "unknown";
135
- const modelDisplayName = modelId.split("/").pop() || modelId;
136
-
137
- // Determine confidence tier based on completeness
138
- const hasFullConfig =
139
- modelId !== "unknown" &&
140
- hardware.accelerator_model &&
141
- engine.serving_engine &&
142
- metrics.output_token_throughput_tps > 0 &&
143
- workload.request_count > 0;
144
- const confidenceTier: ConfidenceTier = hasFullConfig
145
- ? "community_reproducible"
146
- : "community_unverified";
147
-
148
- return {
149
- benchmark_id: generateBenchmarkId(),
150
- source_name: "community",
151
- source_url: source.raw_artifact_url || "",
152
- source_license: "CC-BY-SA-4.0",
153
- ingestion_method: "uploaded_raw_artifact",
154
- retrieved_at: new Date().toISOString(),
155
- benchmark_started_at: source.submit_date,
156
- raw_artifact_url: source.raw_artifact_url,
157
- parser_name: "tokcalc-standard",
158
- parser_version: "v1",
159
- confidence_tier: confidenceTier,
160
- verification_status: "schema_valid",
161
-
162
- model_id: modelId,
163
- model_display_name: modelDisplayName,
164
- model_revision: model.model_revision || "unspecified",
165
- architecture: model.architecture,
166
- parameter_count_b: model.parameter_count_b,
167
- context_window_tokens: engine.max_model_len,
168
- quantization_format: model.quantization_format || "unspecified",
169
- weight_dtype: model.weight_dtype,
170
- kv_cache_dtype: model.kv_cache_dtype || "float16",
171
-
172
- accelerator_vendor: hardware.accelerator_vendor || "NVIDIA",
173
- accelerator_model: hardware.accelerator_model || "unspecified",
174
- gpu_count: hardware.gpu_count || engine.tensor_parallel_size || 1,
175
- memory_per_gpu_gb: hardware.memory_per_gpu_gb || 0,
176
- host_cpu: hardware.host_cpu,
177
- host_memory_gb: hardware.host_memory_gb,
178
- interconnect: hardware.interconnect,
179
- topology: hardware.topology,
180
- cloud_provider: hardware.cloud_provider,
181
- instance_type: hardware.instance_type,
182
- region: hardware.region,
183
- driver_version: hardware.driver_version,
184
- cuda_rocm_metal_version: hardware.cuda_version,
185
-
186
- serving_engine: engine.serving_engine || "unspecified",
187
- engine_version: engine.engine_version || "unspecified",
188
- engine_command: engine.command,
189
- engine_config: {
190
- tensor_parallel_size: engine.tensor_parallel_size,
191
- max_model_len: engine.max_model_len,
192
- max_num_seqs: engine.max_num_seqs,
193
- max_num_batched_tokens: engine.max_num_batched_tokens,
194
- prefix_cache_enabled: engine.prefix_cache_enabled,
195
- chunked_prefill_enabled: engine.chunked_prefill_enabled,
196
- attention_backend: engine.attention_backend,
197
- },
198
-
199
- workload_type: workload.workload_type || "online_poisson",
200
- dataset_name: workload.dataset_name,
201
- request_count: workload.request_count || 0,
202
- concurrency: workload.concurrency || 1,
203
- arrival_rate_rps: workload.arrival_rate_rps,
204
- input_tokens_mean: workload.input_tokens_mean || 0,
205
- input_tokens_p50: workload.input_tokens_p50,
206
- input_tokens_p95: workload.input_tokens_p95,
207
- input_tokens_p99: workload.input_tokens_p99,
208
- output_tokens_mean: workload.output_tokens_mean || 0,
209
- output_tokens_p50: workload.output_tokens_p50,
210
- output_tokens_p95: workload.output_tokens_p95,
211
- output_tokens_p99: workload.output_tokens_p99,
212
- shared_prefix_tokens_mean: workload.shared_prefix_tokens_mean,
213
- shared_prefix_hit_rate: workload.shared_prefix_hit_rate,
214
- streaming_enabled: workload.streaming_enabled ?? true,
215
- sampling_config: workload.sampling_config,
216
-
217
- successful_requests: metrics.successful_requests ?? workload.request_count ?? 0,
218
- failed_requests: metrics.failed_requests ?? 0,
219
- request_throughput_rps: metrics.request_throughput_rps,
220
- output_token_throughput_tps: metrics.output_token_throughput_tps || 0,
221
- total_token_throughput_tps: metrics.total_token_throughput_tps,
222
- ttft_mean_ms: metrics.ttft_mean_ms,
223
- ttft_p50_ms: metrics.ttft_p50,
224
- ttft_p95_ms: metrics.ttft_p95_ms,
225
- ttft_p99_ms: metrics.ttft_p99_ms,
226
- itl_mean_ms: metrics.itl_mean_ms,
227
- itl_p50_ms: metrics.itl_p50,
228
- itl_p95_ms: metrics.itl_p95_ms,
229
- itl_p99_ms: metrics.itl_p99_ms,
230
- e2e_latency_p50_ms: metrics.e2e_latency_p50_ms,
231
- e2e_latency_p95_ms: metrics.e2e_latency_p95_ms,
232
- e2e_latency_p99_ms: metrics.e2e_latency_p99_ms,
233
- gpu_memory_peak_gb: metrics.gpu_memory_peak_gb,
234
- gpu_utilization_mean_pct: metrics.gpu_utilization_mean_pct,
235
- power_mean_w: metrics.power_mean_w,
236
- energy_per_output_token_j: metrics.energy_per_output_token_j,
237
-
238
- metric_definition_version: "tokcalc-v1",
239
- citation_text: `Community benchmark: ${metrics.output_token_throughput_tps || 0} tok/s, ${workload.request_count || 0} requests, submitted by ${source.submitter || "anonymous"}`,
240
- attribution_required: true,
241
- commercial_redistribution_allowed: "yes",
242
- };
243
- } catch {
244
- return null;
245
- }
246
- },
247
- };
@@ -1,152 +0,0 @@
1
- /**
2
- * TensorRT-LLM benchmark parser.
3
- *
4
- * TRT-LLM's `benchmark_serving.py` produces JSON output with fields like:
5
- * - throughput (tokens/s)
6
- * - latency_avg (ms) — end-to-end latency
7
- * - first_token_latency_avg (ms) — TTFT
8
- * - inter_token_latency_avg (ms) — ITL
9
- * - total_input_tokens, total_output_tokens
10
- * - num_requests
11
- * - model
12
- *
13
- * Refs: https://nvidia.github.io/TensorRT-LLM/1.3.0rc4/developer-guide/perf-benchmarking.html
14
- */
15
-
16
- import type {
17
- BenchmarkParser,
18
- BenchmarkRecord,
19
- ConfidenceTier,
20
- } from "./benchmark-schema";
21
- import { generateBenchmarkId } from "./benchmark-schema";
22
-
23
- export const trtllmParser: BenchmarkParser = {
24
- name: "trtllm-json",
25
- version: "v1",
26
- formatLabel: "TensorRT-LLM benchmark JSON",
27
-
28
- detect(rawText: string): boolean {
29
- try {
30
- const data = JSON.parse(rawText);
31
- return !!(
32
- data &&
33
- (data.throughput ||
34
- data.first_token_latency_avg ||
35
- data.inter_token_latency_avg ||
36
- data.latency_avg ||
37
- data.total_input_tokens ||
38
- data.total_output_tokens)
39
- );
40
- } catch {
41
- return false;
42
- }
43
- },
44
-
45
- parse(rawText: string): BenchmarkRecord | null {
46
- try {
47
- const data = JSON.parse(rawText);
48
- if (!data) return null;
49
-
50
- const modelId = data.model || data.model_id || data.model_name || "unknown";
51
- const modelDisplayName = modelId.split("/").pop() || modelId;
52
-
53
- const outputThroughput = data.throughput ?? data.output_throughput ?? 0;
54
- const meanTtft =
55
- data.first_token_latency_avg ?? data.ttft_mean ?? data.mean_ttft_ms;
56
- const meanItl =
57
- data.inter_token_latency_avg ?? data.itl_mean ?? data.mean_itl_ms;
58
- const meanE2E = data.latency_avg ?? data.e2e_latency_mean;
59
-
60
- const numRequests = data.num_requests ?? data.completed ?? 0;
61
- const inputLen = Number(
62
- data.input_len ?? data.mean_input_len ?? data.total_input_tokens / Math.max(numRequests, 1) ?? 0,
63
- );
64
- const outputLen = Number(
65
- data.output_len ?? data.mean_output_len ?? data.total_output_tokens / Math.max(numRequests, 1) ?? 0,
66
- );
67
- const concurrency = data.concurrency ?? data.num_procs ?? 1;
68
-
69
- const confidenceTier: ConfidenceTier =
70
- modelId !== "unknown" && outputThroughput > 0 && numRequests > 0
71
- ? "community_reproducible"
72
- : "community_unverified";
73
-
74
- return {
75
- benchmark_id: generateBenchmarkId(),
76
- source_name: "tensorrt_llm",
77
- source_url: data.source_url || "https://nvidia.github.io/TensorRT-LLM/",
78
- source_license: "Apache-2.0",
79
- ingestion_method: "uploaded_raw_artifact",
80
- retrieved_at: new Date().toISOString(),
81
- parser_name: "trtllm-json",
82
- parser_version: "v1",
83
- confidence_tier: confidenceTier,
84
- verification_status: "schema_valid",
85
-
86
- model_id: modelId,
87
- model_display_name: modelDisplayName,
88
- model_revision: data.model_revision || "unspecified",
89
- architecture: data.architecture,
90
- parameter_count_b: data.parameter_count_b,
91
- context_window_tokens: data.max_model_len,
92
- quantization_format: data.quantization || data.dtype || "unspecified",
93
- weight_dtype: data.dtype,
94
- kv_cache_dtype: data.kv_cache_dtype || "float16",
95
-
96
- accelerator_vendor: "NVIDIA",
97
- accelerator_model: data.gpu_model || "unspecified",
98
- gpu_count: data.tensor_parallel_size || data.gpu_count || 1,
99
- memory_per_gpu_gb: data.gpu_memory || 0,
100
- driver_version: data.driver_version,
101
- cuda_rocm_metal_version: data.cuda_version,
102
-
103
- serving_engine: "TensorRT-LLM",
104
- engine_version: data.trtllm_version || "unspecified",
105
- engine_command: data.command,
106
- engine_config: {
107
- tensor_parallel_size: data.tensor_parallel_size,
108
- max_model_len: data.max_model_len,
109
- max_num_seqs: data.max_num_seqs,
110
- max_num_batched_tokens: data.max_num_batched_tokens,
111
- prefix_cache_enabled: data.enable_prefix_caching,
112
- chunked_prefill_enabled: data.enable_chunked_prefill,
113
- attention_backend: "TensorRT-LLM",
114
- },
115
-
116
- workload_type: data.workload_type || "online_poisson",
117
- dataset_name: data.dataset_name || data.dataset,
118
- request_count: numRequests,
119
- concurrency,
120
- input_tokens_mean: inputLen,
121
- output_tokens_mean: outputLen,
122
- streaming_enabled: data.streaming ?? true,
123
-
124
- successful_requests: numRequests,
125
- failed_requests: data.failed ?? 0,
126
- output_token_throughput_tps: outputThroughput,
127
- total_token_throughput_tps: data.total_throughput,
128
- request_throughput_rps: data.request_throughput,
129
- ttft_mean_ms: meanTtft,
130
- ttft_p50_ms: data.first_token_latency_p50,
131
- ttft_p95_ms: data.first_token_latency_p95,
132
- ttft_p99_ms: data.first_token_latency_p99,
133
- itl_mean_ms: meanItl,
134
- itl_p50_ms: data.inter_token_latency_p50,
135
- itl_p95_ms: data.inter_token_latency_p95,
136
- itl_p99_ms: data.inter_token_latency_p99,
137
- e2e_latency_p50_ms: data.latency_p50,
138
- e2e_latency_p95_ms: data.latency_p95,
139
- e2e_latency_p99_ms: data.latency_p99,
140
-
141
- gpu_memory_peak_gb: data.peak_gpu_memory,
142
-
143
- metric_definition_version: "trtllm-v1",
144
- citation_text: `TRT-LLM benchmark: ${outputThroughput.toFixed(2)} tok/s, TTFT mean ${meanTtft?.toFixed(1) || "?"} ms, ${numRequests} requests`,
145
- attribution_required: true,
146
- commercial_redistribution_allowed: "yes",
147
- };
148
- } catch {
149
- return null;
150
- }
151
- },
152
- };
@@ -1,198 +0,0 @@
1
- /**
2
- * vLLM benchmark JSON parser.
3
- *
4
- * vLLM's `benchmark_serving.py` produces JSON output with fields like:
5
- * - output_throughput: output tokens/s
6
- * - mean_ttft_ms: time to first token
7
- * - p99_ttft_ms
8
- * - mean_itl_ms: inter-token latency
9
- * - mean_tpot_ms: time per output token
10
- * - total_token_throughput
11
- * - duration
12
- * - completed / total_input_tokens / total_output_tokens
13
- *
14
- * Example vLLM benchmark output:
15
- * {
16
- * "elapsed_time": 100.5,
17
- * "num_input_tokens": 1024000,
18
- * "num_output_tokens": 256000,
19
- * "request_throughput": 1.73,
20
- * "output_throughput": 382.89,
21
- * "total_token_throughput": 619.85,
22
- * "mean_ttft_ms": 71.54,
23
- * "median_ttft_ms": 70.21,
24
- * "p99_ttft_ms": 79.49,
25
- * "mean_tpot_ms": 12.45,
26
- * "median_tpot_ms": 12.30,
27
- * "p99_tpot_ms": 14.87,
28
- * "mean_itl_ms": 7.74,
29
- * "median_itl_ms": 7.68,
30
- * "p99_itl_ms": 8.92,
31
- * "input_len": "1024",
32
- * "output_len": "256",
33
- * "num_requests": 1000,
34
- * "model_id": "meta-llama/Llama-3.1-70B-Instruct",
35
- * ...
36
- * }
37
- *
38
- * The actual field names vary by vLLM version. This parser handles the
39
- * common field aliases across v0.5.x → v0.29.x.
40
- */
41
-
42
- import type {
43
- BenchmarkParser,
44
- BenchmarkRecord,
45
- ConfidenceTier,
46
- } from "./benchmark-schema";
47
- import { generateBenchmarkId } from "./benchmark-schema";
48
-
49
- export const vllmParser: BenchmarkParser = {
50
- name: "vllm-json",
51
- version: "v1",
52
- formatLabel: "vLLM benchmark JSON",
53
-
54
- detect(rawText: string): boolean {
55
- try {
56
- const data = JSON.parse(rawText);
57
- // Check for vLLM-specific fields. Require at least 2 to avoid
58
- // catching TRT-LLM or SGLang output which may share `num_requests`.
59
- const vllmFields = [
60
- data.output_throughput,
61
- data.total_token_throughput,
62
- data.mean_ttft_ms,
63
- data.mean_itl_ms,
64
- data.mean_tpot_ms,
65
- data.request_throughput,
66
- ].filter((v) => v !== undefined && v !== null);
67
- return vllmFields.length >= 2;
68
- } catch {
69
- return false;
70
- }
71
- },
72
-
73
- parse(rawText: string): BenchmarkRecord | null {
74
- try {
75
- const data = JSON.parse(rawText);
76
- if (!data) return null;
77
-
78
- // Extract model info
79
- const modelId = data.model_id || data.model || "unknown";
80
- const modelDisplayName = modelId.split("/").pop() || modelId;
81
-
82
- // Extract throughput metrics (handle field name variations)
83
- const outputThroughput = data.output_throughput ?? data.output_token_throughput ?? 0;
84
- const totalThroughput = data.total_token_throughput ?? 0;
85
- const requestThroughput = data.request_throughput ?? 0;
86
-
87
- // Extract latency metrics
88
- const meanTtft = data.mean_ttft_ms ?? data.mean_ttft;
89
- const p50Ttft = data.median_ttft_ms ?? data.p50_ttft_ms ?? data.ttft_p50;
90
- const p95Ttft = data.p95_ttft_ms ?? data.ttft_p95;
91
- const p99Ttft = data.p99_ttft_ms ?? data.p99_ttft;
92
-
93
- const meanItl = data.mean_itl_ms ?? data.mean_itl;
94
- const p50Itl = data.median_itl_ms ?? data.p50_itl_ms ?? data.itl_p50;
95
- const p95Itl = data.p95_itl_ms ?? data.itl_p95;
96
- const p99Itl = data.p99_itl_ms ?? data.p99_itl;
97
-
98
- const meanTpot = data.mean_tpot_ms ?? data.mean_tpot;
99
- const p99Tpot = data.p99_tpot_ms ?? data.p99_tpot;
100
-
101
- // Extract workload info
102
- const numRequests = data.num_requests ?? data.completed ?? 0;
103
- const inputLen = Number(data.input_len ?? data.mean_input_len ?? data.input_tokens_mean ?? 0);
104
- const outputLen = Number(data.output_len ?? data.mean_output_len ?? data.output_tokens_mean ?? 0);
105
- const concurrency = data.num_procs ?? data.concurrency ?? 1;
106
-
107
- // Extract duration
108
- const elapsed = data.elapsed_time ?? data.duration ?? 0;
109
-
110
- // Determine confidence tier
111
- const hasFullConfig = !!data.model_id && !!data.output_throughput && numRequests > 0;
112
- const confidenceTier: ConfidenceTier = hasFullConfig
113
- ? "community_reproducible"
114
- : "community_unverified";
115
-
116
- const record: BenchmarkRecord = {
117
- benchmark_id: generateBenchmarkId(),
118
- source_name: "vllm",
119
- source_url: data.source_url || "https://docs.vllm.ai/en/latest/benchmarking/cli/",
120
- source_license: "Apache-2.0",
121
- ingestion_method: "uploaded_raw_artifact",
122
- retrieved_at: new Date().toISOString(),
123
- benchmark_started_at: data.timestamp || data.run_time,
124
- parser_name: "vllm-json",
125
- parser_version: "v1",
126
- confidence_tier: confidenceTier,
127
- verification_status: "schema_valid",
128
-
129
- model_id: modelId,
130
- model_display_name: modelDisplayName,
131
- model_revision: data.model_revision || data.commit || "unspecified",
132
- architecture: data.architecture || "dense",
133
- parameter_count_b: data.parameter_count_b,
134
- context_window_tokens: data.max_model_len,
135
- quantization_format: data.quantization || data.dtype || "unspecified",
136
- weight_dtype: data.dtype || data.weight_dtype,
137
- kv_cache_dtype: data.kv_cache_dtype || "float16",
138
-
139
- accelerator_vendor: data.gpu_vendor || "NVIDIA",
140
- accelerator_model: data.gpu_model || "unspecified",
141
- gpu_count: data.tensor_parallel_size || data.gpu_count || 1,
142
- memory_per_gpu_gb: data.gpu_memory || 0,
143
- driver_version: data.driver_version,
144
- cuda_rocm_metal_version: data.cuda_version,
145
-
146
- serving_engine: "vLLM",
147
- engine_version: data.vllm_version || "unspecified",
148
- engine_command: data.command,
149
- engine_config: {
150
- tensor_parallel_size: data.tensor_parallel_size,
151
- max_model_len: data.max_model_len,
152
- max_num_seqs: data.max_num_seqs,
153
- max_num_batched_tokens: data.max_num_batched_tokens,
154
- prefix_cache_enabled: data.enable_prefix_caching ?? data.prefix_cache_enabled,
155
- chunked_prefill_enabled: data.enable_chunked_prefill,
156
- attention_backend: data.attention_backend,
157
- },
158
-
159
- workload_type: data.workload_type || "online_poisson",
160
- dataset_name: data.dataset_name || data.dataset,
161
- request_count: numRequests,
162
- concurrency,
163
- input_tokens_mean: inputLen,
164
- output_tokens_mean: outputLen,
165
- streaming_enabled: data.streaming ?? true,
166
-
167
- successful_requests: numRequests,
168
- failed_requests: data.failed ?? (data.total_requests ? data.total_requests - numRequests : 0),
169
- request_throughput_rps: requestThroughput,
170
- output_token_throughput_tps: outputThroughput,
171
- total_token_throughput_tps: totalThroughput,
172
- ttft_mean_ms: meanTtft,
173
- ttft_p50_ms: p50Ttft,
174
- ttft_p95_ms: p95Ttft,
175
- ttft_p99_ms: p99Ttft,
176
- itl_mean_ms: meanItl,
177
- itl_p50_ms: p50Itl,
178
- itl_p95_ms: p95Itl,
179
- itl_p99_ms: p99Itl,
180
- tpot_mean_ms: meanTpot,
181
- e2e_latency_p99_ms: data.mean_lantecy_ms || data.e2e_latency_p99_ms,
182
-
183
- gpu_memory_peak_gb: data.peak_gpu_memory || data.gpu_memory_peak_gb,
184
- gpu_utilization_mean_pct: data.gpu_utilization,
185
-
186
- metric_definition_version: "vllm-v1",
187
- citation_text: `vLLM benchmark: ${outputThroughput.toFixed(2)} output tok/s, TTFT mean ${meanTtft?.toFixed(1) || "?"} ms, ${numRequests} requests`,
188
- attribution_required: true,
189
- commercial_redistribution_allowed: "yes",
190
- };
191
-
192
- return record;
193
- } catch (e) {
194
- console.error("vLLM parse error:", e);
195
- return null;
196
- }
197
- },
198
- };