@tokcalc/mcp-server 0.1.3 → 0.2.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -406
- package/dist/http.js +23504 -0
- package/dist/index.js +21290 -0
- package/package.json +36 -92
- package/.zscripts/build.sh +0 -175
- package/.zscripts/database-runtime-build.sh +0 -33
- package/.zscripts/dev.pid +0 -1
- package/.zscripts/dev.sh +0 -154
- package/.zscripts/mini-services-build.sh +0 -78
- package/.zscripts/mini-services-install.sh +0 -65
- package/.zscripts/mini-services-start.sh +0 -123
- package/.zscripts/python-runtime-build.sh +0 -120
- package/.zscripts/start.sh +0 -145
- package/CAPACITY_STUDY.md +0 -283
- package/CODE_OF_CONDUCT.md +0 -55
- package/CONTRIBUTING.md +0 -177
- package/Caddyfile +0 -23
- package/LICENSE +0 -204
- package/bun.lock +0 -1965
- package/components.json +0 -21
- package/db/custom.db +0 -0
- package/download/README.md +0 -1
- package/download/tokcalc-dark-calculator.png +0 -0
- package/download/tokcalc-dark-default.png +0 -0
- package/download/tokcalc-demo.webm +0 -0
- package/download/tokcalc-github-link.png +0 -0
- package/download/tokcalc-hydration-fixed.png +0 -0
- package/download/tokcalc-issue-resolved.png +0 -0
- package/download/tokcalc-light-mode.png +0 -0
- package/download/tokcalc-light-reference.png +0 -0
- package/download/tokcalc-long-context-qwen.png +0 -0
- package/download/tokcalc-long-context.png +0 -0
- package/download/tokcalc-og-image-preview.png +0 -0
- package/download/tokcalc-phase2-3.png +0 -0
- package/download/tokcalc-plain-english.png +0 -0
- package/download/tokcalc-preview.png +0 -0
- package/download/tokcalc-share-bvb.png +0 -0
- package/download/tokcalc-share-feature.png +0 -0
- package/download/tokcalc-tab-build-vs-buy.png +0 -0
- package/download/tokcalc-tab-calculator.png +0 -0
- package/download/tokcalc-tab-reference.png +0 -0
- package/eslint.config.mjs +0 -50
- package/examples/websocket/frontend.tsx +0 -196
- package/examples/websocket/server.ts +0 -138
- package/mini-services/.gitkeep +0 -0
- package/mini-services/mcp-server/README.md +0 -86
- package/mini-services/mcp-server/bun.lock +0 -202
- package/mini-services/mcp-server/index.ts +0 -504
- package/mini-services/mcp-server/package.json +0 -40
- package/next.config.ts +0 -12
- package/postcss.config.mjs +0 -5
- package/prisma/schema.prisma +0 -32
- package/public/google6f58ca6be85fa903.html +0 -1
- package/public/logo.svg +0 -29
- package/public/manifest.json +0 -51
- package/public/og-icon-256.png +0 -0
- package/public/og.png +0 -0
- package/public/robots.txt +0 -25
- package/public/sitemap.xml +0 -23
- package/public/tokcalc-demo.gif +0 -0
- package/scripts/og-template.html +0 -120
- package/scripts/render-og.mjs +0 -43
- package/server.json +0 -21
- package/src/app/api/pricing/aws/route.ts +0 -186
- package/src/app/api/pricing/azure/route.ts +0 -168
- package/src/app/api/pricing/gcp/route.ts +0 -230
- package/src/app/api/pricing/vast-ai/route.ts +0 -164
- package/src/app/api/route.ts +0 -5
- package/src/app/compare/h100-vs-h200/layout.tsx +0 -30
- package/src/app/compare/h100-vs-h200/page.tsx +0 -328
- package/src/app/globals.css +0 -122
- package/src/app/layout.tsx +0 -276
- package/src/app/page.tsx +0 -2670
- package/src/components/azure-live-pricing.tsx +0 -185
- package/src/components/benchmark-import.tsx +0 -340
- package/src/components/confidence-badge.tsx +0 -116
- package/src/components/live-pricing-comparison.tsx +0 -241
- package/src/components/theme-provider.tsx +0 -11
- package/src/components/theme-toggle.tsx +0 -55
- package/src/components/ui/accordion.tsx +0 -66
- package/src/components/ui/alert-dialog.tsx +0 -157
- package/src/components/ui/alert.tsx +0 -66
- package/src/components/ui/aspect-ratio.tsx +0 -11
- package/src/components/ui/avatar.tsx +0 -53
- package/src/components/ui/badge.tsx +0 -46
- package/src/components/ui/breadcrumb.tsx +0 -109
- package/src/components/ui/button.tsx +0 -59
- package/src/components/ui/calendar.tsx +0 -213
- package/src/components/ui/card.tsx +0 -92
- package/src/components/ui/carousel.tsx +0 -241
- package/src/components/ui/chart.tsx +0 -353
- package/src/components/ui/checkbox.tsx +0 -32
- package/src/components/ui/collapsible.tsx +0 -33
- package/src/components/ui/command.tsx +0 -184
- package/src/components/ui/context-menu.tsx +0 -252
- package/src/components/ui/dialog.tsx +0 -143
- package/src/components/ui/drawer.tsx +0 -135
- package/src/components/ui/dropdown-menu.tsx +0 -257
- package/src/components/ui/form.tsx +0 -167
- package/src/components/ui/hover-card.tsx +0 -44
- package/src/components/ui/input-otp.tsx +0 -77
- package/src/components/ui/input.tsx +0 -21
- package/src/components/ui/label.tsx +0 -24
- package/src/components/ui/menubar.tsx +0 -276
- package/src/components/ui/navigation-menu.tsx +0 -168
- package/src/components/ui/pagination.tsx +0 -127
- package/src/components/ui/popover.tsx +0 -48
- package/src/components/ui/progress.tsx +0 -31
- package/src/components/ui/radio-group.tsx +0 -45
- package/src/components/ui/resizable.tsx +0 -56
- package/src/components/ui/scroll-area.tsx +0 -58
- package/src/components/ui/select.tsx +0 -185
- package/src/components/ui/separator.tsx +0 -28
- package/src/components/ui/sheet.tsx +0 -139
- package/src/components/ui/sidebar.tsx +0 -726
- package/src/components/ui/skeleton.tsx +0 -13
- package/src/components/ui/slider.tsx +0 -63
- package/src/components/ui/sonner.tsx +0 -25
- package/src/components/ui/switch.tsx +0 -31
- package/src/components/ui/table.tsx +0 -116
- package/src/components/ui/tabs.tsx +0 -66
- package/src/components/ui/textarea.tsx +0 -18
- package/src/components/ui/toast.tsx +0 -129
- package/src/components/ui/toaster.tsx +0 -35
- package/src/components/ui/toggle-group.tsx +0 -73
- package/src/components/ui/toggle.tsx +0 -47
- package/src/components/ui/tooltip.tsx +0 -61
- package/src/components/vast-ai-live-pricing.tsx +0 -176
- package/src/hooks/use-mobile.ts +0 -19
- package/src/hooks/use-toast.ts +0 -194
- package/src/lib/benchmark-parser-sglang.ts +0 -150
- package/src/lib/benchmark-parser-tokcalc.ts +0 -247
- package/src/lib/benchmark-parser-trtllm.ts +0 -152
- package/src/lib/benchmark-parser-vllm.ts +0 -198
- package/src/lib/benchmark-schema.ts +0 -263
- package/src/lib/db.ts +0 -13
- package/src/lib/engine-presets.ts +0 -183
- package/src/lib/price-schema.ts +0 -141
- package/src/lib/token-calc.ts +0 -808
- package/src/lib/track.ts +0 -31
- package/src/lib/url-state.ts +0 -256
- package/src/lib/utils.ts +0 -6
- package/tailwind.config.ts +0 -64
- package/tests/database-runtime-build.sh +0 -75
- package/tests/python-runtime-build.sh +0 -64
- package/tests/python-runtime-container.sh +0 -31
- package/tool-results/bash_1789888171144_2c5381860539.txt +0 -161
- package/tool-results/bash_1789888175925_49c53ba3c61b.txt +0 -191
- package/tool-results/bash_1789888181202_49c53ba3c61b.txt +0 -191
- package/tool-results/bash_1789888195219_4a86a5c91411.txt +0 -200
- package/tool-results/bash_1789888203128_6cca13c71b47.txt +0 -199
- package/tool-results/bash_1789929256963_2a52aff0d0a8.txt +0 -160
- package/tool-results/read_1789888151021_69f58eec6a5b.txt +0 -653
- package/tool-results/read_1789888153837_1d3a8bfc2a94.txt +0 -653
- package/tool-results/read_1789888163087_ccc406d47505.txt +0 -122
- package/tool-results/read_1789888167347_67d1d7c9830a.txt +0 -122
- package/tool-results/read_1789929252529_d90e8f383a25.txt +0 -285
- package/tsconfig.json +0 -42
- package/upload/Pasted Content_1789887800864.txt +0 -652
- package/upload/Pasted Content_1789887909561.txt +0 -652
- package/upload/Pasted Content_1789887918428.txt +0 -652
- package/upload/Pasted Content_1789887959420.txt +0 -652
- package/upload/Pasted Content_1789888020485.txt +0 -652
- package/upload/Pasted Content_1789888058079.txt +0 -652
- package/upload/Pasted Content_1789888885033.txt +0 -686
- package/upload/Pasted Content_1789928912741.txt +0 -285
- package/upload/Pasted Content_1789928938402.txt +0 -285
- package/upload/Pasted Content_1789929160389.txt +0 -285
- package/upload/Pasted Content_1789929176660.txt +0 -285
- package/upload/issue_vision.json +0 -28
- package/upload/pasted_image_1789883175209.png +0 -0
- package/upload/pasted_image_1789899056690.png +0 -0
- package/upload/pasted_image_1789900371483.png +0 -0
- package/upload/pasted_image_1789900472823.png +0 -0
- package/upload/pasted_image_1789900490374.png +0 -0
- package/upload/pasted_image_1789900585552.png +0 -0
- package/upload/pasted_image_1789900606519.png +0 -0
- package/upload/pasted_image_1789901598705.png +0 -0
- package/upload/pasted_image_1789901613545.png +0 -0
- package/upload/pasted_image_1789978382674.png +0 -0
- package/upload/pasted_image_1789978392749.png +0 -0
- package/upload/pasted_image_1789978474879.png +0 -0
- package/upload/pasted_image_1789978523652.png +0 -0
- package/upload/pasted_image_1789984219089.png +0 -0
- package/upload/pasted_image_1789984491896.png +0 -0
- package/upload/pasted_image_1789985017950.png +0 -0
- package/upload/pasted_image_1789985036765.png +0 -0
- package/upload/pasted_image_1789985049848.png +0 -0
- package/upload/pasted_image_1790002427833.png +0 -0
- package/upload/pasted_image_1790002659944.png +0 -0
- package/upload/pasted_image_1790037038476.png +0 -0
- package/upload/screenshot_analysis.json +0 -28
- package/upload/vision_output.json +0 -28
|
@@ -1,247 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Community benchmark submission template.
|
|
3
|
-
*
|
|
4
|
-
* Users download this JSON file, fill in their benchmark results,
|
|
5
|
-
* and paste it into tokcalc's BenchmarkImport textarea.
|
|
6
|
-
*
|
|
7
|
-
* The tokcalc-standard parser recognizes this format and creates
|
|
8
|
-
* a BenchmarkRecord with confidence_tier = "community_reproducible"
|
|
9
|
-
* (if all required fields are filled) or "community_unverified" (if some are missing).
|
|
10
|
-
*
|
|
11
|
-
* This is the "tier 4" ingestion path from the Perplexity research:
|
|
12
|
-
* "A tokcalc-standard JSON schema for community uploads."
|
|
13
|
-
*/
|
|
14
|
-
|
|
15
|
-
export const BENCHMARK_TEMPLATE = `{
|
|
16
|
-
"format": "tokcalc-standard",
|
|
17
|
-
"version": "1.0",
|
|
18
|
-
|
|
19
|
-
"source": {
|
|
20
|
-
"submitter": "your-github-username",
|
|
21
|
-
"submit_date": "2026-09-24",
|
|
22
|
-
"raw_artifact_url": "https://github.com/your-username/your-repo/blob/main/benchmark-results.json",
|
|
23
|
-
"notes": "Describe how you ran this benchmark. Include the exact command you used."
|
|
24
|
-
},
|
|
25
|
-
|
|
26
|
-
"model": {
|
|
27
|
-
"model_id": "meta-llama/Llama-3.3-70B-Instruct",
|
|
28
|
-
"model_revision": "commit hash or release date",
|
|
29
|
-
"quantization_format": "FP8",
|
|
30
|
-
"weight_dtype": "float8_e4m3",
|
|
31
|
-
"kv_cache_dtype": "float16"
|
|
32
|
-
},
|
|
33
|
-
|
|
34
|
-
"hardware": {
|
|
35
|
-
"accelerator_vendor": "NVIDIA",
|
|
36
|
-
"accelerator_model": "H100 SXM5 80GB",
|
|
37
|
-
"gpu_count": 1,
|
|
38
|
-
"memory_per_gpu_gb": 80,
|
|
39
|
-
"interconnect": "NVLink",
|
|
40
|
-
"cloud_provider": "RunPod",
|
|
41
|
-
"instance_type": "H100 80GB SXM",
|
|
42
|
-
"region": "us-east-1",
|
|
43
|
-
"driver_version": "550.54.15",
|
|
44
|
-
"cuda_version": "12.4"
|
|
45
|
-
},
|
|
46
|
-
|
|
47
|
-
"engine": {
|
|
48
|
-
"serving_engine": "vLLM",
|
|
49
|
-
"engine_version": "0.29.0",
|
|
50
|
-
"tensor_parallel_size": 1,
|
|
51
|
-
"max_model_len": 32768,
|
|
52
|
-
"max_num_seqs": 256,
|
|
53
|
-
"max_num_batched_tokens": 8192,
|
|
54
|
-
"prefix_cache_enabled": true,
|
|
55
|
-
"chunked_prefill_enabled": true,
|
|
56
|
-
"attention_backend": "FlashInfer",
|
|
57
|
-
"command": "vllm serve meta-llama/Llama-3.3-70B-Instruct --quantization fp8 --tensor-parallel-size 1 --max-model-len 32768 --max-num-seqs 256"
|
|
58
|
-
},
|
|
59
|
-
|
|
60
|
-
"workload": {
|
|
61
|
-
"workload_type": "online_poisson",
|
|
62
|
-
"dataset_name": "ShareGPT",
|
|
63
|
-
"request_count": 1000,
|
|
64
|
-
"concurrency": 64,
|
|
65
|
-
"input_tokens_mean": 1024,
|
|
66
|
-
"input_tokens_p95": 4096,
|
|
67
|
-
"output_tokens_mean": 256,
|
|
68
|
-
"output_tokens_p95": 1024,
|
|
69
|
-
"streaming_enabled": true,
|
|
70
|
-
"sampling_config": {
|
|
71
|
-
"temperature": 0.7,
|
|
72
|
-
"top_p": 0.95,
|
|
73
|
-
"max_tokens": 512
|
|
74
|
-
}
|
|
75
|
-
},
|
|
76
|
-
|
|
77
|
-
"metrics": {
|
|
78
|
-
"successful_requests": 1000,
|
|
79
|
-
"failed_requests": 0,
|
|
80
|
-
"output_token_throughput_tps": 2450.5,
|
|
81
|
-
"total_token_throughput_tps": 11000.0,
|
|
82
|
-
"request_throughput_rps": 3.91,
|
|
83
|
-
"ttft_mean_ms": 112.3,
|
|
84
|
-
"ttft_p50_ms": 108.5,
|
|
85
|
-
"ttft_p95_ms": 145.2,
|
|
86
|
-
"ttft_p99_ms": 189.7,
|
|
87
|
-
"itl_mean_ms": 5.67,
|
|
88
|
-
"itl_p50_ms": 5.45,
|
|
89
|
-
"itl_p95_ms": 7.23,
|
|
90
|
-
"itl_p99_ms": 8.91,
|
|
91
|
-
"gpu_memory_peak_gb": 78.5,
|
|
92
|
-
"gpu_utilization_mean_pct": 92.5
|
|
93
|
-
}
|
|
94
|
-
}`;
|
|
95
|
-
|
|
96
|
-
/**
|
|
97
|
-
* Parse a tokcalc-standard benchmark JSON.
|
|
98
|
-
* This is the community submission format — users fill in the template
|
|
99
|
-
* and paste it into the BenchmarkImport textarea.
|
|
100
|
-
*/
|
|
101
|
-
import type {
|
|
102
|
-
BenchmarkParser,
|
|
103
|
-
BenchmarkRecord,
|
|
104
|
-
ConfidenceTier,
|
|
105
|
-
} from "./benchmark-schema";
|
|
106
|
-
import { generateBenchmarkId } from "./benchmark-schema";
|
|
107
|
-
|
|
108
|
-
export const tokcalcStandardParser: BenchmarkParser = {
|
|
109
|
-
name: "tokcalc-standard",
|
|
110
|
-
version: "v1",
|
|
111
|
-
formatLabel: "tokcalc community benchmark",
|
|
112
|
-
|
|
113
|
-
detect(rawText: string): boolean {
|
|
114
|
-
try {
|
|
115
|
-
const data = JSON.parse(rawText);
|
|
116
|
-
return data?.format === "tokcalc-standard" || !!data?.metrics?.output_token_throughput_tps;
|
|
117
|
-
} catch {
|
|
118
|
-
return false;
|
|
119
|
-
}
|
|
120
|
-
},
|
|
121
|
-
|
|
122
|
-
parse(rawText: string): BenchmarkRecord | null {
|
|
123
|
-
try {
|
|
124
|
-
const data = JSON.parse(rawText);
|
|
125
|
-
if (!data) return null;
|
|
126
|
-
|
|
127
|
-
const model = data.model || {};
|
|
128
|
-
const hardware = data.hardware || {};
|
|
129
|
-
const engine = data.engine || {};
|
|
130
|
-
const workload = data.workload || {};
|
|
131
|
-
const metrics = data.metrics || {};
|
|
132
|
-
const source = data.source || {};
|
|
133
|
-
|
|
134
|
-
const modelId = model.model_id || "unknown";
|
|
135
|
-
const modelDisplayName = modelId.split("/").pop() || modelId;
|
|
136
|
-
|
|
137
|
-
// Determine confidence tier based on completeness
|
|
138
|
-
const hasFullConfig =
|
|
139
|
-
modelId !== "unknown" &&
|
|
140
|
-
hardware.accelerator_model &&
|
|
141
|
-
engine.serving_engine &&
|
|
142
|
-
metrics.output_token_throughput_tps > 0 &&
|
|
143
|
-
workload.request_count > 0;
|
|
144
|
-
const confidenceTier: ConfidenceTier = hasFullConfig
|
|
145
|
-
? "community_reproducible"
|
|
146
|
-
: "community_unverified";
|
|
147
|
-
|
|
148
|
-
return {
|
|
149
|
-
benchmark_id: generateBenchmarkId(),
|
|
150
|
-
source_name: "community",
|
|
151
|
-
source_url: source.raw_artifact_url || "",
|
|
152
|
-
source_license: "CC-BY-SA-4.0",
|
|
153
|
-
ingestion_method: "uploaded_raw_artifact",
|
|
154
|
-
retrieved_at: new Date().toISOString(),
|
|
155
|
-
benchmark_started_at: source.submit_date,
|
|
156
|
-
raw_artifact_url: source.raw_artifact_url,
|
|
157
|
-
parser_name: "tokcalc-standard",
|
|
158
|
-
parser_version: "v1",
|
|
159
|
-
confidence_tier: confidenceTier,
|
|
160
|
-
verification_status: "schema_valid",
|
|
161
|
-
|
|
162
|
-
model_id: modelId,
|
|
163
|
-
model_display_name: modelDisplayName,
|
|
164
|
-
model_revision: model.model_revision || "unspecified",
|
|
165
|
-
architecture: model.architecture,
|
|
166
|
-
parameter_count_b: model.parameter_count_b,
|
|
167
|
-
context_window_tokens: engine.max_model_len,
|
|
168
|
-
quantization_format: model.quantization_format || "unspecified",
|
|
169
|
-
weight_dtype: model.weight_dtype,
|
|
170
|
-
kv_cache_dtype: model.kv_cache_dtype || "float16",
|
|
171
|
-
|
|
172
|
-
accelerator_vendor: hardware.accelerator_vendor || "NVIDIA",
|
|
173
|
-
accelerator_model: hardware.accelerator_model || "unspecified",
|
|
174
|
-
gpu_count: hardware.gpu_count || engine.tensor_parallel_size || 1,
|
|
175
|
-
memory_per_gpu_gb: hardware.memory_per_gpu_gb || 0,
|
|
176
|
-
host_cpu: hardware.host_cpu,
|
|
177
|
-
host_memory_gb: hardware.host_memory_gb,
|
|
178
|
-
interconnect: hardware.interconnect,
|
|
179
|
-
topology: hardware.topology,
|
|
180
|
-
cloud_provider: hardware.cloud_provider,
|
|
181
|
-
instance_type: hardware.instance_type,
|
|
182
|
-
region: hardware.region,
|
|
183
|
-
driver_version: hardware.driver_version,
|
|
184
|
-
cuda_rocm_metal_version: hardware.cuda_version,
|
|
185
|
-
|
|
186
|
-
serving_engine: engine.serving_engine || "unspecified",
|
|
187
|
-
engine_version: engine.engine_version || "unspecified",
|
|
188
|
-
engine_command: engine.command,
|
|
189
|
-
engine_config: {
|
|
190
|
-
tensor_parallel_size: engine.tensor_parallel_size,
|
|
191
|
-
max_model_len: engine.max_model_len,
|
|
192
|
-
max_num_seqs: engine.max_num_seqs,
|
|
193
|
-
max_num_batched_tokens: engine.max_num_batched_tokens,
|
|
194
|
-
prefix_cache_enabled: engine.prefix_cache_enabled,
|
|
195
|
-
chunked_prefill_enabled: engine.chunked_prefill_enabled,
|
|
196
|
-
attention_backend: engine.attention_backend,
|
|
197
|
-
},
|
|
198
|
-
|
|
199
|
-
workload_type: workload.workload_type || "online_poisson",
|
|
200
|
-
dataset_name: workload.dataset_name,
|
|
201
|
-
request_count: workload.request_count || 0,
|
|
202
|
-
concurrency: workload.concurrency || 1,
|
|
203
|
-
arrival_rate_rps: workload.arrival_rate_rps,
|
|
204
|
-
input_tokens_mean: workload.input_tokens_mean || 0,
|
|
205
|
-
input_tokens_p50: workload.input_tokens_p50,
|
|
206
|
-
input_tokens_p95: workload.input_tokens_p95,
|
|
207
|
-
input_tokens_p99: workload.input_tokens_p99,
|
|
208
|
-
output_tokens_mean: workload.output_tokens_mean || 0,
|
|
209
|
-
output_tokens_p50: workload.output_tokens_p50,
|
|
210
|
-
output_tokens_p95: workload.output_tokens_p95,
|
|
211
|
-
output_tokens_p99: workload.output_tokens_p99,
|
|
212
|
-
shared_prefix_tokens_mean: workload.shared_prefix_tokens_mean,
|
|
213
|
-
shared_prefix_hit_rate: workload.shared_prefix_hit_rate,
|
|
214
|
-
streaming_enabled: workload.streaming_enabled ?? true,
|
|
215
|
-
sampling_config: workload.sampling_config,
|
|
216
|
-
|
|
217
|
-
successful_requests: metrics.successful_requests ?? workload.request_count ?? 0,
|
|
218
|
-
failed_requests: metrics.failed_requests ?? 0,
|
|
219
|
-
request_throughput_rps: metrics.request_throughput_rps,
|
|
220
|
-
output_token_throughput_tps: metrics.output_token_throughput_tps || 0,
|
|
221
|
-
total_token_throughput_tps: metrics.total_token_throughput_tps,
|
|
222
|
-
ttft_mean_ms: metrics.ttft_mean_ms,
|
|
223
|
-
ttft_p50_ms: metrics.ttft_p50,
|
|
224
|
-
ttft_p95_ms: metrics.ttft_p95_ms,
|
|
225
|
-
ttft_p99_ms: metrics.ttft_p99_ms,
|
|
226
|
-
itl_mean_ms: metrics.itl_mean_ms,
|
|
227
|
-
itl_p50_ms: metrics.itl_p50,
|
|
228
|
-
itl_p95_ms: metrics.itl_p95_ms,
|
|
229
|
-
itl_p99_ms: metrics.itl_p99_ms,
|
|
230
|
-
e2e_latency_p50_ms: metrics.e2e_latency_p50_ms,
|
|
231
|
-
e2e_latency_p95_ms: metrics.e2e_latency_p95_ms,
|
|
232
|
-
e2e_latency_p99_ms: metrics.e2e_latency_p99_ms,
|
|
233
|
-
gpu_memory_peak_gb: metrics.gpu_memory_peak_gb,
|
|
234
|
-
gpu_utilization_mean_pct: metrics.gpu_utilization_mean_pct,
|
|
235
|
-
power_mean_w: metrics.power_mean_w,
|
|
236
|
-
energy_per_output_token_j: metrics.energy_per_output_token_j,
|
|
237
|
-
|
|
238
|
-
metric_definition_version: "tokcalc-v1",
|
|
239
|
-
citation_text: `Community benchmark: ${metrics.output_token_throughput_tps || 0} tok/s, ${workload.request_count || 0} requests, submitted by ${source.submitter || "anonymous"}`,
|
|
240
|
-
attribution_required: true,
|
|
241
|
-
commercial_redistribution_allowed: "yes",
|
|
242
|
-
};
|
|
243
|
-
} catch {
|
|
244
|
-
return null;
|
|
245
|
-
}
|
|
246
|
-
},
|
|
247
|
-
};
|
|
@@ -1,152 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* TensorRT-LLM benchmark parser.
|
|
3
|
-
*
|
|
4
|
-
* TRT-LLM's `benchmark_serving.py` produces JSON output with fields like:
|
|
5
|
-
* - throughput (tokens/s)
|
|
6
|
-
* - latency_avg (ms) — end-to-end latency
|
|
7
|
-
* - first_token_latency_avg (ms) — TTFT
|
|
8
|
-
* - inter_token_latency_avg (ms) — ITL
|
|
9
|
-
* - total_input_tokens, total_output_tokens
|
|
10
|
-
* - num_requests
|
|
11
|
-
* - model
|
|
12
|
-
*
|
|
13
|
-
* Refs: https://nvidia.github.io/TensorRT-LLM/1.3.0rc4/developer-guide/perf-benchmarking.html
|
|
14
|
-
*/
|
|
15
|
-
|
|
16
|
-
import type {
|
|
17
|
-
BenchmarkParser,
|
|
18
|
-
BenchmarkRecord,
|
|
19
|
-
ConfidenceTier,
|
|
20
|
-
} from "./benchmark-schema";
|
|
21
|
-
import { generateBenchmarkId } from "./benchmark-schema";
|
|
22
|
-
|
|
23
|
-
export const trtllmParser: BenchmarkParser = {
|
|
24
|
-
name: "trtllm-json",
|
|
25
|
-
version: "v1",
|
|
26
|
-
formatLabel: "TensorRT-LLM benchmark JSON",
|
|
27
|
-
|
|
28
|
-
detect(rawText: string): boolean {
|
|
29
|
-
try {
|
|
30
|
-
const data = JSON.parse(rawText);
|
|
31
|
-
return !!(
|
|
32
|
-
data &&
|
|
33
|
-
(data.throughput ||
|
|
34
|
-
data.first_token_latency_avg ||
|
|
35
|
-
data.inter_token_latency_avg ||
|
|
36
|
-
data.latency_avg ||
|
|
37
|
-
data.total_input_tokens ||
|
|
38
|
-
data.total_output_tokens)
|
|
39
|
-
);
|
|
40
|
-
} catch {
|
|
41
|
-
return false;
|
|
42
|
-
}
|
|
43
|
-
},
|
|
44
|
-
|
|
45
|
-
parse(rawText: string): BenchmarkRecord | null {
|
|
46
|
-
try {
|
|
47
|
-
const data = JSON.parse(rawText);
|
|
48
|
-
if (!data) return null;
|
|
49
|
-
|
|
50
|
-
const modelId = data.model || data.model_id || data.model_name || "unknown";
|
|
51
|
-
const modelDisplayName = modelId.split("/").pop() || modelId;
|
|
52
|
-
|
|
53
|
-
const outputThroughput = data.throughput ?? data.output_throughput ?? 0;
|
|
54
|
-
const meanTtft =
|
|
55
|
-
data.first_token_latency_avg ?? data.ttft_mean ?? data.mean_ttft_ms;
|
|
56
|
-
const meanItl =
|
|
57
|
-
data.inter_token_latency_avg ?? data.itl_mean ?? data.mean_itl_ms;
|
|
58
|
-
const meanE2E = data.latency_avg ?? data.e2e_latency_mean;
|
|
59
|
-
|
|
60
|
-
const numRequests = data.num_requests ?? data.completed ?? 0;
|
|
61
|
-
const inputLen = Number(
|
|
62
|
-
data.input_len ?? data.mean_input_len ?? data.total_input_tokens / Math.max(numRequests, 1) ?? 0,
|
|
63
|
-
);
|
|
64
|
-
const outputLen = Number(
|
|
65
|
-
data.output_len ?? data.mean_output_len ?? data.total_output_tokens / Math.max(numRequests, 1) ?? 0,
|
|
66
|
-
);
|
|
67
|
-
const concurrency = data.concurrency ?? data.num_procs ?? 1;
|
|
68
|
-
|
|
69
|
-
const confidenceTier: ConfidenceTier =
|
|
70
|
-
modelId !== "unknown" && outputThroughput > 0 && numRequests > 0
|
|
71
|
-
? "community_reproducible"
|
|
72
|
-
: "community_unverified";
|
|
73
|
-
|
|
74
|
-
return {
|
|
75
|
-
benchmark_id: generateBenchmarkId(),
|
|
76
|
-
source_name: "tensorrt_llm",
|
|
77
|
-
source_url: data.source_url || "https://nvidia.github.io/TensorRT-LLM/",
|
|
78
|
-
source_license: "Apache-2.0",
|
|
79
|
-
ingestion_method: "uploaded_raw_artifact",
|
|
80
|
-
retrieved_at: new Date().toISOString(),
|
|
81
|
-
parser_name: "trtllm-json",
|
|
82
|
-
parser_version: "v1",
|
|
83
|
-
confidence_tier: confidenceTier,
|
|
84
|
-
verification_status: "schema_valid",
|
|
85
|
-
|
|
86
|
-
model_id: modelId,
|
|
87
|
-
model_display_name: modelDisplayName,
|
|
88
|
-
model_revision: data.model_revision || "unspecified",
|
|
89
|
-
architecture: data.architecture,
|
|
90
|
-
parameter_count_b: data.parameter_count_b,
|
|
91
|
-
context_window_tokens: data.max_model_len,
|
|
92
|
-
quantization_format: data.quantization || data.dtype || "unspecified",
|
|
93
|
-
weight_dtype: data.dtype,
|
|
94
|
-
kv_cache_dtype: data.kv_cache_dtype || "float16",
|
|
95
|
-
|
|
96
|
-
accelerator_vendor: "NVIDIA",
|
|
97
|
-
accelerator_model: data.gpu_model || "unspecified",
|
|
98
|
-
gpu_count: data.tensor_parallel_size || data.gpu_count || 1,
|
|
99
|
-
memory_per_gpu_gb: data.gpu_memory || 0,
|
|
100
|
-
driver_version: data.driver_version,
|
|
101
|
-
cuda_rocm_metal_version: data.cuda_version,
|
|
102
|
-
|
|
103
|
-
serving_engine: "TensorRT-LLM",
|
|
104
|
-
engine_version: data.trtllm_version || "unspecified",
|
|
105
|
-
engine_command: data.command,
|
|
106
|
-
engine_config: {
|
|
107
|
-
tensor_parallel_size: data.tensor_parallel_size,
|
|
108
|
-
max_model_len: data.max_model_len,
|
|
109
|
-
max_num_seqs: data.max_num_seqs,
|
|
110
|
-
max_num_batched_tokens: data.max_num_batched_tokens,
|
|
111
|
-
prefix_cache_enabled: data.enable_prefix_caching,
|
|
112
|
-
chunked_prefill_enabled: data.enable_chunked_prefill,
|
|
113
|
-
attention_backend: "TensorRT-LLM",
|
|
114
|
-
},
|
|
115
|
-
|
|
116
|
-
workload_type: data.workload_type || "online_poisson",
|
|
117
|
-
dataset_name: data.dataset_name || data.dataset,
|
|
118
|
-
request_count: numRequests,
|
|
119
|
-
concurrency,
|
|
120
|
-
input_tokens_mean: inputLen,
|
|
121
|
-
output_tokens_mean: outputLen,
|
|
122
|
-
streaming_enabled: data.streaming ?? true,
|
|
123
|
-
|
|
124
|
-
successful_requests: numRequests,
|
|
125
|
-
failed_requests: data.failed ?? 0,
|
|
126
|
-
output_token_throughput_tps: outputThroughput,
|
|
127
|
-
total_token_throughput_tps: data.total_throughput,
|
|
128
|
-
request_throughput_rps: data.request_throughput,
|
|
129
|
-
ttft_mean_ms: meanTtft,
|
|
130
|
-
ttft_p50_ms: data.first_token_latency_p50,
|
|
131
|
-
ttft_p95_ms: data.first_token_latency_p95,
|
|
132
|
-
ttft_p99_ms: data.first_token_latency_p99,
|
|
133
|
-
itl_mean_ms: meanItl,
|
|
134
|
-
itl_p50_ms: data.inter_token_latency_p50,
|
|
135
|
-
itl_p95_ms: data.inter_token_latency_p95,
|
|
136
|
-
itl_p99_ms: data.inter_token_latency_p99,
|
|
137
|
-
e2e_latency_p50_ms: data.latency_p50,
|
|
138
|
-
e2e_latency_p95_ms: data.latency_p95,
|
|
139
|
-
e2e_latency_p99_ms: data.latency_p99,
|
|
140
|
-
|
|
141
|
-
gpu_memory_peak_gb: data.peak_gpu_memory,
|
|
142
|
-
|
|
143
|
-
metric_definition_version: "trtllm-v1",
|
|
144
|
-
citation_text: `TRT-LLM benchmark: ${outputThroughput.toFixed(2)} tok/s, TTFT mean ${meanTtft?.toFixed(1) || "?"} ms, ${numRequests} requests`,
|
|
145
|
-
attribution_required: true,
|
|
146
|
-
commercial_redistribution_allowed: "yes",
|
|
147
|
-
};
|
|
148
|
-
} catch {
|
|
149
|
-
return null;
|
|
150
|
-
}
|
|
151
|
-
},
|
|
152
|
-
};
|
|
@@ -1,198 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* vLLM benchmark JSON parser.
|
|
3
|
-
*
|
|
4
|
-
* vLLM's `benchmark_serving.py` produces JSON output with fields like:
|
|
5
|
-
* - output_throughput: output tokens/s
|
|
6
|
-
* - mean_ttft_ms: time to first token
|
|
7
|
-
* - p99_ttft_ms
|
|
8
|
-
* - mean_itl_ms: inter-token latency
|
|
9
|
-
* - mean_tpot_ms: time per output token
|
|
10
|
-
* - total_token_throughput
|
|
11
|
-
* - duration
|
|
12
|
-
* - completed / total_input_tokens / total_output_tokens
|
|
13
|
-
*
|
|
14
|
-
* Example vLLM benchmark output:
|
|
15
|
-
* {
|
|
16
|
-
* "elapsed_time": 100.5,
|
|
17
|
-
* "num_input_tokens": 1024000,
|
|
18
|
-
* "num_output_tokens": 256000,
|
|
19
|
-
* "request_throughput": 1.73,
|
|
20
|
-
* "output_throughput": 382.89,
|
|
21
|
-
* "total_token_throughput": 619.85,
|
|
22
|
-
* "mean_ttft_ms": 71.54,
|
|
23
|
-
* "median_ttft_ms": 70.21,
|
|
24
|
-
* "p99_ttft_ms": 79.49,
|
|
25
|
-
* "mean_tpot_ms": 12.45,
|
|
26
|
-
* "median_tpot_ms": 12.30,
|
|
27
|
-
* "p99_tpot_ms": 14.87,
|
|
28
|
-
* "mean_itl_ms": 7.74,
|
|
29
|
-
* "median_itl_ms": 7.68,
|
|
30
|
-
* "p99_itl_ms": 8.92,
|
|
31
|
-
* "input_len": "1024",
|
|
32
|
-
* "output_len": "256",
|
|
33
|
-
* "num_requests": 1000,
|
|
34
|
-
* "model_id": "meta-llama/Llama-3.1-70B-Instruct",
|
|
35
|
-
* ...
|
|
36
|
-
* }
|
|
37
|
-
*
|
|
38
|
-
* The actual field names vary by vLLM version. This parser handles the
|
|
39
|
-
* common field aliases across v0.5.x → v0.29.x.
|
|
40
|
-
*/
|
|
41
|
-
|
|
42
|
-
import type {
|
|
43
|
-
BenchmarkParser,
|
|
44
|
-
BenchmarkRecord,
|
|
45
|
-
ConfidenceTier,
|
|
46
|
-
} from "./benchmark-schema";
|
|
47
|
-
import { generateBenchmarkId } from "./benchmark-schema";
|
|
48
|
-
|
|
49
|
-
export const vllmParser: BenchmarkParser = {
|
|
50
|
-
name: "vllm-json",
|
|
51
|
-
version: "v1",
|
|
52
|
-
formatLabel: "vLLM benchmark JSON",
|
|
53
|
-
|
|
54
|
-
detect(rawText: string): boolean {
|
|
55
|
-
try {
|
|
56
|
-
const data = JSON.parse(rawText);
|
|
57
|
-
// Check for vLLM-specific fields. Require at least 2 to avoid
|
|
58
|
-
// catching TRT-LLM or SGLang output which may share `num_requests`.
|
|
59
|
-
const vllmFields = [
|
|
60
|
-
data.output_throughput,
|
|
61
|
-
data.total_token_throughput,
|
|
62
|
-
data.mean_ttft_ms,
|
|
63
|
-
data.mean_itl_ms,
|
|
64
|
-
data.mean_tpot_ms,
|
|
65
|
-
data.request_throughput,
|
|
66
|
-
].filter((v) => v !== undefined && v !== null);
|
|
67
|
-
return vllmFields.length >= 2;
|
|
68
|
-
} catch {
|
|
69
|
-
return false;
|
|
70
|
-
}
|
|
71
|
-
},
|
|
72
|
-
|
|
73
|
-
parse(rawText: string): BenchmarkRecord | null {
|
|
74
|
-
try {
|
|
75
|
-
const data = JSON.parse(rawText);
|
|
76
|
-
if (!data) return null;
|
|
77
|
-
|
|
78
|
-
// Extract model info
|
|
79
|
-
const modelId = data.model_id || data.model || "unknown";
|
|
80
|
-
const modelDisplayName = modelId.split("/").pop() || modelId;
|
|
81
|
-
|
|
82
|
-
// Extract throughput metrics (handle field name variations)
|
|
83
|
-
const outputThroughput = data.output_throughput ?? data.output_token_throughput ?? 0;
|
|
84
|
-
const totalThroughput = data.total_token_throughput ?? 0;
|
|
85
|
-
const requestThroughput = data.request_throughput ?? 0;
|
|
86
|
-
|
|
87
|
-
// Extract latency metrics
|
|
88
|
-
const meanTtft = data.mean_ttft_ms ?? data.mean_ttft;
|
|
89
|
-
const p50Ttft = data.median_ttft_ms ?? data.p50_ttft_ms ?? data.ttft_p50;
|
|
90
|
-
const p95Ttft = data.p95_ttft_ms ?? data.ttft_p95;
|
|
91
|
-
const p99Ttft = data.p99_ttft_ms ?? data.p99_ttft;
|
|
92
|
-
|
|
93
|
-
const meanItl = data.mean_itl_ms ?? data.mean_itl;
|
|
94
|
-
const p50Itl = data.median_itl_ms ?? data.p50_itl_ms ?? data.itl_p50;
|
|
95
|
-
const p95Itl = data.p95_itl_ms ?? data.itl_p95;
|
|
96
|
-
const p99Itl = data.p99_itl_ms ?? data.p99_itl;
|
|
97
|
-
|
|
98
|
-
const meanTpot = data.mean_tpot_ms ?? data.mean_tpot;
|
|
99
|
-
const p99Tpot = data.p99_tpot_ms ?? data.p99_tpot;
|
|
100
|
-
|
|
101
|
-
// Extract workload info
|
|
102
|
-
const numRequests = data.num_requests ?? data.completed ?? 0;
|
|
103
|
-
const inputLen = Number(data.input_len ?? data.mean_input_len ?? data.input_tokens_mean ?? 0);
|
|
104
|
-
const outputLen = Number(data.output_len ?? data.mean_output_len ?? data.output_tokens_mean ?? 0);
|
|
105
|
-
const concurrency = data.num_procs ?? data.concurrency ?? 1;
|
|
106
|
-
|
|
107
|
-
// Extract duration
|
|
108
|
-
const elapsed = data.elapsed_time ?? data.duration ?? 0;
|
|
109
|
-
|
|
110
|
-
// Determine confidence tier
|
|
111
|
-
const hasFullConfig = !!data.model_id && !!data.output_throughput && numRequests > 0;
|
|
112
|
-
const confidenceTier: ConfidenceTier = hasFullConfig
|
|
113
|
-
? "community_reproducible"
|
|
114
|
-
: "community_unverified";
|
|
115
|
-
|
|
116
|
-
const record: BenchmarkRecord = {
|
|
117
|
-
benchmark_id: generateBenchmarkId(),
|
|
118
|
-
source_name: "vllm",
|
|
119
|
-
source_url: data.source_url || "https://docs.vllm.ai/en/latest/benchmarking/cli/",
|
|
120
|
-
source_license: "Apache-2.0",
|
|
121
|
-
ingestion_method: "uploaded_raw_artifact",
|
|
122
|
-
retrieved_at: new Date().toISOString(),
|
|
123
|
-
benchmark_started_at: data.timestamp || data.run_time,
|
|
124
|
-
parser_name: "vllm-json",
|
|
125
|
-
parser_version: "v1",
|
|
126
|
-
confidence_tier: confidenceTier,
|
|
127
|
-
verification_status: "schema_valid",
|
|
128
|
-
|
|
129
|
-
model_id: modelId,
|
|
130
|
-
model_display_name: modelDisplayName,
|
|
131
|
-
model_revision: data.model_revision || data.commit || "unspecified",
|
|
132
|
-
architecture: data.architecture || "dense",
|
|
133
|
-
parameter_count_b: data.parameter_count_b,
|
|
134
|
-
context_window_tokens: data.max_model_len,
|
|
135
|
-
quantization_format: data.quantization || data.dtype || "unspecified",
|
|
136
|
-
weight_dtype: data.dtype || data.weight_dtype,
|
|
137
|
-
kv_cache_dtype: data.kv_cache_dtype || "float16",
|
|
138
|
-
|
|
139
|
-
accelerator_vendor: data.gpu_vendor || "NVIDIA",
|
|
140
|
-
accelerator_model: data.gpu_model || "unspecified",
|
|
141
|
-
gpu_count: data.tensor_parallel_size || data.gpu_count || 1,
|
|
142
|
-
memory_per_gpu_gb: data.gpu_memory || 0,
|
|
143
|
-
driver_version: data.driver_version,
|
|
144
|
-
cuda_rocm_metal_version: data.cuda_version,
|
|
145
|
-
|
|
146
|
-
serving_engine: "vLLM",
|
|
147
|
-
engine_version: data.vllm_version || "unspecified",
|
|
148
|
-
engine_command: data.command,
|
|
149
|
-
engine_config: {
|
|
150
|
-
tensor_parallel_size: data.tensor_parallel_size,
|
|
151
|
-
max_model_len: data.max_model_len,
|
|
152
|
-
max_num_seqs: data.max_num_seqs,
|
|
153
|
-
max_num_batched_tokens: data.max_num_batched_tokens,
|
|
154
|
-
prefix_cache_enabled: data.enable_prefix_caching ?? data.prefix_cache_enabled,
|
|
155
|
-
chunked_prefill_enabled: data.enable_chunked_prefill,
|
|
156
|
-
attention_backend: data.attention_backend,
|
|
157
|
-
},
|
|
158
|
-
|
|
159
|
-
workload_type: data.workload_type || "online_poisson",
|
|
160
|
-
dataset_name: data.dataset_name || data.dataset,
|
|
161
|
-
request_count: numRequests,
|
|
162
|
-
concurrency,
|
|
163
|
-
input_tokens_mean: inputLen,
|
|
164
|
-
output_tokens_mean: outputLen,
|
|
165
|
-
streaming_enabled: data.streaming ?? true,
|
|
166
|
-
|
|
167
|
-
successful_requests: numRequests,
|
|
168
|
-
failed_requests: data.failed ?? (data.total_requests ? data.total_requests - numRequests : 0),
|
|
169
|
-
request_throughput_rps: requestThroughput,
|
|
170
|
-
output_token_throughput_tps: outputThroughput,
|
|
171
|
-
total_token_throughput_tps: totalThroughput,
|
|
172
|
-
ttft_mean_ms: meanTtft,
|
|
173
|
-
ttft_p50_ms: p50Ttft,
|
|
174
|
-
ttft_p95_ms: p95Ttft,
|
|
175
|
-
ttft_p99_ms: p99Ttft,
|
|
176
|
-
itl_mean_ms: meanItl,
|
|
177
|
-
itl_p50_ms: p50Itl,
|
|
178
|
-
itl_p95_ms: p95Itl,
|
|
179
|
-
itl_p99_ms: p99Itl,
|
|
180
|
-
tpot_mean_ms: meanTpot,
|
|
181
|
-
e2e_latency_p99_ms: data.mean_lantecy_ms || data.e2e_latency_p99_ms,
|
|
182
|
-
|
|
183
|
-
gpu_memory_peak_gb: data.peak_gpu_memory || data.gpu_memory_peak_gb,
|
|
184
|
-
gpu_utilization_mean_pct: data.gpu_utilization,
|
|
185
|
-
|
|
186
|
-
metric_definition_version: "vllm-v1",
|
|
187
|
-
citation_text: `vLLM benchmark: ${outputThroughput.toFixed(2)} output tok/s, TTFT mean ${meanTtft?.toFixed(1) || "?"} ms, ${numRequests} requests`,
|
|
188
|
-
attribution_required: true,
|
|
189
|
-
commercial_redistribution_allowed: "yes",
|
|
190
|
-
};
|
|
191
|
-
|
|
192
|
-
return record;
|
|
193
|
-
} catch (e) {
|
|
194
|
-
console.error("vLLM parse error:", e);
|
|
195
|
-
return null;
|
|
196
|
-
}
|
|
197
|
-
},
|
|
198
|
-
};
|