@tokcalc/mcp-server 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.zscripts/build.sh +175 -0
- package/.zscripts/database-runtime-build.sh +33 -0
- package/.zscripts/dev.pid +1 -0
- package/.zscripts/dev.sh +154 -0
- package/.zscripts/mini-services-build.sh +78 -0
- package/.zscripts/mini-services-install.sh +65 -0
- package/.zscripts/mini-services-start.sh +123 -0
- package/.zscripts/python-runtime-build.sh +120 -0
- package/.zscripts/start.sh +145 -0
- package/CAPACITY_STUDY.md +283 -0
- package/CODE_OF_CONDUCT.md +55 -0
- package/CONTRIBUTING.md +177 -0
- package/Caddyfile +23 -0
- package/LICENSE +204 -0
- package/README.md +406 -86
- package/bun.lock +1965 -0
- package/components.json +21 -0
- package/db/custom.db +0 -0
- package/download/README.md +1 -0
- package/download/tokcalc-dark-calculator.png +0 -0
- package/download/tokcalc-dark-default.png +0 -0
- package/download/tokcalc-demo.webm +0 -0
- package/download/tokcalc-github-link.png +0 -0
- package/download/tokcalc-hydration-fixed.png +0 -0
- package/download/tokcalc-issue-resolved.png +0 -0
- package/download/tokcalc-light-mode.png +0 -0
- package/download/tokcalc-light-reference.png +0 -0
- package/download/tokcalc-long-context-qwen.png +0 -0
- package/download/tokcalc-long-context.png +0 -0
- package/download/tokcalc-og-image-preview.png +0 -0
- package/download/tokcalc-phase2-3.png +0 -0
- package/download/tokcalc-plain-english.png +0 -0
- package/download/tokcalc-preview.png +0 -0
- package/download/tokcalc-share-bvb.png +0 -0
- package/download/tokcalc-share-feature.png +0 -0
- package/download/tokcalc-tab-build-vs-buy.png +0 -0
- package/download/tokcalc-tab-calculator.png +0 -0
- package/download/tokcalc-tab-reference.png +0 -0
- package/eslint.config.mjs +50 -0
- package/examples/websocket/frontend.tsx +196 -0
- package/examples/websocket/server.ts +138 -0
- package/mini-services/.gitkeep +0 -0
- package/mini-services/mcp-server/README.md +86 -0
- package/mini-services/mcp-server/bun.lock +202 -0
- package/mini-services/mcp-server/index.ts +504 -0
- package/mini-services/mcp-server/package.json +40 -0
- package/next.config.ts +12 -0
- package/package.json +92 -35
- package/postcss.config.mjs +5 -0
- package/prisma/schema.prisma +32 -0
- package/public/google6f58ca6be85fa903.html +1 -0
- package/public/logo.svg +29 -0
- package/public/manifest.json +51 -0
- package/public/og-icon-256.png +0 -0
- package/public/og.png +0 -0
- package/public/robots.txt +25 -0
- package/public/sitemap.xml +23 -0
- package/public/tokcalc-demo.gif +0 -0
- package/scripts/og-template.html +120 -0
- package/scripts/render-og.mjs +43 -0
- package/server.json +21 -0
- package/src/app/api/pricing/aws/route.ts +186 -0
- package/src/app/api/pricing/azure/route.ts +168 -0
- package/src/app/api/pricing/gcp/route.ts +230 -0
- package/src/app/api/pricing/vast-ai/route.ts +164 -0
- package/src/app/api/route.ts +5 -0
- package/src/app/compare/h100-vs-h200/layout.tsx +30 -0
- package/src/app/compare/h100-vs-h200/page.tsx +328 -0
- package/src/app/globals.css +122 -0
- package/src/app/layout.tsx +276 -0
- package/src/app/page.tsx +2670 -0
- package/src/components/azure-live-pricing.tsx +185 -0
- package/src/components/benchmark-import.tsx +340 -0
- package/src/components/confidence-badge.tsx +116 -0
- package/src/components/live-pricing-comparison.tsx +241 -0
- package/src/components/theme-provider.tsx +11 -0
- package/src/components/theme-toggle.tsx +55 -0
- package/src/components/ui/accordion.tsx +66 -0
- package/src/components/ui/alert-dialog.tsx +157 -0
- package/src/components/ui/alert.tsx +66 -0
- package/src/components/ui/aspect-ratio.tsx +11 -0
- package/src/components/ui/avatar.tsx +53 -0
- package/src/components/ui/badge.tsx +46 -0
- package/src/components/ui/breadcrumb.tsx +109 -0
- package/src/components/ui/button.tsx +59 -0
- package/src/components/ui/calendar.tsx +213 -0
- package/src/components/ui/card.tsx +92 -0
- package/src/components/ui/carousel.tsx +241 -0
- package/src/components/ui/chart.tsx +353 -0
- package/src/components/ui/checkbox.tsx +32 -0
- package/src/components/ui/collapsible.tsx +33 -0
- package/src/components/ui/command.tsx +184 -0
- package/src/components/ui/context-menu.tsx +252 -0
- package/src/components/ui/dialog.tsx +143 -0
- package/src/components/ui/drawer.tsx +135 -0
- package/src/components/ui/dropdown-menu.tsx +257 -0
- package/src/components/ui/form.tsx +167 -0
- package/src/components/ui/hover-card.tsx +44 -0
- package/src/components/ui/input-otp.tsx +77 -0
- package/src/components/ui/input.tsx +21 -0
- package/src/components/ui/label.tsx +24 -0
- package/src/components/ui/menubar.tsx +276 -0
- package/src/components/ui/navigation-menu.tsx +168 -0
- package/src/components/ui/pagination.tsx +127 -0
- package/src/components/ui/popover.tsx +48 -0
- package/src/components/ui/progress.tsx +31 -0
- package/src/components/ui/radio-group.tsx +45 -0
- package/src/components/ui/resizable.tsx +56 -0
- package/src/components/ui/scroll-area.tsx +58 -0
- package/src/components/ui/select.tsx +185 -0
- package/src/components/ui/separator.tsx +28 -0
- package/src/components/ui/sheet.tsx +139 -0
- package/src/components/ui/sidebar.tsx +726 -0
- package/src/components/ui/skeleton.tsx +13 -0
- package/src/components/ui/slider.tsx +63 -0
- package/src/components/ui/sonner.tsx +25 -0
- package/src/components/ui/switch.tsx +31 -0
- package/src/components/ui/table.tsx +116 -0
- package/src/components/ui/tabs.tsx +66 -0
- package/src/components/ui/textarea.tsx +18 -0
- package/src/components/ui/toast.tsx +129 -0
- package/src/components/ui/toaster.tsx +35 -0
- package/src/components/ui/toggle-group.tsx +73 -0
- package/src/components/ui/toggle.tsx +47 -0
- package/src/components/ui/tooltip.tsx +61 -0
- package/src/components/vast-ai-live-pricing.tsx +176 -0
- package/src/hooks/use-mobile.ts +19 -0
- package/src/hooks/use-toast.ts +194 -0
- package/src/lib/benchmark-parser-sglang.ts +150 -0
- package/src/lib/benchmark-parser-tokcalc.ts +247 -0
- package/src/lib/benchmark-parser-trtllm.ts +152 -0
- package/src/lib/benchmark-parser-vllm.ts +198 -0
- package/src/lib/benchmark-schema.ts +263 -0
- package/src/lib/db.ts +13 -0
- package/src/lib/engine-presets.ts +183 -0
- package/src/lib/price-schema.ts +141 -0
- package/src/lib/token-calc.ts +808 -0
- package/src/lib/track.ts +31 -0
- package/src/lib/url-state.ts +256 -0
- package/src/lib/utils.ts +6 -0
- package/tailwind.config.ts +64 -0
- package/tests/database-runtime-build.sh +75 -0
- package/tests/python-runtime-build.sh +64 -0
- package/tests/python-runtime-container.sh +31 -0
- package/tool-results/bash_1789888171144_2c5381860539.txt +161 -0
- package/tool-results/bash_1789888175925_49c53ba3c61b.txt +191 -0
- package/tool-results/bash_1789888181202_49c53ba3c61b.txt +191 -0
- package/tool-results/bash_1789888195219_4a86a5c91411.txt +200 -0
- package/tool-results/bash_1789888203128_6cca13c71b47.txt +199 -0
- package/tool-results/bash_1789929256963_2a52aff0d0a8.txt +160 -0
- package/tool-results/read_1789888151021_69f58eec6a5b.txt +653 -0
- package/tool-results/read_1789888153837_1d3a8bfc2a94.txt +653 -0
- package/tool-results/read_1789888163087_ccc406d47505.txt +122 -0
- package/tool-results/read_1789888167347_67d1d7c9830a.txt +122 -0
- package/tool-results/read_1789929252529_d90e8f383a25.txt +285 -0
- package/tsconfig.json +42 -0
- package/upload/Pasted Content_1789887800864.txt +652 -0
- package/upload/Pasted Content_1789887909561.txt +652 -0
- package/upload/Pasted Content_1789887918428.txt +652 -0
- package/upload/Pasted Content_1789887959420.txt +652 -0
- package/upload/Pasted Content_1789888020485.txt +652 -0
- package/upload/Pasted Content_1789888058079.txt +652 -0
- package/upload/Pasted Content_1789888885033.txt +686 -0
- package/upload/Pasted Content_1789928912741.txt +285 -0
- package/upload/Pasted Content_1789928938402.txt +285 -0
- package/upload/Pasted Content_1789929160389.txt +285 -0
- package/upload/Pasted Content_1789929176660.txt +285 -0
- package/upload/issue_vision.json +28 -0
- package/upload/pasted_image_1789883175209.png +0 -0
- package/upload/pasted_image_1789899056690.png +0 -0
- package/upload/pasted_image_1789900371483.png +0 -0
- package/upload/pasted_image_1789900472823.png +0 -0
- package/upload/pasted_image_1789900490374.png +0 -0
- package/upload/pasted_image_1789900585552.png +0 -0
- package/upload/pasted_image_1789900606519.png +0 -0
- package/upload/pasted_image_1789901598705.png +0 -0
- package/upload/pasted_image_1789901613545.png +0 -0
- package/upload/pasted_image_1789978382674.png +0 -0
- package/upload/pasted_image_1789978392749.png +0 -0
- package/upload/pasted_image_1789978474879.png +0 -0
- package/upload/pasted_image_1789978523652.png +0 -0
- package/upload/pasted_image_1789984219089.png +0 -0
- package/upload/pasted_image_1789984491896.png +0 -0
- package/upload/pasted_image_1789985017950.png +0 -0
- package/upload/pasted_image_1789985036765.png +0 -0
- package/upload/pasted_image_1789985049848.png +0 -0
- package/upload/pasted_image_1790002427833.png +0 -0
- package/upload/pasted_image_1790002659944.png +0 -0
- package/upload/pasted_image_1790037038476.png +0 -0
- package/upload/screenshot_analysis.json +28 -0
- package/upload/vision_output.json +28 -0
- package/dist/index.js +0 -21014
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Benchmark provenance schema for tokcalc.
|
|
3
|
+
*
|
|
4
|
+
* Based on the Perplexity research brief which specified:
|
|
5
|
+
* "Do not build a single scalar called 'tokens/sec.' A valid observed record
|
|
6
|
+
* should always preserve the workload, serving software, engine flags, hardware
|
|
7
|
+
* topology, model revision, quantization, and metric definition."
|
|
8
|
+
*
|
|
9
|
+
* This file defines:
|
|
10
|
+
* 1. The BenchmarkRecord type (50+ fields across 7 categories)
|
|
11
|
+
* 2. A confidence tier system (official → community → unverified)
|
|
12
|
+
* 3. Parser interface for ingesting raw benchmark artifacts
|
|
13
|
+
*
|
|
14
|
+
* Design principle: tokcalc retains separate estimate types:
|
|
15
|
+
* - Theoretical model (what we have now — physics formulas)
|
|
16
|
+
* - Observed benchmark (this file — real measured data)
|
|
17
|
+
* - Calibrated estimate (theory adjusted by observed data — future)
|
|
18
|
+
* - User-provided run (user uploads their benchmark — future)
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/* ============================================================
|
|
22
|
+
CONFIDENCE TIERS
|
|
23
|
+
============================================================ */
|
|
24
|
+
|
|
25
|
+
export type ConfidenceTier =
|
|
26
|
+
| "official_audited" // MLPerf submission, official vendor benchmark
|
|
27
|
+
| "official_reproducible" // vLLM/SGLang/TRT-LLM docs example with full config
|
|
28
|
+
| "community_reproducible" // User ran it with documented config + raw artifact
|
|
29
|
+
| "community_unverified" // User-reported number without raw artifact
|
|
30
|
+
| "derived"; // Computed from other records (e.g., calibration)
|
|
31
|
+
|
|
32
|
+
export type VerificationStatus =
|
|
33
|
+
| "unreviewed"
|
|
34
|
+
| "schema_valid"
|
|
35
|
+
| "reproduced"
|
|
36
|
+
| "curated"
|
|
37
|
+
| "rejected";
|
|
38
|
+
|
|
39
|
+
/* ============================================================
|
|
40
|
+
CORE BENCHMARK RECORD
|
|
41
|
+
============================================================ */
|
|
42
|
+
|
|
43
|
+
export interface BenchmarkRecord {
|
|
44
|
+
/* ---- Core identity and provenance ---- */
|
|
45
|
+
benchmark_id: string;
|
|
46
|
+
source_name: "vllm" | "sglang" | "tensorrt_llm" | "mlperf" | "llmperf" | "community" | "manual_curated";
|
|
47
|
+
source_url: string;
|
|
48
|
+
source_license: string;
|
|
49
|
+
ingestion_method: "official_api" | "official_repo" | "uploaded_raw_artifact" | "manual_curated";
|
|
50
|
+
retrieved_at: string; // ISO 8601
|
|
51
|
+
benchmark_started_at?: string; // ISO 8601
|
|
52
|
+
raw_artifact_url?: string;
|
|
53
|
+
artifact_sha256?: string;
|
|
54
|
+
parser_name?: string;
|
|
55
|
+
parser_version?: string;
|
|
56
|
+
confidence_tier: ConfidenceTier;
|
|
57
|
+
verification_status: VerificationStatus;
|
|
58
|
+
|
|
59
|
+
/* ---- Model identity ---- */
|
|
60
|
+
model_id: string;
|
|
61
|
+
model_display_name: string;
|
|
62
|
+
model_repository?: string;
|
|
63
|
+
model_revision: string; // commit hash / version / date
|
|
64
|
+
architecture?: "dense" | "moe" | "vlm" | "encoder" | "decoder_only";
|
|
65
|
+
parameter_count_b?: number;
|
|
66
|
+
context_window_tokens?: number;
|
|
67
|
+
quantization_format: string; // FP16, FP8, GGUF Q4_K_M, AWQ, etc.
|
|
68
|
+
weight_dtype?: string; // float16, bfloat16, float8_e4m3, int8, etc.
|
|
69
|
+
kv_cache_dtype?: string; // "float16" by default
|
|
70
|
+
|
|
71
|
+
/* ---- Hardware and topology ---- */
|
|
72
|
+
accelerator_vendor: "NVIDIA" | "AMD" | "Intel" | "Apple" | "Google" | "Groq" | "Cerebras" | string;
|
|
73
|
+
accelerator_model: string;
|
|
74
|
+
gpu_count: number;
|
|
75
|
+
memory_per_gpu_gb: number;
|
|
76
|
+
host_cpu?: string;
|
|
77
|
+
host_memory_gb?: number;
|
|
78
|
+
interconnect?: string;
|
|
79
|
+
topology?: string;
|
|
80
|
+
cloud_provider?: string;
|
|
81
|
+
instance_type?: string;
|
|
82
|
+
region?: string;
|
|
83
|
+
driver_version?: string;
|
|
84
|
+
cuda_rocm_metal_version?: string;
|
|
85
|
+
|
|
86
|
+
/* ---- Serving stack and configuration ---- */
|
|
87
|
+
serving_engine: "vLLM" | "SGLang" | "TensorRT-LLM" | "TGI" | "llama.cpp" | "MLX" | string;
|
|
88
|
+
engine_version: string;
|
|
89
|
+
engine_command?: string;
|
|
90
|
+
engine_config?: {
|
|
91
|
+
tensor_parallel_size?: number;
|
|
92
|
+
pipeline_parallel_size?: number;
|
|
93
|
+
max_model_len?: number;
|
|
94
|
+
max_num_seqs?: number;
|
|
95
|
+
max_num_batched_tokens?: number;
|
|
96
|
+
prefix_cache_enabled?: boolean;
|
|
97
|
+
speculative_decoding?: {
|
|
98
|
+
enabled: boolean;
|
|
99
|
+
draft_model?: string;
|
|
100
|
+
acceptance_rate?: number;
|
|
101
|
+
};
|
|
102
|
+
chunked_prefill_enabled?: boolean;
|
|
103
|
+
attention_backend?: string;
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
/* ---- Workload definition ---- */
|
|
107
|
+
workload_type: "offline" | "online_poisson" | "fixed_rate" | "closed_loop" | "trace_replay" | "synthetic";
|
|
108
|
+
dataset_name?: string;
|
|
109
|
+
request_count: number;
|
|
110
|
+
concurrency: number;
|
|
111
|
+
arrival_rate_rps?: number;
|
|
112
|
+
input_tokens_mean: number;
|
|
113
|
+
input_tokens_p50?: number;
|
|
114
|
+
input_tokens_p95?: number;
|
|
115
|
+
input_tokens_p99?: number;
|
|
116
|
+
output_tokens_mean: number;
|
|
117
|
+
output_tokens_p50?: number;
|
|
118
|
+
output_tokens_p95?: number;
|
|
119
|
+
output_tokens_p99?: number;
|
|
120
|
+
shared_prefix_tokens_mean?: number;
|
|
121
|
+
shared_prefix_hit_rate?: number;
|
|
122
|
+
streaming_enabled: boolean;
|
|
123
|
+
sampling_config?: {
|
|
124
|
+
temperature?: number;
|
|
125
|
+
top_p?: number;
|
|
126
|
+
max_tokens?: number;
|
|
127
|
+
seed?: number;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
/* ---- Observed metrics ---- */
|
|
131
|
+
successful_requests: number;
|
|
132
|
+
failed_requests: number;
|
|
133
|
+
request_throughput_rps?: number;
|
|
134
|
+
output_token_throughput_tps: number; // THE key metric
|
|
135
|
+
total_token_throughput_tps?: number;
|
|
136
|
+
ttft_mean_ms?: number;
|
|
137
|
+
ttft_p50_ms?: number;
|
|
138
|
+
ttft_p95_ms?: number;
|
|
139
|
+
ttft_p99_ms?: number;
|
|
140
|
+
itl_mean_ms?: number;
|
|
141
|
+
itl_p50_ms?: number;
|
|
142
|
+
itl_p95_ms?: number;
|
|
143
|
+
itl_p99_ms?: number;
|
|
144
|
+
tpot_mean_ms?: number; // Time per output token (vLLM's term)
|
|
145
|
+
e2e_latency_p50_ms?: number;
|
|
146
|
+
e2e_latency_p95_ms?: number;
|
|
147
|
+
e2e_latency_p99_ms?: number;
|
|
148
|
+
gpu_memory_peak_gb?: number;
|
|
149
|
+
gpu_utilization_mean_pct?: number;
|
|
150
|
+
power_mean_w?: number;
|
|
151
|
+
energy_per_output_token_j?: number;
|
|
152
|
+
|
|
153
|
+
/* ---- Comparability and display ---- */
|
|
154
|
+
metric_definition_version: string;
|
|
155
|
+
slo_ttft_ms?: number;
|
|
156
|
+
slo_itl_ms?: number;
|
|
157
|
+
slo_e2e_ms?: number;
|
|
158
|
+
comparability_group?: string;
|
|
159
|
+
normalization_notes?: string;
|
|
160
|
+
known_limitations?: string;
|
|
161
|
+
citation_text: string;
|
|
162
|
+
attribution_required: boolean;
|
|
163
|
+
commercial_redistribution_allowed: "yes" | "no" | "unknown" | "contract_required";
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/* ============================================================
|
|
167
|
+
CALIBRATION RESULT — compares theoretical vs observed
|
|
168
|
+
============================================================ */
|
|
169
|
+
|
|
170
|
+
export interface CalibrationResult {
|
|
171
|
+
/** What tokcalc's formulas predicted */
|
|
172
|
+
estimated: {
|
|
173
|
+
decodeTokensPerSec: number;
|
|
174
|
+
aggregateTokensPerSec: number;
|
|
175
|
+
prefillTokensPerSec: number;
|
|
176
|
+
ttftMs: number;
|
|
177
|
+
itlMs: number;
|
|
178
|
+
modelSizeGb: number;
|
|
179
|
+
};
|
|
180
|
+
/** What the benchmark actually measured */
|
|
181
|
+
observed: {
|
|
182
|
+
outputTokenThroughputTps?: number;
|
|
183
|
+
ttftMeanMs?: number;
|
|
184
|
+
ttftP95Ms?: number;
|
|
185
|
+
itlMeanMs?: number;
|
|
186
|
+
itlP95Ms?: number;
|
|
187
|
+
gpuMemoryPeakGb?: number;
|
|
188
|
+
};
|
|
189
|
+
/** Calibration error (observed - estimated) / estimated × 100 */
|
|
190
|
+
calibrationErrors: {
|
|
191
|
+
throughputErrorPct?: number; // positive = observed was faster than predicted
|
|
192
|
+
ttftErrorPct?: number; // positive = observed TTFT was longer than predicted
|
|
193
|
+
itlErrorPct?: number;
|
|
194
|
+
memoryErrorPct?: number;
|
|
195
|
+
};
|
|
196
|
+
/** Does the observed data validate or contradict our model? */
|
|
197
|
+
verdict: "validated" | "underestimated" | "overestimated" | "insufficient_data";
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/* ============================================================
|
|
201
|
+
PARSER INTERFACE
|
|
202
|
+
============================================================ */
|
|
203
|
+
|
|
204
|
+
export interface BenchmarkParser {
|
|
205
|
+
name: string;
|
|
206
|
+
version: string;
|
|
207
|
+
/** Detect if raw text looks like this engine's output */
|
|
208
|
+
detect(rawText: string): boolean;
|
|
209
|
+
/** Parse raw text into a BenchmarkRecord */
|
|
210
|
+
parse(rawText: string): BenchmarkRecord | null;
|
|
211
|
+
/** Human-readable format name */
|
|
212
|
+
formatLabel: string;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/* ============================================================
|
|
216
|
+
HELPER: generate UUID (no dependency)
|
|
217
|
+
============================================================ */
|
|
218
|
+
|
|
219
|
+
export function generateBenchmarkId(): string {
|
|
220
|
+
// RFC 4122 v4 UUID — no dependency needed
|
|
221
|
+
return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (c) => {
|
|
222
|
+
const r = (Math.random() * 16) | 0;
|
|
223
|
+
const v = c === "x" ? r : (r & 0x3) | 0x8;
|
|
224
|
+
return v.toString(16);
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/* ============================================================
|
|
229
|
+
HELPER: format calibration verdict for display
|
|
230
|
+
============================================================ */
|
|
231
|
+
|
|
232
|
+
export function formatCalibrationVerdict(verdict: CalibrationResult["verdict"]): {
|
|
233
|
+
label: string;
|
|
234
|
+
color: string;
|
|
235
|
+
description: string;
|
|
236
|
+
} {
|
|
237
|
+
switch (verdict) {
|
|
238
|
+
case "validated":
|
|
239
|
+
return {
|
|
240
|
+
label: "Validated",
|
|
241
|
+
color: "text-emerald-500",
|
|
242
|
+
description: "Observed benchmark matches theoretical estimate within ±20%. Formula is accurate for this config.",
|
|
243
|
+
};
|
|
244
|
+
case "underestimated":
|
|
245
|
+
return {
|
|
246
|
+
label: "Underestimated",
|
|
247
|
+
color: "text-amber-500",
|
|
248
|
+
description: "Real performance was BETTER than tokcalc predicted. Formula may be too conservative.",
|
|
249
|
+
};
|
|
250
|
+
case "overestimated":
|
|
251
|
+
return {
|
|
252
|
+
label: "Overestimated",
|
|
253
|
+
color: "text-red-500",
|
|
254
|
+
description: "Real performance was WORSE than tokcalc predicted. Formula may be too optimistic.",
|
|
255
|
+
};
|
|
256
|
+
case "insufficient_data":
|
|
257
|
+
return {
|
|
258
|
+
label: "Insufficient data",
|
|
259
|
+
color: "text-muted-foreground",
|
|
260
|
+
description: "Not enough observed metrics to calibrate against the theoretical estimate.",
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
}
|
package/src/lib/db.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { PrismaClient } from '@prisma/client'
|
|
2
|
+
|
|
3
|
+
const globalForPrisma = globalThis as unknown as {
|
|
4
|
+
prisma: PrismaClient | undefined
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export const db =
|
|
8
|
+
globalForPrisma.prisma ??
|
|
9
|
+
new PrismaClient({
|
|
10
|
+
log: ['query'],
|
|
11
|
+
})
|
|
12
|
+
|
|
13
|
+
if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = db
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Engine-aware presets for tokcalc.
|
|
3
|
+
*
|
|
4
|
+
* When the user picks an inference engine (vLLM, SGLang, TRT-LLM, llama.cpp),
|
|
5
|
+
* the calculator adjusts:
|
|
6
|
+
* 1. Efficiency factors (ETA_MEM, ETA_COMPUTE) — each engine has different
|
|
7
|
+
* real-world utilization due to kernel optimizations, scheduler design, etc.
|
|
8
|
+
* 2. Default continuous batching multiplier — vLLM/SGLang have it on by
|
|
9
|
+
* default (2.0×), llama.cpp doesn't (1.0×)
|
|
10
|
+
* 3. Supported quantizations — filters the quant dropdown to only show
|
|
11
|
+
* formats the selected engine supports
|
|
12
|
+
* 4. Engine-specific notes — helps users understand trade-offs
|
|
13
|
+
*
|
|
14
|
+
* Per Perplexity research (Prompt #2):
|
|
15
|
+
* "There is no universally best engine; workload shape matters."
|
|
16
|
+
* "comparisons emphasize that throughput leadership varies with workload,
|
|
17
|
+
* structured output, prefix reuse, and operational maturity"
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import type { Quantization } from "./token-calc";
|
|
21
|
+
|
|
22
|
+
export type EngineId = "vllm" | "sglang" | "trtllm" | "llamacpp" | "generic";
|
|
23
|
+
|
|
24
|
+
export interface EnginePreset {
|
|
25
|
+
id: EngineId;
|
|
26
|
+
name: string;
|
|
27
|
+
shortName: string;
|
|
28
|
+
description: string;
|
|
29
|
+
/** Memory bandwidth utilization (real-world) — replaces ETA_MEM when engine is selected */
|
|
30
|
+
etaMem: number;
|
|
31
|
+
/** Compute utilization — replaces ETA_COMPUTE */
|
|
32
|
+
etaCompute: number;
|
|
33
|
+
/** Default continuous batching multiplier when engine is selected */
|
|
34
|
+
defaultBatchingMultiplier: number;
|
|
35
|
+
/** Which quantization formats this engine supports */
|
|
36
|
+
supportedQuants: Quantization[];
|
|
37
|
+
/** Engine-specific notes shown in UI */
|
|
38
|
+
notes: string;
|
|
39
|
+
/** Source URL for the engine */
|
|
40
|
+
sourceUrl: string;
|
|
41
|
+
/** Whether continuous batching is on by default in this engine */
|
|
42
|
+
batchingOnByDefault: boolean;
|
|
43
|
+
/** Whether prefix caching is on by default */
|
|
44
|
+
prefixCacheOnByDefault: boolean;
|
|
45
|
+
/** Key features of this engine */
|
|
46
|
+
features: string[];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export const ENGINE_PRESETS: EnginePreset[] = [
|
|
50
|
+
{
|
|
51
|
+
id: "vllm",
|
|
52
|
+
name: "vLLM",
|
|
53
|
+
shortName: "vLLM",
|
|
54
|
+
description: "High-throughput LLM serving with PagedAttention + continuous batching. The most popular open-source serving engine.",
|
|
55
|
+
etaMem: 0.65, // Default — PagedAttention reduces waste to <4%
|
|
56
|
+
etaCompute: 0.50, // Default
|
|
57
|
+
defaultBatchingMultiplier: 2.0, // vLLM enables continuous batching by default
|
|
58
|
+
supportedQuants: ["fp32", "fp16", "bf16", "int8", "fp8", "gptq4", "awq4", "int4"],
|
|
59
|
+
notes: "PagedAttention + continuous batching ON by default. Real-world throughput is 2-4× naive batching. Supports FP8 natively on H100/H200. GGUF not supported — use llama.cpp for that.",
|
|
60
|
+
sourceUrl: "https://docs.vllm.ai",
|
|
61
|
+
batchingOnByDefault: true,
|
|
62
|
+
prefixCacheOnByDefault: true, // APC (Automatic Prefix Caching)
|
|
63
|
+
features: [
|
|
64
|
+
"PagedAttention (KV cache memory management)",
|
|
65
|
+
"Continuous batching (iteration-level scheduling)",
|
|
66
|
+
"Automatic prefix caching (APC)",
|
|
67
|
+
"Tensor parallelism (up to 8×)",
|
|
68
|
+
"Speculative decoding",
|
|
69
|
+
"FP8 native on H100/H200",
|
|
70
|
+
],
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
id: "sglang",
|
|
74
|
+
name: "SGLang",
|
|
75
|
+
shortName: "SGLang",
|
|
76
|
+
description: "High-performance serving with RadixAttention (prefix-tree KV cache) + structured generation. Best for agent workloads with repeated prefixes.",
|
|
77
|
+
etaMem: 0.68, // Slightly higher — RadixAttention is more efficient than APC
|
|
78
|
+
etaCompute: 0.52,
|
|
79
|
+
defaultBatchingMultiplier: 2.0, // Continuous batching on by default
|
|
80
|
+
supportedQuants: ["fp32", "fp16", "bf16", "int8", "fp8", "gptq4", "awq4"],
|
|
81
|
+
notes: "RadixAttention provides automatic prefix reuse without explicit cache management. Better for agent/RAG workloads with shared system prompts. Slightly higher memory efficiency than vLLM for prefix-heavy workloads.",
|
|
82
|
+
sourceUrl: "https://github.com/sgl-project/sglang",
|
|
83
|
+
batchingOnByDefault: true,
|
|
84
|
+
prefixCacheOnByDefault: true, // RadixAttention IS the prefix cache
|
|
85
|
+
features: [
|
|
86
|
+
"RadixAttention (prefix-tree KV cache)",
|
|
87
|
+
"Continuous batching",
|
|
88
|
+
"Structured generation (JSON mode)",
|
|
89
|
+
"Tensor parallelism",
|
|
90
|
+
"Speculative decoding",
|
|
91
|
+
"PD disaggregation (experimental)",
|
|
92
|
+
],
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
id: "trtllm",
|
|
96
|
+
name: "TensorRT-LLM",
|
|
97
|
+
shortName: "TRT-LLM",
|
|
98
|
+
description: "NVIDIA-optimized inference engine. Best raw throughput on NVIDIA GPUs with quantized models. Requires model compilation.",
|
|
99
|
+
etaMem: 0.70, // NVIDIA-optimized kernels achieve higher utilization
|
|
100
|
+
etaCompute: 0.55,
|
|
101
|
+
defaultBatchingMultiplier: 2.0,
|
|
102
|
+
supportedQuants: ["fp32", "fp16", "bf16", "int8", "fp8", "nvfp4", "int4"],
|
|
103
|
+
notes: "NVIDIA-optimized kernels achieve 5-10% higher utilization than vLLM on same hardware. NVFP4 native on Blackwell (B200/B300). Requires model compilation (TRT engine build) — slower to iterate than vLLM. Inflight batching is TRT-LLM's version of continuous batching.",
|
|
104
|
+
sourceUrl: "https://github.com/NVIDIA/TensorRT-LLM",
|
|
105
|
+
batchingOnByDefault: true, // Inflight batching
|
|
106
|
+
prefixCacheOnByDefault: true,
|
|
107
|
+
features: [
|
|
108
|
+
"NVIDIA-optimized kernels (FlashAttention, fused MLP)",
|
|
109
|
+
"Inflight batching (continuous batching equivalent)",
|
|
110
|
+
"FP8 + NVFP4 native on H100/H200/B200",
|
|
111
|
+
"Model compilation (TRT engine build)",
|
|
112
|
+
"Tensor parallelism",
|
|
113
|
+
"Plugin system for custom layers",
|
|
114
|
+
],
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
id: "llamacpp",
|
|
118
|
+
name: "llama.cpp",
|
|
119
|
+
shortName: "llama.cpp",
|
|
120
|
+
description: "C++ inference engine for CPU, Apple Silicon, and consumer GPUs. Best for local/edge deployment. GGUF format.",
|
|
121
|
+
etaMem: 0.55, // Lower — CPU/GPU mixed, less optimized than vLLM
|
|
122
|
+
etaCompute: 0.40, // Lower — CPU-bound on non-GPU paths
|
|
123
|
+
defaultBatchingMultiplier: 1.0, // No continuous batching by default
|
|
124
|
+
supportedQuants: ["fp32", "fp16", "bf16", "int8", "gguf-q2k", "gguf-q3km", "gguf-q4km", "gguf-q5km", "gguf-q6k", "gguf-q8", "int4"],
|
|
125
|
+
notes: "Best for local/edge: Mac M-series (MLX backend), consumer GPUs (CUDA), CPU-only machines. Supports all GGUF quantization variants (Q2_K through Q8_0). No continuous batching — throughput is single-stream only. KV cache is contiguous (not paged).",
|
|
126
|
+
sourceUrl: "https://github.com/ggml-org/llama.cpp",
|
|
127
|
+
batchingOnByDefault: false,
|
|
128
|
+
prefixCacheOnByDefault: false,
|
|
129
|
+
features: [
|
|
130
|
+
"GGUF format (Q2_K through Q8_0)",
|
|
131
|
+
"CPU inference (no GPU needed)",
|
|
132
|
+
"Apple Silicon (Metal backend)",
|
|
133
|
+
"Consumer GPU support (CUDA)",
|
|
134
|
+
"Lowest memory footprint",
|
|
135
|
+
"No server needed (CLI + server mode)",
|
|
136
|
+
],
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
id: "generic",
|
|
140
|
+
name: "Generic (engine-agnostic)",
|
|
141
|
+
shortName: "Generic",
|
|
142
|
+
description: "No specific engine selected. Uses conservative default efficiency factors. Best when you don't know which engine you'll use yet.",
|
|
143
|
+
etaMem: 0.65, // Default — same as current ETA_MEM
|
|
144
|
+
etaCompute: 0.50, // Default
|
|
145
|
+
defaultBatchingMultiplier: 1.5, // Conservative default
|
|
146
|
+
supportedQuants: [
|
|
147
|
+
"fp32", "fp16", "bf16", "int8", "int4",
|
|
148
|
+
"gguf-q2k", "gguf-q3km", "gguf-q4km", "gguf-q5km", "gguf-q6k", "gguf-q8",
|
|
149
|
+
"gptq4", "awq4", "exl2-6bpw", "fp8", "nvfp4",
|
|
150
|
+
],
|
|
151
|
+
notes: "Conservative estimates. When you know your engine, select it for more accurate results. Each engine has different real-world utilization due to kernel optimizations and scheduler design.",
|
|
152
|
+
sourceUrl: "",
|
|
153
|
+
batchingOnByDefault: false,
|
|
154
|
+
prefixCacheOnByDefault: false,
|
|
155
|
+
features: [],
|
|
156
|
+
},
|
|
157
|
+
];
|
|
158
|
+
|
|
159
|
+
export const ENGINE_MAP: Record<EngineId, EnginePreset> = Object.fromEntries(
|
|
160
|
+
ENGINE_PRESETS.map((e) => [e.id, e]),
|
|
161
|
+
) as Record<EngineId, EnginePreset>;
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Get the engine preset by ID (defaults to "generic").
|
|
165
|
+
*/
|
|
166
|
+
export function getEngine(id: EngineId | string | undefined): EnginePreset {
|
|
167
|
+
return ENGINE_MAP[id as EngineId] || ENGINE_MAP.generic;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Check if a quantization format is supported by an engine.
|
|
172
|
+
*/
|
|
173
|
+
export function isQuantSupported(engine: EngineId, quant: Quantization): boolean {
|
|
174
|
+
const preset = getEngine(engine);
|
|
175
|
+
return preset.supportedQuants.includes(quant);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Get the list of quantizations supported by an engine, as Quantization[].
|
|
180
|
+
*/
|
|
181
|
+
export function getSupportedQuants(engine: EngineId): Quantization[] {
|
|
182
|
+
return getEngine(engine).supportedQuants;
|
|
183
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Price record type for live cloud GPU pricing.
|
|
3
|
+
*
|
|
4
|
+
* Based on the Perplexity research brief (Prompt #3):
|
|
5
|
+
* "Use an append-only historical model. A price should never be overwritten
|
|
6
|
+
* without retaining the prior record."
|
|
7
|
+
*
|
|
8
|
+
* Phase 1: Azure Retail Prices API (unauthenticated, easiest)
|
|
9
|
+
* Phase 2: Vast.ai marketplace API (dynamic spot pricing)
|
|
10
|
+
* Phase 3: AWS / GCP / OCI official APIs (authenticated)
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
export type PriceProvider =
|
|
14
|
+
| "azure"
|
|
15
|
+
| "aws"
|
|
16
|
+
| "gcp"
|
|
17
|
+
| "oci"
|
|
18
|
+
| "vast_ai"
|
|
19
|
+
| "runpod"
|
|
20
|
+
| "lambda"
|
|
21
|
+
| "modal"
|
|
22
|
+
| "tensor"
|
|
23
|
+
| "default";
|
|
24
|
+
|
|
25
|
+
export type PurchaseOption =
|
|
26
|
+
| "on_demand"
|
|
27
|
+
| "spot"
|
|
28
|
+
| "reserved"
|
|
29
|
+
| "savings_plan"
|
|
30
|
+
| "marketplace"
|
|
31
|
+
| "serverless";
|
|
32
|
+
|
|
33
|
+
export type BillingUnit =
|
|
34
|
+
| "instance_hour"
|
|
35
|
+
| "gpu_hour"
|
|
36
|
+
| "gpu_second";
|
|
37
|
+
|
|
38
|
+
export interface PriceRecord {
|
|
39
|
+
id: string;
|
|
40
|
+
provider: PriceProvider;
|
|
41
|
+
productFamily: "cloud_vm" | "gpu_marketplace";
|
|
42
|
+
|
|
43
|
+
/** Cloud SKU identifier (e.g., "Standard_ND96isr_H100_v5") */
|
|
44
|
+
providerSku: string;
|
|
45
|
+
providerSkuName: string;
|
|
46
|
+
|
|
47
|
+
/** GPU info extracted from the SKU */
|
|
48
|
+
acceleratorVendor?: string;
|
|
49
|
+
acceleratorModel?: string;
|
|
50
|
+
acceleratorCount?: number;
|
|
51
|
+
acceleratorMemoryGb?: number;
|
|
52
|
+
|
|
53
|
+
/** Region/zone */
|
|
54
|
+
region?: string;
|
|
55
|
+
locationLabel?: string;
|
|
56
|
+
|
|
57
|
+
purchaseOption: PurchaseOption;
|
|
58
|
+
billingUnit: BillingUnit;
|
|
59
|
+
|
|
60
|
+
/** Price in USD */
|
|
61
|
+
amount: number;
|
|
62
|
+
currency: "USD";
|
|
63
|
+
|
|
64
|
+
/** Per-GPU derived price (if instance-level, divide by GPU count) */
|
|
65
|
+
derivedPricePerGpuHour?: number;
|
|
66
|
+
isDerived: boolean;
|
|
67
|
+
derivationFormula?: string;
|
|
68
|
+
|
|
69
|
+
/** Validity window */
|
|
70
|
+
effectiveFrom?: string;
|
|
71
|
+
effectiveTo?: string;
|
|
72
|
+
retrievedAt: string;
|
|
73
|
+
|
|
74
|
+
/** Provenance */
|
|
75
|
+
sourceType: "official_api" | "official_price_page" | "manual_verified";
|
|
76
|
+
sourceUrl: string;
|
|
77
|
+
|
|
78
|
+
/** Display confidence */
|
|
79
|
+
confidence: "high" | "medium" | "low";
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Match a GPU from tokcalc's catalog to a cloud provider's SKU.
|
|
84
|
+
* Returns the best guess based on GPU name + VRAM.
|
|
85
|
+
*/
|
|
86
|
+
export function matchGpuToCloudSku(
|
|
87
|
+
gpuName: string,
|
|
88
|
+
gpuVramGb: number,
|
|
89
|
+
provider: PriceProvider,
|
|
90
|
+
): { sku: string; skuName: string } | null {
|
|
91
|
+
const name = gpuName.toLowerCase();
|
|
92
|
+
|
|
93
|
+
// Azure SKU patterns
|
|
94
|
+
if (provider === "azure") {
|
|
95
|
+
if (name.includes("h100")) return { sku: "Standard_ND96isr_H100_v5", skuName: "ND H100 v5 (96 GPU = 8× H100)" };
|
|
96
|
+
if (name.includes("h200")) return { sku: "Standard_ND-H200-v5", skuName: "ND H200 v5" };
|
|
97
|
+
if (name.includes("a100") && gpuVramGb >= 80) return { sku: "Standard_ND96asr_v4", skuName: "ND A100 v4 (8× A100 80GB)" };
|
|
98
|
+
if (name.includes("a100") && gpuVramGb < 80) return { sku: "Standard_ND96amsr_A100_v4", skuName: "ND A100 v4 (40GB)" };
|
|
99
|
+
if (name.includes("v100")) return { sku: "Standard_NC24ads_A100_v4", skuName: "NC A100 v4" };
|
|
100
|
+
if (name.includes("t4")) return { sku: "Standard_NC4as_T4_v3", skuName: "NC T4 v3" };
|
|
101
|
+
if (name.includes("l40s") || name.includes("l40")) return { sku: "Standard_NDm_A100_v4", skuName: "NDm L40S" };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// AWS SKU patterns
|
|
105
|
+
if (provider === "aws") {
|
|
106
|
+
if (name.includes("h100")) return { sku: "p5.48xlarge", skuName: "p5.48xlarge (8× H100)" };
|
|
107
|
+
if (name.includes("h200")) return { sku: "p5e.48xlarge", skuName: "p5e.48xlarge (8× H200)" };
|
|
108
|
+
if (name.includes("b200")) return { sku: "p6.48xlarge", skuName: "p6.48xlarge (8× B200)" };
|
|
109
|
+
if (name.includes("a100") && gpuVramGb >= 80) return { sku: "p4de.24xlarge", skuName: "p4de.24xlarge (8× A100 80GB)" };
|
|
110
|
+
if (name.includes("a100") && gpuVramGb < 80) return { sku: "p4d.24xlarge", skuName: "p4d.24xlarge (8× A100 40GB)" };
|
|
111
|
+
if (name.includes("l40s")) return { sku: "g6.48xlarge", skuName: "g6.48xlarge (8× L40S)" };
|
|
112
|
+
if (name.includes("l4")) return { sku: "g6.xlarge", skuName: "g6.xlarge (1× L4)" };
|
|
113
|
+
if (name.includes("t4")) return { sku: "g4dn.12xlarge", skuName: "g4dn.12xlarge (4× T4)" };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Format a price record for display in the UI.
|
|
121
|
+
*/
|
|
122
|
+
export function formatPriceRecord(r: PriceRecord): {
|
|
123
|
+
displayPrice: string;
|
|
124
|
+
isLive: boolean;
|
|
125
|
+
ageLabel: string;
|
|
126
|
+
} {
|
|
127
|
+
const perGpu = r.derivedPricePerGpuHour ?? r.amount;
|
|
128
|
+
const displayPrice = `$${perGpu.toFixed(2)}/hr`;
|
|
129
|
+
|
|
130
|
+
// "Live" if retrieved within last 24 hours
|
|
131
|
+
const ageMs = Date.now() - new Date(r.retrievedAt).getTime();
|
|
132
|
+
const isLive = ageMs < 24 * 60 * 60 * 1000;
|
|
133
|
+
|
|
134
|
+
let ageLabel: string;
|
|
135
|
+
if (ageMs < 60 * 1000) ageLabel = "just now";
|
|
136
|
+
else if (ageMs < 60 * 60 * 1000) ageLabel = `${Math.floor(ageMs / 60000)}m ago`;
|
|
137
|
+
else if (ageMs < 24 * 60 * 60 * 1000) ageLabel = `${Math.floor(ageMs / 3600000)}h ago`;
|
|
138
|
+
else ageLabel = `${Math.floor(ageMs / 86400000)}d ago`;
|
|
139
|
+
|
|
140
|
+
return { displayPrice, isLive, ageLabel };
|
|
141
|
+
}
|