@tokcalc/mcp-server 0.1.3 → 0.2.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +86 -406
  2. package/dist/http.js +23504 -0
  3. package/dist/index.js +21290 -0
  4. package/package.json +36 -92
  5. package/.zscripts/build.sh +0 -175
  6. package/.zscripts/database-runtime-build.sh +0 -33
  7. package/.zscripts/dev.pid +0 -1
  8. package/.zscripts/dev.sh +0 -154
  9. package/.zscripts/mini-services-build.sh +0 -78
  10. package/.zscripts/mini-services-install.sh +0 -65
  11. package/.zscripts/mini-services-start.sh +0 -123
  12. package/.zscripts/python-runtime-build.sh +0 -120
  13. package/.zscripts/start.sh +0 -145
  14. package/CAPACITY_STUDY.md +0 -283
  15. package/CODE_OF_CONDUCT.md +0 -55
  16. package/CONTRIBUTING.md +0 -177
  17. package/Caddyfile +0 -23
  18. package/LICENSE +0 -204
  19. package/bun.lock +0 -1965
  20. package/components.json +0 -21
  21. package/db/custom.db +0 -0
  22. package/download/README.md +0 -1
  23. package/download/tokcalc-dark-calculator.png +0 -0
  24. package/download/tokcalc-dark-default.png +0 -0
  25. package/download/tokcalc-demo.webm +0 -0
  26. package/download/tokcalc-github-link.png +0 -0
  27. package/download/tokcalc-hydration-fixed.png +0 -0
  28. package/download/tokcalc-issue-resolved.png +0 -0
  29. package/download/tokcalc-light-mode.png +0 -0
  30. package/download/tokcalc-light-reference.png +0 -0
  31. package/download/tokcalc-long-context-qwen.png +0 -0
  32. package/download/tokcalc-long-context.png +0 -0
  33. package/download/tokcalc-og-image-preview.png +0 -0
  34. package/download/tokcalc-phase2-3.png +0 -0
  35. package/download/tokcalc-plain-english.png +0 -0
  36. package/download/tokcalc-preview.png +0 -0
  37. package/download/tokcalc-share-bvb.png +0 -0
  38. package/download/tokcalc-share-feature.png +0 -0
  39. package/download/tokcalc-tab-build-vs-buy.png +0 -0
  40. package/download/tokcalc-tab-calculator.png +0 -0
  41. package/download/tokcalc-tab-reference.png +0 -0
  42. package/eslint.config.mjs +0 -50
  43. package/examples/websocket/frontend.tsx +0 -196
  44. package/examples/websocket/server.ts +0 -138
  45. package/mini-services/.gitkeep +0 -0
  46. package/mini-services/mcp-server/README.md +0 -86
  47. package/mini-services/mcp-server/bun.lock +0 -202
  48. package/mini-services/mcp-server/index.ts +0 -504
  49. package/mini-services/mcp-server/package.json +0 -40
  50. package/next.config.ts +0 -12
  51. package/postcss.config.mjs +0 -5
  52. package/prisma/schema.prisma +0 -32
  53. package/public/google6f58ca6be85fa903.html +0 -1
  54. package/public/logo.svg +0 -29
  55. package/public/manifest.json +0 -51
  56. package/public/og-icon-256.png +0 -0
  57. package/public/og.png +0 -0
  58. package/public/robots.txt +0 -25
  59. package/public/sitemap.xml +0 -23
  60. package/public/tokcalc-demo.gif +0 -0
  61. package/scripts/og-template.html +0 -120
  62. package/scripts/render-og.mjs +0 -43
  63. package/server.json +0 -21
  64. package/src/app/api/pricing/aws/route.ts +0 -186
  65. package/src/app/api/pricing/azure/route.ts +0 -168
  66. package/src/app/api/pricing/gcp/route.ts +0 -230
  67. package/src/app/api/pricing/vast-ai/route.ts +0 -164
  68. package/src/app/api/route.ts +0 -5
  69. package/src/app/compare/h100-vs-h200/layout.tsx +0 -30
  70. package/src/app/compare/h100-vs-h200/page.tsx +0 -328
  71. package/src/app/globals.css +0 -122
  72. package/src/app/layout.tsx +0 -276
  73. package/src/app/page.tsx +0 -2670
  74. package/src/components/azure-live-pricing.tsx +0 -185
  75. package/src/components/benchmark-import.tsx +0 -340
  76. package/src/components/confidence-badge.tsx +0 -116
  77. package/src/components/live-pricing-comparison.tsx +0 -241
  78. package/src/components/theme-provider.tsx +0 -11
  79. package/src/components/theme-toggle.tsx +0 -55
  80. package/src/components/ui/accordion.tsx +0 -66
  81. package/src/components/ui/alert-dialog.tsx +0 -157
  82. package/src/components/ui/alert.tsx +0 -66
  83. package/src/components/ui/aspect-ratio.tsx +0 -11
  84. package/src/components/ui/avatar.tsx +0 -53
  85. package/src/components/ui/badge.tsx +0 -46
  86. package/src/components/ui/breadcrumb.tsx +0 -109
  87. package/src/components/ui/button.tsx +0 -59
  88. package/src/components/ui/calendar.tsx +0 -213
  89. package/src/components/ui/card.tsx +0 -92
  90. package/src/components/ui/carousel.tsx +0 -241
  91. package/src/components/ui/chart.tsx +0 -353
  92. package/src/components/ui/checkbox.tsx +0 -32
  93. package/src/components/ui/collapsible.tsx +0 -33
  94. package/src/components/ui/command.tsx +0 -184
  95. package/src/components/ui/context-menu.tsx +0 -252
  96. package/src/components/ui/dialog.tsx +0 -143
  97. package/src/components/ui/drawer.tsx +0 -135
  98. package/src/components/ui/dropdown-menu.tsx +0 -257
  99. package/src/components/ui/form.tsx +0 -167
  100. package/src/components/ui/hover-card.tsx +0 -44
  101. package/src/components/ui/input-otp.tsx +0 -77
  102. package/src/components/ui/input.tsx +0 -21
  103. package/src/components/ui/label.tsx +0 -24
  104. package/src/components/ui/menubar.tsx +0 -276
  105. package/src/components/ui/navigation-menu.tsx +0 -168
  106. package/src/components/ui/pagination.tsx +0 -127
  107. package/src/components/ui/popover.tsx +0 -48
  108. package/src/components/ui/progress.tsx +0 -31
  109. package/src/components/ui/radio-group.tsx +0 -45
  110. package/src/components/ui/resizable.tsx +0 -56
  111. package/src/components/ui/scroll-area.tsx +0 -58
  112. package/src/components/ui/select.tsx +0 -185
  113. package/src/components/ui/separator.tsx +0 -28
  114. package/src/components/ui/sheet.tsx +0 -139
  115. package/src/components/ui/sidebar.tsx +0 -726
  116. package/src/components/ui/skeleton.tsx +0 -13
  117. package/src/components/ui/slider.tsx +0 -63
  118. package/src/components/ui/sonner.tsx +0 -25
  119. package/src/components/ui/switch.tsx +0 -31
  120. package/src/components/ui/table.tsx +0 -116
  121. package/src/components/ui/tabs.tsx +0 -66
  122. package/src/components/ui/textarea.tsx +0 -18
  123. package/src/components/ui/toast.tsx +0 -129
  124. package/src/components/ui/toaster.tsx +0 -35
  125. package/src/components/ui/toggle-group.tsx +0 -73
  126. package/src/components/ui/toggle.tsx +0 -47
  127. package/src/components/ui/tooltip.tsx +0 -61
  128. package/src/components/vast-ai-live-pricing.tsx +0 -176
  129. package/src/hooks/use-mobile.ts +0 -19
  130. package/src/hooks/use-toast.ts +0 -194
  131. package/src/lib/benchmark-parser-sglang.ts +0 -150
  132. package/src/lib/benchmark-parser-tokcalc.ts +0 -247
  133. package/src/lib/benchmark-parser-trtllm.ts +0 -152
  134. package/src/lib/benchmark-parser-vllm.ts +0 -198
  135. package/src/lib/benchmark-schema.ts +0 -263
  136. package/src/lib/db.ts +0 -13
  137. package/src/lib/engine-presets.ts +0 -183
  138. package/src/lib/price-schema.ts +0 -141
  139. package/src/lib/token-calc.ts +0 -808
  140. package/src/lib/track.ts +0 -31
  141. package/src/lib/url-state.ts +0 -256
  142. package/src/lib/utils.ts +0 -6
  143. package/tailwind.config.ts +0 -64
  144. package/tests/database-runtime-build.sh +0 -75
  145. package/tests/python-runtime-build.sh +0 -64
  146. package/tests/python-runtime-container.sh +0 -31
  147. package/tool-results/bash_1789888171144_2c5381860539.txt +0 -161
  148. package/tool-results/bash_1789888175925_49c53ba3c61b.txt +0 -191
  149. package/tool-results/bash_1789888181202_49c53ba3c61b.txt +0 -191
  150. package/tool-results/bash_1789888195219_4a86a5c91411.txt +0 -200
  151. package/tool-results/bash_1789888203128_6cca13c71b47.txt +0 -199
  152. package/tool-results/bash_1789929256963_2a52aff0d0a8.txt +0 -160
  153. package/tool-results/read_1789888151021_69f58eec6a5b.txt +0 -653
  154. package/tool-results/read_1789888153837_1d3a8bfc2a94.txt +0 -653
  155. package/tool-results/read_1789888163087_ccc406d47505.txt +0 -122
  156. package/tool-results/read_1789888167347_67d1d7c9830a.txt +0 -122
  157. package/tool-results/read_1789929252529_d90e8f383a25.txt +0 -285
  158. package/tsconfig.json +0 -42
  159. package/upload/Pasted Content_1789887800864.txt +0 -652
  160. package/upload/Pasted Content_1789887909561.txt +0 -652
  161. package/upload/Pasted Content_1789887918428.txt +0 -652
  162. package/upload/Pasted Content_1789887959420.txt +0 -652
  163. package/upload/Pasted Content_1789888020485.txt +0 -652
  164. package/upload/Pasted Content_1789888058079.txt +0 -652
  165. package/upload/Pasted Content_1789888885033.txt +0 -686
  166. package/upload/Pasted Content_1789928912741.txt +0 -285
  167. package/upload/Pasted Content_1789928938402.txt +0 -285
  168. package/upload/Pasted Content_1789929160389.txt +0 -285
  169. package/upload/Pasted Content_1789929176660.txt +0 -285
  170. package/upload/issue_vision.json +0 -28
  171. package/upload/pasted_image_1789883175209.png +0 -0
  172. package/upload/pasted_image_1789899056690.png +0 -0
  173. package/upload/pasted_image_1789900371483.png +0 -0
  174. package/upload/pasted_image_1789900472823.png +0 -0
  175. package/upload/pasted_image_1789900490374.png +0 -0
  176. package/upload/pasted_image_1789900585552.png +0 -0
  177. package/upload/pasted_image_1789900606519.png +0 -0
  178. package/upload/pasted_image_1789901598705.png +0 -0
  179. package/upload/pasted_image_1789901613545.png +0 -0
  180. package/upload/pasted_image_1789978382674.png +0 -0
  181. package/upload/pasted_image_1789978392749.png +0 -0
  182. package/upload/pasted_image_1789978474879.png +0 -0
  183. package/upload/pasted_image_1789978523652.png +0 -0
  184. package/upload/pasted_image_1789984219089.png +0 -0
  185. package/upload/pasted_image_1789984491896.png +0 -0
  186. package/upload/pasted_image_1789985017950.png +0 -0
  187. package/upload/pasted_image_1789985036765.png +0 -0
  188. package/upload/pasted_image_1789985049848.png +0 -0
  189. package/upload/pasted_image_1790002427833.png +0 -0
  190. package/upload/pasted_image_1790002659944.png +0 -0
  191. package/upload/pasted_image_1790037038476.png +0 -0
  192. package/upload/screenshot_analysis.json +0 -28
  193. package/upload/vision_output.json +0 -28
@@ -1,263 +0,0 @@
1
- /**
2
- * Benchmark provenance schema for tokcalc.
3
- *
4
- * Based on the Perplexity research brief which specified:
5
- * "Do not build a single scalar called 'tokens/sec.' A valid observed record
6
- * should always preserve the workload, serving software, engine flags, hardware
7
- * topology, model revision, quantization, and metric definition."
8
- *
9
- * This file defines:
10
- * 1. The BenchmarkRecord type (50+ fields across 7 categories)
11
- * 2. A confidence tier system (official → community → unverified)
12
- * 3. Parser interface for ingesting raw benchmark artifacts
13
- *
14
- * Design principle: tokcalc retains separate estimate types:
15
- * - Theoretical model (what we have now — physics formulas)
16
- * - Observed benchmark (this file — real measured data)
17
- * - Calibrated estimate (theory adjusted by observed data — future)
18
- * - User-provided run (user uploads their benchmark — future)
19
- */
20
-
21
- /* ============================================================
22
- CONFIDENCE TIERS
23
- ============================================================ */
24
-
25
- export type ConfidenceTier =
26
- | "official_audited" // MLPerf submission, official vendor benchmark
27
- | "official_reproducible" // vLLM/SGLang/TRT-LLM docs example with full config
28
- | "community_reproducible" // User ran it with documented config + raw artifact
29
- | "community_unverified" // User-reported number without raw artifact
30
- | "derived"; // Computed from other records (e.g., calibration)
31
-
32
- export type VerificationStatus =
33
- | "unreviewed"
34
- | "schema_valid"
35
- | "reproduced"
36
- | "curated"
37
- | "rejected";
38
-
39
- /* ============================================================
40
- CORE BENCHMARK RECORD
41
- ============================================================ */
42
-
43
- export interface BenchmarkRecord {
44
- /* ---- Core identity and provenance ---- */
45
- benchmark_id: string;
46
- source_name: "vllm" | "sglang" | "tensorrt_llm" | "mlperf" | "llmperf" | "community" | "manual_curated";
47
- source_url: string;
48
- source_license: string;
49
- ingestion_method: "official_api" | "official_repo" | "uploaded_raw_artifact" | "manual_curated";
50
- retrieved_at: string; // ISO 8601
51
- benchmark_started_at?: string; // ISO 8601
52
- raw_artifact_url?: string;
53
- artifact_sha256?: string;
54
- parser_name?: string;
55
- parser_version?: string;
56
- confidence_tier: ConfidenceTier;
57
- verification_status: VerificationStatus;
58
-
59
- /* ---- Model identity ---- */
60
- model_id: string;
61
- model_display_name: string;
62
- model_repository?: string;
63
- model_revision: string; // commit hash / version / date
64
- architecture?: "dense" | "moe" | "vlm" | "encoder" | "decoder_only";
65
- parameter_count_b?: number;
66
- context_window_tokens?: number;
67
- quantization_format: string; // FP16, FP8, GGUF Q4_K_M, AWQ, etc.
68
- weight_dtype?: string; // float16, bfloat16, float8_e4m3, int8, etc.
69
- kv_cache_dtype?: string; // "float16" by default
70
-
71
- /* ---- Hardware and topology ---- */
72
- accelerator_vendor: "NVIDIA" | "AMD" | "Intel" | "Apple" | "Google" | "Groq" | "Cerebras" | string;
73
- accelerator_model: string;
74
- gpu_count: number;
75
- memory_per_gpu_gb: number;
76
- host_cpu?: string;
77
- host_memory_gb?: number;
78
- interconnect?: string;
79
- topology?: string;
80
- cloud_provider?: string;
81
- instance_type?: string;
82
- region?: string;
83
- driver_version?: string;
84
- cuda_rocm_metal_version?: string;
85
-
86
- /* ---- Serving stack and configuration ---- */
87
- serving_engine: "vLLM" | "SGLang" | "TensorRT-LLM" | "TGI" | "llama.cpp" | "MLX" | string;
88
- engine_version: string;
89
- engine_command?: string;
90
- engine_config?: {
91
- tensor_parallel_size?: number;
92
- pipeline_parallel_size?: number;
93
- max_model_len?: number;
94
- max_num_seqs?: number;
95
- max_num_batched_tokens?: number;
96
- prefix_cache_enabled?: boolean;
97
- speculative_decoding?: {
98
- enabled: boolean;
99
- draft_model?: string;
100
- acceptance_rate?: number;
101
- };
102
- chunked_prefill_enabled?: boolean;
103
- attention_backend?: string;
104
- };
105
-
106
- /* ---- Workload definition ---- */
107
- workload_type: "offline" | "online_poisson" | "fixed_rate" | "closed_loop" | "trace_replay" | "synthetic";
108
- dataset_name?: string;
109
- request_count: number;
110
- concurrency: number;
111
- arrival_rate_rps?: number;
112
- input_tokens_mean: number;
113
- input_tokens_p50?: number;
114
- input_tokens_p95?: number;
115
- input_tokens_p99?: number;
116
- output_tokens_mean: number;
117
- output_tokens_p50?: number;
118
- output_tokens_p95?: number;
119
- output_tokens_p99?: number;
120
- shared_prefix_tokens_mean?: number;
121
- shared_prefix_hit_rate?: number;
122
- streaming_enabled: boolean;
123
- sampling_config?: {
124
- temperature?: number;
125
- top_p?: number;
126
- max_tokens?: number;
127
- seed?: number;
128
- };
129
-
130
- /* ---- Observed metrics ---- */
131
- successful_requests: number;
132
- failed_requests: number;
133
- request_throughput_rps?: number;
134
- output_token_throughput_tps: number; // THE key metric
135
- total_token_throughput_tps?: number;
136
- ttft_mean_ms?: number;
137
- ttft_p50_ms?: number;
138
- ttft_p95_ms?: number;
139
- ttft_p99_ms?: number;
140
- itl_mean_ms?: number;
141
- itl_p50_ms?: number;
142
- itl_p95_ms?: number;
143
- itl_p99_ms?: number;
144
- tpot_mean_ms?: number; // Time per output token (vLLM's term)
145
- e2e_latency_p50_ms?: number;
146
- e2e_latency_p95_ms?: number;
147
- e2e_latency_p99_ms?: number;
148
- gpu_memory_peak_gb?: number;
149
- gpu_utilization_mean_pct?: number;
150
- power_mean_w?: number;
151
- energy_per_output_token_j?: number;
152
-
153
- /* ---- Comparability and display ---- */
154
- metric_definition_version: string;
155
- slo_ttft_ms?: number;
156
- slo_itl_ms?: number;
157
- slo_e2e_ms?: number;
158
- comparability_group?: string;
159
- normalization_notes?: string;
160
- known_limitations?: string;
161
- citation_text: string;
162
- attribution_required: boolean;
163
- commercial_redistribution_allowed: "yes" | "no" | "unknown" | "contract_required";
164
- }
165
-
166
- /* ============================================================
167
- CALIBRATION RESULT — compares theoretical vs observed
168
- ============================================================ */
169
-
170
- export interface CalibrationResult {
171
- /** What tokcalc's formulas predicted */
172
- estimated: {
173
- decodeTokensPerSec: number;
174
- aggregateTokensPerSec: number;
175
- prefillTokensPerSec: number;
176
- ttftMs: number;
177
- itlMs: number;
178
- modelSizeGb: number;
179
- };
180
- /** What the benchmark actually measured */
181
- observed: {
182
- outputTokenThroughputTps?: number;
183
- ttftMeanMs?: number;
184
- ttftP95Ms?: number;
185
- itlMeanMs?: number;
186
- itlP95Ms?: number;
187
- gpuMemoryPeakGb?: number;
188
- };
189
- /** Calibration error (observed - estimated) / estimated × 100 */
190
- calibrationErrors: {
191
- throughputErrorPct?: number; // positive = observed was faster than predicted
192
- ttftErrorPct?: number; // positive = observed TTFT was longer than predicted
193
- itlErrorPct?: number;
194
- memoryErrorPct?: number;
195
- };
196
- /** Does the observed data validate or contradict our model? */
197
- verdict: "validated" | "underestimated" | "overestimated" | "insufficient_data";
198
- }
199
-
200
- /* ============================================================
201
- PARSER INTERFACE
202
- ============================================================ */
203
-
204
- export interface BenchmarkParser {
205
- name: string;
206
- version: string;
207
- /** Detect if raw text looks like this engine's output */
208
- detect(rawText: string): boolean;
209
- /** Parse raw text into a BenchmarkRecord */
210
- parse(rawText: string): BenchmarkRecord | null;
211
- /** Human-readable format name */
212
- formatLabel: string;
213
- }
214
-
215
- /* ============================================================
216
- HELPER: generate UUID (no dependency)
217
- ============================================================ */
218
-
219
- export function generateBenchmarkId(): string {
220
- // RFC 4122 v4 UUID — no dependency needed
221
- return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (c) => {
222
- const r = (Math.random() * 16) | 0;
223
- const v = c === "x" ? r : (r & 0x3) | 0x8;
224
- return v.toString(16);
225
- });
226
- }
227
-
228
- /* ============================================================
229
- HELPER: format calibration verdict for display
230
- ============================================================ */
231
-
232
- export function formatCalibrationVerdict(verdict: CalibrationResult["verdict"]): {
233
- label: string;
234
- color: string;
235
- description: string;
236
- } {
237
- switch (verdict) {
238
- case "validated":
239
- return {
240
- label: "Validated",
241
- color: "text-emerald-500",
242
- description: "Observed benchmark matches theoretical estimate within ±20%. Formula is accurate for this config.",
243
- };
244
- case "underestimated":
245
- return {
246
- label: "Underestimated",
247
- color: "text-amber-500",
248
- description: "Real performance was BETTER than tokcalc predicted. Formula may be too conservative.",
249
- };
250
- case "overestimated":
251
- return {
252
- label: "Overestimated",
253
- color: "text-red-500",
254
- description: "Real performance was WORSE than tokcalc predicted. Formula may be too optimistic.",
255
- };
256
- case "insufficient_data":
257
- return {
258
- label: "Insufficient data",
259
- color: "text-muted-foreground",
260
- description: "Not enough observed metrics to calibrate against the theoretical estimate.",
261
- };
262
- }
263
- }
package/src/lib/db.ts DELETED
@@ -1,13 +0,0 @@
1
- import { PrismaClient } from '@prisma/client'
2
-
3
- const globalForPrisma = globalThis as unknown as {
4
- prisma: PrismaClient | undefined
5
- }
6
-
7
- export const db =
8
- globalForPrisma.prisma ??
9
- new PrismaClient({
10
- log: ['query'],
11
- })
12
-
13
- if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = db
@@ -1,183 +0,0 @@
1
- /**
2
- * Engine-aware presets for tokcalc.
3
- *
4
- * When the user picks an inference engine (vLLM, SGLang, TRT-LLM, llama.cpp),
5
- * the calculator adjusts:
6
- * 1. Efficiency factors (ETA_MEM, ETA_COMPUTE) — each engine has different
7
- * real-world utilization due to kernel optimizations, scheduler design, etc.
8
- * 2. Default continuous batching multiplier — vLLM/SGLang have it on by
9
- * default (2.0×), llama.cpp doesn't (1.0×)
10
- * 3. Supported quantizations — filters the quant dropdown to only show
11
- * formats the selected engine supports
12
- * 4. Engine-specific notes — helps users understand trade-offs
13
- *
14
- * Per Perplexity research (Prompt #2):
15
- * "There is no universally best engine; workload shape matters."
16
- * "comparisons emphasize that throughput leadership varies with workload,
17
- * structured output, prefix reuse, and operational maturity"
18
- */
19
-
20
- import type { Quantization } from "./token-calc";
21
-
22
- export type EngineId = "vllm" | "sglang" | "trtllm" | "llamacpp" | "generic";
23
-
24
- export interface EnginePreset {
25
- id: EngineId;
26
- name: string;
27
- shortName: string;
28
- description: string;
29
- /** Memory bandwidth utilization (real-world) — replaces ETA_MEM when engine is selected */
30
- etaMem: number;
31
- /** Compute utilization — replaces ETA_COMPUTE */
32
- etaCompute: number;
33
- /** Default continuous batching multiplier when engine is selected */
34
- defaultBatchingMultiplier: number;
35
- /** Which quantization formats this engine supports */
36
- supportedQuants: Quantization[];
37
- /** Engine-specific notes shown in UI */
38
- notes: string;
39
- /** Source URL for the engine */
40
- sourceUrl: string;
41
- /** Whether continuous batching is on by default in this engine */
42
- batchingOnByDefault: boolean;
43
- /** Whether prefix caching is on by default */
44
- prefixCacheOnByDefault: boolean;
45
- /** Key features of this engine */
46
- features: string[];
47
- }
48
-
49
- export const ENGINE_PRESETS: EnginePreset[] = [
50
- {
51
- id: "vllm",
52
- name: "vLLM",
53
- shortName: "vLLM",
54
- description: "High-throughput LLM serving with PagedAttention + continuous batching. The most popular open-source serving engine.",
55
- etaMem: 0.65, // Default — PagedAttention reduces waste to <4%
56
- etaCompute: 0.50, // Default
57
- defaultBatchingMultiplier: 2.0, // vLLM enables continuous batching by default
58
- supportedQuants: ["fp32", "fp16", "bf16", "int8", "fp8", "gptq4", "awq4", "int4"],
59
- notes: "PagedAttention + continuous batching ON by default. Real-world throughput is 2-4× naive batching. Supports FP8 natively on H100/H200. GGUF not supported — use llama.cpp for that.",
60
- sourceUrl: "https://docs.vllm.ai",
61
- batchingOnByDefault: true,
62
- prefixCacheOnByDefault: true, // APC (Automatic Prefix Caching)
63
- features: [
64
- "PagedAttention (KV cache memory management)",
65
- "Continuous batching (iteration-level scheduling)",
66
- "Automatic prefix caching (APC)",
67
- "Tensor parallelism (up to 8×)",
68
- "Speculative decoding",
69
- "FP8 native on H100/H200",
70
- ],
71
- },
72
- {
73
- id: "sglang",
74
- name: "SGLang",
75
- shortName: "SGLang",
76
- description: "High-performance serving with RadixAttention (prefix-tree KV cache) + structured generation. Best for agent workloads with repeated prefixes.",
77
- etaMem: 0.68, // Slightly higher — RadixAttention is more efficient than APC
78
- etaCompute: 0.52,
79
- defaultBatchingMultiplier: 2.0, // Continuous batching on by default
80
- supportedQuants: ["fp32", "fp16", "bf16", "int8", "fp8", "gptq4", "awq4"],
81
- notes: "RadixAttention provides automatic prefix reuse without explicit cache management. Better for agent/RAG workloads with shared system prompts. Slightly higher memory efficiency than vLLM for prefix-heavy workloads.",
82
- sourceUrl: "https://github.com/sgl-project/sglang",
83
- batchingOnByDefault: true,
84
- prefixCacheOnByDefault: true, // RadixAttention IS the prefix cache
85
- features: [
86
- "RadixAttention (prefix-tree KV cache)",
87
- "Continuous batching",
88
- "Structured generation (JSON mode)",
89
- "Tensor parallelism",
90
- "Speculative decoding",
91
- "PD disaggregation (experimental)",
92
- ],
93
- },
94
- {
95
- id: "trtllm",
96
- name: "TensorRT-LLM",
97
- shortName: "TRT-LLM",
98
- description: "NVIDIA-optimized inference engine. Best raw throughput on NVIDIA GPUs with quantized models. Requires model compilation.",
99
- etaMem: 0.70, // NVIDIA-optimized kernels achieve higher utilization
100
- etaCompute: 0.55,
101
- defaultBatchingMultiplier: 2.0,
102
- supportedQuants: ["fp32", "fp16", "bf16", "int8", "fp8", "nvfp4", "int4"],
103
- notes: "NVIDIA-optimized kernels achieve 5-10% higher utilization than vLLM on same hardware. NVFP4 native on Blackwell (B200/B300). Requires model compilation (TRT engine build) — slower to iterate than vLLM. Inflight batching is TRT-LLM's version of continuous batching.",
104
- sourceUrl: "https://github.com/NVIDIA/TensorRT-LLM",
105
- batchingOnByDefault: true, // Inflight batching
106
- prefixCacheOnByDefault: true,
107
- features: [
108
- "NVIDIA-optimized kernels (FlashAttention, fused MLP)",
109
- "Inflight batching (continuous batching equivalent)",
110
- "FP8 + NVFP4 native on H100/H200/B200",
111
- "Model compilation (TRT engine build)",
112
- "Tensor parallelism",
113
- "Plugin system for custom layers",
114
- ],
115
- },
116
- {
117
- id: "llamacpp",
118
- name: "llama.cpp",
119
- shortName: "llama.cpp",
120
- description: "C++ inference engine for CPU, Apple Silicon, and consumer GPUs. Best for local/edge deployment. GGUF format.",
121
- etaMem: 0.55, // Lower — CPU/GPU mixed, less optimized than vLLM
122
- etaCompute: 0.40, // Lower — CPU-bound on non-GPU paths
123
- defaultBatchingMultiplier: 1.0, // No continuous batching by default
124
- supportedQuants: ["fp32", "fp16", "bf16", "int8", "gguf-q2k", "gguf-q3km", "gguf-q4km", "gguf-q5km", "gguf-q6k", "gguf-q8", "int4"],
125
- notes: "Best for local/edge: Mac M-series (MLX backend), consumer GPUs (CUDA), CPU-only machines. Supports all GGUF quantization variants (Q2_K through Q8_0). No continuous batching — throughput is single-stream only. KV cache is contiguous (not paged).",
126
- sourceUrl: "https://github.com/ggml-org/llama.cpp",
127
- batchingOnByDefault: false,
128
- prefixCacheOnByDefault: false,
129
- features: [
130
- "GGUF format (Q2_K through Q8_0)",
131
- "CPU inference (no GPU needed)",
132
- "Apple Silicon (Metal backend)",
133
- "Consumer GPU support (CUDA)",
134
- "Lowest memory footprint",
135
- "No server needed (CLI + server mode)",
136
- ],
137
- },
138
- {
139
- id: "generic",
140
- name: "Generic (engine-agnostic)",
141
- shortName: "Generic",
142
- description: "No specific engine selected. Uses conservative default efficiency factors. Best when you don't know which engine you'll use yet.",
143
- etaMem: 0.65, // Default — same as current ETA_MEM
144
- etaCompute: 0.50, // Default
145
- defaultBatchingMultiplier: 1.5, // Conservative default
146
- supportedQuants: [
147
- "fp32", "fp16", "bf16", "int8", "int4",
148
- "gguf-q2k", "gguf-q3km", "gguf-q4km", "gguf-q5km", "gguf-q6k", "gguf-q8",
149
- "gptq4", "awq4", "exl2-6bpw", "fp8", "nvfp4",
150
- ],
151
- notes: "Conservative estimates. When you know your engine, select it for more accurate results. Each engine has different real-world utilization due to kernel optimizations and scheduler design.",
152
- sourceUrl: "",
153
- batchingOnByDefault: false,
154
- prefixCacheOnByDefault: false,
155
- features: [],
156
- },
157
- ];
158
-
159
- export const ENGINE_MAP: Record<EngineId, EnginePreset> = Object.fromEntries(
160
- ENGINE_PRESETS.map((e) => [e.id, e]),
161
- ) as Record<EngineId, EnginePreset>;
162
-
163
- /**
164
- * Get the engine preset by ID (defaults to "generic").
165
- */
166
- export function getEngine(id: EngineId | string | undefined): EnginePreset {
167
- return ENGINE_MAP[id as EngineId] || ENGINE_MAP.generic;
168
- }
169
-
170
- /**
171
- * Check if a quantization format is supported by an engine.
172
- */
173
- export function isQuantSupported(engine: EngineId, quant: Quantization): boolean {
174
- const preset = getEngine(engine);
175
- return preset.supportedQuants.includes(quant);
176
- }
177
-
178
- /**
179
- * Get the list of quantizations supported by an engine, as Quantization[].
180
- */
181
- export function getSupportedQuants(engine: EngineId): Quantization[] {
182
- return getEngine(engine).supportedQuants;
183
- }
@@ -1,141 +0,0 @@
1
- /**
2
- * Price record type for live cloud GPU pricing.
3
- *
4
- * Based on the Perplexity research brief (Prompt #3):
5
- * "Use an append-only historical model. A price should never be overwritten
6
- * without retaining the prior record."
7
- *
8
- * Phase 1: Azure Retail Prices API (unauthenticated, easiest)
9
- * Phase 2: Vast.ai marketplace API (dynamic spot pricing)
10
- * Phase 3: AWS / GCP / OCI official APIs (authenticated)
11
- */
12
-
13
- export type PriceProvider =
14
- | "azure"
15
- | "aws"
16
- | "gcp"
17
- | "oci"
18
- | "vast_ai"
19
- | "runpod"
20
- | "lambda"
21
- | "modal"
22
- | "tensor"
23
- | "default";
24
-
25
- export type PurchaseOption =
26
- | "on_demand"
27
- | "spot"
28
- | "reserved"
29
- | "savings_plan"
30
- | "marketplace"
31
- | "serverless";
32
-
33
- export type BillingUnit =
34
- | "instance_hour"
35
- | "gpu_hour"
36
- | "gpu_second";
37
-
38
- export interface PriceRecord {
39
- id: string;
40
- provider: PriceProvider;
41
- productFamily: "cloud_vm" | "gpu_marketplace";
42
-
43
- /** Cloud SKU identifier (e.g., "Standard_ND96isr_H100_v5") */
44
- providerSku: string;
45
- providerSkuName: string;
46
-
47
- /** GPU info extracted from the SKU */
48
- acceleratorVendor?: string;
49
- acceleratorModel?: string;
50
- acceleratorCount?: number;
51
- acceleratorMemoryGb?: number;
52
-
53
- /** Region/zone */
54
- region?: string;
55
- locationLabel?: string;
56
-
57
- purchaseOption: PurchaseOption;
58
- billingUnit: BillingUnit;
59
-
60
- /** Price in USD */
61
- amount: number;
62
- currency: "USD";
63
-
64
- /** Per-GPU derived price (if instance-level, divide by GPU count) */
65
- derivedPricePerGpuHour?: number;
66
- isDerived: boolean;
67
- derivationFormula?: string;
68
-
69
- /** Validity window */
70
- effectiveFrom?: string;
71
- effectiveTo?: string;
72
- retrievedAt: string;
73
-
74
- /** Provenance */
75
- sourceType: "official_api" | "official_price_page" | "manual_verified";
76
- sourceUrl: string;
77
-
78
- /** Display confidence */
79
- confidence: "high" | "medium" | "low";
80
- }
81
-
82
- /**
83
- * Match a GPU from tokcalc's catalog to a cloud provider's SKU.
84
- * Returns the best guess based on GPU name + VRAM.
85
- */
86
- export function matchGpuToCloudSku(
87
- gpuName: string,
88
- gpuVramGb: number,
89
- provider: PriceProvider,
90
- ): { sku: string; skuName: string } | null {
91
- const name = gpuName.toLowerCase();
92
-
93
- // Azure SKU patterns
94
- if (provider === "azure") {
95
- if (name.includes("h100")) return { sku: "Standard_ND96isr_H100_v5", skuName: "ND H100 v5 (96 GPU = 8× H100)" };
96
- if (name.includes("h200")) return { sku: "Standard_ND-H200-v5", skuName: "ND H200 v5" };
97
- if (name.includes("a100") && gpuVramGb >= 80) return { sku: "Standard_ND96asr_v4", skuName: "ND A100 v4 (8× A100 80GB)" };
98
- if (name.includes("a100") && gpuVramGb < 80) return { sku: "Standard_ND96amsr_A100_v4", skuName: "ND A100 v4 (40GB)" };
99
- if (name.includes("v100")) return { sku: "Standard_NC24ads_A100_v4", skuName: "NC A100 v4" };
100
- if (name.includes("t4")) return { sku: "Standard_NC4as_T4_v3", skuName: "NC T4 v3" };
101
- if (name.includes("l40s") || name.includes("l40")) return { sku: "Standard_NDm_A100_v4", skuName: "NDm L40S" };
102
- }
103
-
104
- // AWS SKU patterns
105
- if (provider === "aws") {
106
- if (name.includes("h100")) return { sku: "p5.48xlarge", skuName: "p5.48xlarge (8× H100)" };
107
- if (name.includes("h200")) return { sku: "p5e.48xlarge", skuName: "p5e.48xlarge (8× H200)" };
108
- if (name.includes("b200")) return { sku: "p6.48xlarge", skuName: "p6.48xlarge (8× B200)" };
109
- if (name.includes("a100") && gpuVramGb >= 80) return { sku: "p4de.24xlarge", skuName: "p4de.24xlarge (8× A100 80GB)" };
110
- if (name.includes("a100") && gpuVramGb < 80) return { sku: "p4d.24xlarge", skuName: "p4d.24xlarge (8× A100 40GB)" };
111
- if (name.includes("l40s")) return { sku: "g6.48xlarge", skuName: "g6.48xlarge (8× L40S)" };
112
- if (name.includes("l4")) return { sku: "g6.xlarge", skuName: "g6.xlarge (1× L4)" };
113
- if (name.includes("t4")) return { sku: "g4dn.12xlarge", skuName: "g4dn.12xlarge (4× T4)" };
114
- }
115
-
116
- return null;
117
- }
118
-
119
- /**
120
- * Format a price record for display in the UI.
121
- */
122
- export function formatPriceRecord(r: PriceRecord): {
123
- displayPrice: string;
124
- isLive: boolean;
125
- ageLabel: string;
126
- } {
127
- const perGpu = r.derivedPricePerGpuHour ?? r.amount;
128
- const displayPrice = `$${perGpu.toFixed(2)}/hr`;
129
-
130
- // "Live" if retrieved within last 24 hours
131
- const ageMs = Date.now() - new Date(r.retrievedAt).getTime();
132
- const isLive = ageMs < 24 * 60 * 60 * 1000;
133
-
134
- let ageLabel: string;
135
- if (ageMs < 60 * 1000) ageLabel = "just now";
136
- else if (ageMs < 60 * 60 * 1000) ageLabel = `${Math.floor(ageMs / 60000)}m ago`;
137
- else if (ageMs < 24 * 60 * 60 * 1000) ageLabel = `${Math.floor(ageMs / 3600000)}h ago`;
138
- else ageLabel = `${Math.floor(ageMs / 86400000)}d ago`;
139
-
140
- return { displayPrice, isLive, ageLabel };
141
- }