@mlx-bun/inference 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (467) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +1475 -2
  3. package/THIRD_PARTY_NOTICES.md +3404 -0
  4. package/dist/native/libmlx_bun_ane.dylib +0 -0
  5. package/dist/native/libmlx_bun_expert_io.dylib +0 -0
  6. package/dist/native/mlx-bun-frame-extract +0 -0
  7. package/examples/fit-model.ts +25 -0
  8. package/examples/qwen3-forward.ts +42 -0
  9. package/examples/qwen3-generate.ts +28 -0
  10. package/examples/trellis-expand.ts +23 -0
  11. package/package.json +129 -4
  12. package/src/adapters/available.ts +110 -0
  13. package/src/adapters/index.ts +3 -0
  14. package/src/adapters/manager.ts +316 -0
  15. package/src/artifacts/auxiliary-files.ts +45 -0
  16. package/src/artifacts/config-dialects.ts +94 -0
  17. package/src/artifacts/config.ts +341 -0
  18. package/src/artifacts/draft-vocabulary.ts +30 -0
  19. package/src/artifacts/expert-offload-build.ts +75 -0
  20. package/src/artifacts/expert-offload.ts +89 -0
  21. package/src/artifacts/experts/index.ts +3 -0
  22. package/src/artifacts/experts/io.ts +386 -0
  23. package/src/artifacts/experts/residency.ts +1134 -0
  24. package/src/artifacts/experts/usage.ts +415 -0
  25. package/src/artifacts/fingerprint.ts +44 -0
  26. package/src/artifacts/glm52/container.ts +305 -0
  27. package/src/artifacts/glm52/expert-layout.ts +249 -0
  28. package/src/artifacts/glm52/index.ts +7 -0
  29. package/src/artifacts/glm52/kernel-layout.ts +124 -0
  30. package/src/artifacts/glm52/layout.ts +152 -0
  31. package/src/artifacts/glm52/memory.ts +500 -0
  32. package/src/artifacts/glm52/resident-weights.ts +135 -0
  33. package/src/artifacts/glm52/weights.ts +503 -0
  34. package/src/artifacts/glm52-config.ts +262 -0
  35. package/src/artifacts/identity.ts +94 -0
  36. package/src/artifacts/index.ts +15 -0
  37. package/src/artifacts/mmap.ts +90 -0
  38. package/src/artifacts/safetensors-writer.ts +249 -0
  39. package/src/artifacts/safetensors.ts +164 -0
  40. package/src/artifacts/turboquant-fold.ts +42 -0
  41. package/src/artifacts/weight-audit.ts +43 -0
  42. package/src/artifacts/weights.ts +255 -0
  43. package/src/contracts/index.ts +2 -0
  44. package/src/contracts/mlx/attention.ts +8 -0
  45. package/src/contracts/mlx/cache.ts +510 -0
  46. package/src/contracts/mlx/checkpoint.ts +10 -0
  47. package/src/contracts/mlx/draft-checkpoint.ts +14 -0
  48. package/src/contracts/mlx/draft-projection.ts +27 -0
  49. package/src/contracts/mlx/draft-target.ts +84 -0
  50. package/src/contracts/mlx/drafter.ts +176 -0
  51. package/src/contracts/mlx/forward-work.ts +23 -0
  52. package/src/contracts/mlx/glm52-weights.ts +25 -0
  53. package/src/contracts/mlx/graph.ts +309 -0
  54. package/src/contracts/mlx/head.ts +6 -0
  55. package/src/contracts/mlx/index.ts +15 -0
  56. package/src/contracts/mlx/kv-codec.ts +9 -0
  57. package/src/contracts/mlx/media.ts +113 -0
  58. package/src/contracts/mlx/positions.ts +16 -0
  59. package/src/contracts/mlx/row-table.ts +2 -0
  60. package/src/contracts/mlx/trainable.ts +103 -0
  61. package/src/contracts/mlx/whisper.ts +96 -0
  62. package/src/contracts/portable/composition.ts +106 -0
  63. package/src/contracts/portable/denoising.ts +17 -0
  64. package/src/contracts/portable/execution.ts +79 -0
  65. package/src/contracts/portable/generation.ts +103 -0
  66. package/src/contracts/portable/graph.ts +80 -0
  67. package/src/contracts/portable/index.ts +15 -0
  68. package/src/contracts/portable/memory-plan.ts +26 -0
  69. package/src/contracts/portable/object-cache.ts +8 -0
  70. package/src/contracts/portable/prefill.ts +63 -0
  71. package/src/contracts/portable/prefix-cache.ts +33 -0
  72. package/src/contracts/portable/prompt.ts +2 -0
  73. package/src/contracts/portable/resources.ts +25 -0
  74. package/src/contracts/portable/rollback.ts +7 -0
  75. package/src/contracts/portable/sampling.ts +44 -0
  76. package/src/contracts/portable/scheduling.ts +54 -0
  77. package/src/contracts/portable/token-delivery.ts +4 -0
  78. package/src/contracts/portable/tsconfig.json +14 -0
  79. package/src/embeddings/index.ts +1 -0
  80. package/src/embeddings/text.ts +85 -0
  81. package/src/execution/admission.ts +59 -0
  82. package/src/execution/batch-group.ts +1240 -0
  83. package/src/execution/batch-types.ts +225 -0
  84. package/src/execution/cancellation.ts +38 -0
  85. package/src/execution/continuation-request.ts +45 -0
  86. package/src/execution/continuation-types.ts +10 -0
  87. package/src/execution/coordinator.ts +70 -0
  88. package/src/execution/denoising-group.ts +180 -0
  89. package/src/execution/engine.ts +56 -0
  90. package/src/execution/fill-group.ts +505 -0
  91. package/src/execution/fit.ts +184 -0
  92. package/src/execution/gateway-binding.ts +178 -0
  93. package/src/execution/grammar-group.ts +224 -0
  94. package/src/execution/index.ts +18 -0
  95. package/src/execution/methods.ts +75 -0
  96. package/src/execution/mixed-iteration.ts +96 -0
  97. package/src/execution/plan.ts +65 -0
  98. package/src/execution/prefill-cohort.ts +100 -0
  99. package/src/execution/prefill-rows.ts +263 -0
  100. package/src/execution/request-policy.ts +19 -0
  101. package/src/execution/row-sampling.ts +49 -0
  102. package/src/execution/scheduler.ts +92 -0
  103. package/src/execution/session.ts +247 -0
  104. package/src/execution/speculative-group.ts +797 -0
  105. package/src/execution/tasks.ts +44 -0
  106. package/src/generation/autoregressive.ts +840 -0
  107. package/src/generation/bindings/autoregressive.ts +104 -0
  108. package/src/generation/bindings/denoising.ts +60 -0
  109. package/src/generation/bindings/prefill-policy.ts +36 -0
  110. package/src/generation/checkpoint-identity.ts +60 -0
  111. package/src/generation/denoising.ts +85 -0
  112. package/src/generation/diffusion.ts +583 -0
  113. package/src/generation/fill/echo-index.ts +188 -0
  114. package/src/generation/fill/index.ts +3 -0
  115. package/src/generation/fill/proposal.ts +69 -0
  116. package/src/generation/fill/schema-rows.ts +506 -0
  117. package/src/generation/fill/session.ts +464 -0
  118. package/src/generation/fill/tool-boundary.ts +103 -0
  119. package/src/generation/fill-append.ts +58 -0
  120. package/src/generation/generate.ts +44 -0
  121. package/src/generation/grammar-step.ts +39 -0
  122. package/src/generation/index.ts +14 -0
  123. package/src/generation/phase-dispatch.ts +115 -0
  124. package/src/generation/prefill.ts +33 -0
  125. package/src/generation/result.ts +49 -0
  126. package/src/generation/scopes.ts +157 -0
  127. package/src/generation/speculative/acceptance.ts +41 -0
  128. package/src/generation/speculative/adaptive-draft-gate.ts +166 -0
  129. package/src/generation/speculative/bindings/assistant-rows.ts +102 -0
  130. package/src/generation/speculative/bindings/autoregressive-rows.ts +124 -0
  131. package/src/generation/speculative/bindings/binding.ts +106 -0
  132. package/src/generation/speculative/bindings/deepspec-rows.ts +33 -0
  133. package/src/generation/speculative/bindings/draft-target.ts +10 -0
  134. package/src/generation/speculative/bindings/glm52-mtp-rows.ts +128 -0
  135. package/src/generation/speculative/bindings/projected-context-rows.ts +78 -0
  136. package/src/generation/speculative/bindings/projected-draft-rows.ts +95 -0
  137. package/src/generation/speculative/bindings/qwen-mtp-rows.ts +336 -0
  138. package/src/generation/speculative/bindings/two-model-rows.ts +63 -0
  139. package/src/generation/speculative/cache-identity.ts +16 -0
  140. package/src/generation/speculative/draft-kind.ts +149 -0
  141. package/src/generation/speculative/draft-policy.ts +41 -0
  142. package/src/generation/speculative/draft-registry.ts +118 -0
  143. package/src/generation/speculative/draft-vocab.ts +110 -0
  144. package/src/generation/speculative/index.ts +17 -0
  145. package/src/generation/speculative/output.ts +49 -0
  146. package/src/generation/speculative/round.ts +307 -0
  147. package/src/generation/speculative/source.ts +144 -0
  148. package/src/generation/speculative/sources/assistant-source.ts +64 -0
  149. package/src/generation/speculative/sources/deepspec-source.ts +70 -0
  150. package/src/generation/speculative/sources/dflash-source.ts +84 -0
  151. package/src/generation/speculative/sources/dflash2-source.ts +86 -0
  152. package/src/generation/speculative/sources/glm52-mtp-source.ts +66 -0
  153. package/src/generation/speculative/sources/ngram-source.ts +165 -0
  154. package/src/generation/speculative/sources/qwen-mtp-source.ts +154 -0
  155. package/src/generation/speculative/sources/two-model.ts +75 -0
  156. package/src/generation/types.ts +194 -0
  157. package/src/index.ts +20 -0
  158. package/src/input/audio/audiotoolbox.ts +112 -0
  159. package/src/input/audio/decode.ts +133 -0
  160. package/src/input/audio/features.ts +202 -0
  161. package/src/input/audio/index.ts +6 -0
  162. package/src/input/audio/transcode.ts +67 -0
  163. package/src/input/audio/whisper-mel.ts +185 -0
  164. package/src/input/audio/whisper-tokenizer.ts +238 -0
  165. package/src/input/chat-template.ts +389 -0
  166. package/src/input/index.ts +6 -0
  167. package/src/input/media-fetch.ts +439 -0
  168. package/src/input/packed-mixed.ts +43 -0
  169. package/src/input/special-tokens.ts +75 -0
  170. package/src/input/token-groups.ts +49 -0
  171. package/src/input/tokenizer.ts +336 -0
  172. package/src/input/tool-call.ts +689 -0
  173. package/src/input/vision/index.ts +3 -0
  174. package/src/input/vision/preprocess.ts +247 -0
  175. package/src/input/vision/prompt.ts +398 -0
  176. package/src/input/vision/qwen3vl-preprocess.ts +409 -0
  177. package/src/input/vision/qwen3vl-prompt.ts +237 -0
  178. package/src/input/vision/video-frames.ts +100 -0
  179. package/src/kernels/ane/fill.ts +101 -0
  180. package/src/kernels/ane/linear.ts +416 -0
  181. package/src/kernels/attention/flash.ts +694 -0
  182. package/src/kernels/attention/kvq4-decode.ts +155 -0
  183. package/src/kernels/attention/masks.ts +40 -0
  184. package/src/kernels/attention/multi-query.ts +193 -0
  185. package/src/kernels/attention/paged.ts +130 -0
  186. package/src/kernels/delta/gated.ts +306 -0
  187. package/src/kernels/delta/index.ts +3 -0
  188. package/src/kernels/delta/silu-rms-heads.ts +69 -0
  189. package/src/kernels/delta/state.ts +74 -0
  190. package/src/kernels/gated-rms-norm.ts +64 -0
  191. package/src/kernels/glm52/layout.ts +37 -0
  192. package/src/kernels/glm52/streamed-experts.ts +360 -0
  193. package/src/kernels/logits.ts +38 -0
  194. package/src/kernels/quantization/affine-row.ts +154 -0
  195. package/src/kernels/quantization/affine-rows.ts +36 -0
  196. package/src/kernels/quantization/affine-split-mma.ts +144 -0
  197. package/src/kernels/quantization/affine3-mma.ts +117 -0
  198. package/src/kernels/quantization/affine3-rows.ts +103 -0
  199. package/src/kernels/quantization/affine4-mma.ts +113 -0
  200. package/src/kernels/quantization/affine4-rows.ts +102 -0
  201. package/src/kernels/residual-rms-norm.ts +65 -0
  202. package/src/kernels/sampling/index.ts +2 -0
  203. package/src/kernels/sampling/normalized-argmax.ts +116 -0
  204. package/src/kernels/sampling/token-bitmask.ts +52 -0
  205. package/src/kernels/softcap.ts +35 -0
  206. package/src/kernels/trellis/codebook.ts +123 -0
  207. package/src/kernels/trellis/down-codebook.ts +184 -0
  208. package/src/kernels/trellis/down-factored.ts +180 -0
  209. package/src/kernels/trellis/down-k3-interleaved-factored.ts +146 -0
  210. package/src/kernels/trellis/down-mma-wide.ts +195 -0
  211. package/src/kernels/trellis/down-mma.ts +145 -0
  212. package/src/kernels/trellis/expand.ts +42 -0
  213. package/src/kernels/trellis/gate-up-codebook-mma.ts +141 -0
  214. package/src/kernels/trellis/gate-up-codebook.ts +137 -0
  215. package/src/kernels/trellis/gate-up-factored.ts +197 -0
  216. package/src/kernels/trellis/gate-up-mma-wide.ts +147 -0
  217. package/src/kernels/trellis/gate-up-mma.ts +148 -0
  218. package/src/kernels/trellis/gate-up.ts +248 -0
  219. package/src/kernels/trellis/geometry.ts +22 -0
  220. package/src/kernels/trellis/index.ts +103 -0
  221. package/src/kernels/trellis/launch.ts +12 -0
  222. package/src/kernels/trellis/mixed-gate-up-factored.ts +119 -0
  223. package/src/kernels/trellis/mixed-gate-up.ts +146 -0
  224. package/src/kernels/trellis/qualified.ts +34 -0
  225. package/src/kernels/trellis/reduce.ts +150 -0
  226. package/src/kernels/trellis/scatter.ts +407 -0
  227. package/src/kernels/trellis/splitk-prefill.ts +152 -0
  228. package/src/kernels/trellis/tiled-prefill.ts +131 -0
  229. package/src/kernels/trellis/vector-expand.ts +58 -0
  230. package/src/kernels/trellis/wide-prefill.ts +89 -0
  231. package/src/kernels/turboquant/decode.ts +82 -0
  232. package/src/kernels/turboquant/index.ts +5 -0
  233. package/src/kernels/turboquant/inverse.ts +122 -0
  234. package/src/kernels/turboquant/ops.ts +860 -0
  235. package/src/kernels/turboquant/signs.ts +130 -0
  236. package/src/kernels/turboquant/tables.ts +124 -0
  237. package/src/layers/affine-verify-linear.ts +75 -0
  238. package/src/layers/ane-prefill-split.ts +283 -0
  239. package/src/layers/compiled-normalization.ts +41 -0
  240. package/src/layers/delta-heads.ts +23 -0
  241. package/src/layers/dense-embedding.ts +29 -0
  242. package/src/layers/dense-linear.ts +32 -0
  243. package/src/layers/dequantized-linear.ts +72 -0
  244. package/src/layers/folded-attention.ts +42 -0
  245. package/src/layers/geglu.ts +130 -0
  246. package/src/layers/helpers.ts +7 -0
  247. package/src/layers/index.ts +36 -0
  248. package/src/layers/kv4-decode-attention.ts +15 -0
  249. package/src/layers/layer-norm.ts +24 -0
  250. package/src/layers/loading.ts +92 -0
  251. package/src/layers/lora.ts +150 -0
  252. package/src/layers/normalization.ts +9 -0
  253. package/src/layers/prefix-layout.ts +33 -0
  254. package/src/layers/quantized-append-attention.ts +32 -0
  255. package/src/layers/quantized-attention.ts +353 -0
  256. package/src/layers/quantized-embedding.ts +39 -0
  257. package/src/layers/quantized-linear.ts +137 -0
  258. package/src/layers/quantized-switch-linear.ts +37 -0
  259. package/src/layers/qwen-append.ts +12 -0
  260. package/src/layers/qwen-mrope.ts +238 -0
  261. package/src/layers/rope-2d.ts +110 -0
  262. package/src/layers/rope.ts +389 -0
  263. package/src/layers/swiglu.ts +23 -0
  264. package/src/layers/trellis-codebook-down.ts +52 -0
  265. package/src/layers/trellis-codebook-gate-up.ts +42 -0
  266. package/src/layers/trellis-down.ts +117 -0
  267. package/src/layers/trellis-gate-up.ts +71 -0
  268. package/src/layers/trellis-linear.ts +211 -0
  269. package/src/layers/trellis-prompt.ts +102 -0
  270. package/src/layers/trellis-reduce-gate-up.ts +73 -0
  271. package/src/layers/trellis-scatter-down.ts +98 -0
  272. package/src/models/ane-install.ts +74 -0
  273. package/src/models/audio/conformer.ts +656 -0
  274. package/src/models/cache-composition.ts +44 -0
  275. package/src/models/capabilities.ts +89 -0
  276. package/src/models/chat-template.ts +10 -0
  277. package/src/models/composition.ts +107 -0
  278. package/src/models/diffusion-gemma/experts.ts +59 -0
  279. package/src/models/diffusion-gemma/family.ts +21 -0
  280. package/src/models/diffusion-gemma/model.ts +900 -0
  281. package/src/models/diffusion-gemma/vision.ts +549 -0
  282. package/src/models/drafter-loaders.ts +49 -0
  283. package/src/models/drafters.ts +39 -0
  284. package/src/models/factory.ts +351 -0
  285. package/src/models/families.ts +70 -0
  286. package/src/models/family.ts +125 -0
  287. package/src/models/gemma4/assistant-target.ts +46 -0
  288. package/src/models/gemma4/assistant.ts +327 -0
  289. package/src/models/gemma4/blocks.ts +755 -0
  290. package/src/models/gemma4/compiled-decode.ts +452 -0
  291. package/src/models/gemma4/family.ts +39 -0
  292. package/src/models/gemma4/gemma4-12b.ts +677 -0
  293. package/src/models/gemma4/gemma4-26b.ts +544 -0
  294. package/src/models/gemma4/gemma4-e4b.ts +599 -0
  295. package/src/models/gemma4/graphs.ts +147 -0
  296. package/src/models/gemma4/media.ts +88 -0
  297. package/src/models/gemma4/model.ts +860 -0
  298. package/src/models/gemma4/serving-blocks.ts +72 -0
  299. package/src/models/gemma4/trainable.ts +186 -0
  300. package/src/models/gemma4/written-blocks.ts +232 -0
  301. package/src/models/gemma4/written-forward.ts +159 -0
  302. package/src/models/gemma4/written-input.ts +29 -0
  303. package/src/models/glm52/atlas.ts +507 -0
  304. package/src/models/glm52/coupling.ts +388 -0
  305. package/src/models/glm52/dsa.ts +601 -0
  306. package/src/models/glm52/family.ts +20 -0
  307. package/src/models/glm52/mla.ts +812 -0
  308. package/src/models/glm52/model.ts +1127 -0
  309. package/src/models/glm52/moe.ts +223 -0
  310. package/src/models/glm52/mtp.ts +60 -0
  311. package/src/models/glm52/pilot.ts +537 -0
  312. package/src/models/glm52/reference.ts +445 -0
  313. package/src/models/glm52/residency.ts +557 -0
  314. package/src/models/glm52/streamed-experts.ts +447 -0
  315. package/src/models/graph.ts +59 -0
  316. package/src/models/implementation.ts +230 -0
  317. package/src/models/index.ts +15 -0
  318. package/src/models/media-input.ts +15 -0
  319. package/src/models/memory-plan.ts +39 -0
  320. package/src/models/minicpm5/family.ts +29 -0
  321. package/src/models/minicpm5/model.ts +394 -0
  322. package/src/models/minicpm5/trainable.ts +90 -0
  323. package/src/models/profile.ts +337 -0
  324. package/src/models/qwen/blocks.ts +658 -0
  325. package/src/models/qwen/checkpoint.ts +190 -0
  326. package/src/models/qwen/families.ts +57 -0
  327. package/src/models/qwen/media-input.ts +161 -0
  328. package/src/models/qwen/mtp.ts +101 -0
  329. package/src/models/qwen/qwen3-moe.ts +355 -0
  330. package/src/models/qwen/qwen3.ts +323 -0
  331. package/src/models/qwen/qwen38-27b-trellis-m1max.ts +1532 -0
  332. package/src/models/qwen/qwen38-27b-trellis-m4pro.ts +1092 -0
  333. package/src/models/qwen/qwen3_5.ts +433 -0
  334. package/src/models/qwen/written-layers.ts +285 -0
  335. package/src/models/row-table.ts +1 -0
  336. package/src/models/runtime.ts +73 -0
  337. package/src/models/speculative/deepspec-artifact.ts +10 -0
  338. package/src/models/speculative/deepspec.ts +744 -0
  339. package/src/models/speculative/dflash.ts +553 -0
  340. package/src/models/speculative/dflash2-artifact.ts +9 -0
  341. package/src/models/speculative/dflash2-operations.ts +1 -0
  342. package/src/models/speculative/dflash2.ts +576 -0
  343. package/src/models/speculative/loader.ts +29 -0
  344. package/src/models/support.ts +79 -0
  345. package/src/models/universal/archs.ts +365 -0
  346. package/src/models/universal/blocks.ts +479 -0
  347. package/src/models/universal/dense.ts +255 -0
  348. package/src/models/universal/family.ts +17 -0
  349. package/src/models/vision/qwen3vl.ts +566 -0
  350. package/src/models/vision/siglip.ts +520 -0
  351. package/src/models/vision/unified.ts +176 -0
  352. package/src/models/whisper/family.ts +19 -0
  353. package/src/models/whisper/fast.ts +734 -0
  354. package/src/models/whisper/model.ts +430 -0
  355. package/src/runtime/compiled-trace.ts +16 -0
  356. package/src/runtime/config.ts +77 -0
  357. package/src/runtime/expert-trace.ts +71 -0
  358. package/src/runtime/graph-forward.ts +65 -0
  359. package/src/runtime/integers.ts +6 -0
  360. package/src/runtime/native.ts +16 -0
  361. package/src/runtime/resources.ts +53 -0
  362. package/src/runtime/row-operations.ts +42 -0
  363. package/src/runtime/trace.ts +161 -0
  364. package/src/sampling/curve.ts +122 -0
  365. package/src/sampling/draft.ts +55 -0
  366. package/src/sampling/extras.ts +45 -0
  367. package/src/sampling/filters.ts +134 -0
  368. package/src/sampling/grammar.ts +630 -0
  369. package/src/sampling/hlg.ts +290 -0
  370. package/src/sampling/index.ts +9 -0
  371. package/src/sampling/processors.ts +142 -0
  372. package/src/sampling/sampler.ts +120 -0
  373. package/src/sampling/step.ts +343 -0
  374. package/src/sampling/types.ts +161 -0
  375. package/src/sampling/window-filters.ts +46 -0
  376. package/src/scoring/decisions.ts +96 -0
  377. package/src/scoring/full-sequence.ts +103 -0
  378. package/src/scoring/index.ts +9 -0
  379. package/src/scoring/kl.ts +29 -0
  380. package/src/scoring/perplexity.ts +165 -0
  381. package/src/scoring/response-logps.ts +65 -0
  382. package/src/state/affine-attention.ts +180 -0
  383. package/src/state/attachment.ts +68 -0
  384. package/src/state/attention-read.ts +215 -0
  385. package/src/state/batched-kv.ts +294 -0
  386. package/src/state/batched-mask.ts +445 -0
  387. package/src/state/batched-quant.ts +389 -0
  388. package/src/state/batched-quantized-kv.ts +209 -0
  389. package/src/state/batched-rotating-quant.ts +457 -0
  390. package/src/state/batched-rotating-state.ts +149 -0
  391. package/src/state/batched-rotating.ts +430 -0
  392. package/src/state/batched-row-storage.ts +179 -0
  393. package/src/state/batched-ssm.ts +54 -0
  394. package/src/state/batched-turboquant-kv.ts +144 -0
  395. package/src/state/bf16-first-kv.ts +663 -0
  396. package/src/state/cache-service.ts +592 -0
  397. package/src/state/capabilities.ts +65 -0
  398. package/src/state/checkpoint.ts +46 -0
  399. package/src/state/continuation-persistence.ts +126 -0
  400. package/src/state/decode-step-inputs.ts +31 -0
  401. package/src/state/decoded-kv-donor.ts +12 -0
  402. package/src/state/delayed-quantized-kv.ts +141 -0
  403. package/src/state/delayed-rotating-quantized-kv.ts +178 -0
  404. package/src/state/delayed-turboquant-kv.ts +123 -0
  405. package/src/state/durability.ts +214 -0
  406. package/src/state/encoder-cache.ts +26 -0
  407. package/src/state/full-kv-row-append.ts +33 -0
  408. package/src/state/full-kv-row-donor.ts +54 -0
  409. package/src/state/full-prefill-padding.ts +56 -0
  410. package/src/state/full-prefill-row.ts +112 -0
  411. package/src/state/full-transitioning-kv-rows.ts +70 -0
  412. package/src/state/glm52-cache.ts +783 -0
  413. package/src/state/grouped-heads-kv4-cache.ts +26 -0
  414. package/src/state/index.ts +38 -0
  415. package/src/state/kv-attention-view.ts +77 -0
  416. package/src/state/kv-budget.ts +14 -0
  417. package/src/state/kv-scheme.ts +315 -0
  418. package/src/state/kv-storage.ts +57 -0
  419. package/src/state/kv-tensor-rows.ts +212 -0
  420. package/src/state/kv.ts +324 -0
  421. package/src/state/kv4-head256-cache.ts +46 -0
  422. package/src/state/latent-legos.ts +895 -0
  423. package/src/state/layout.ts +277 -0
  424. package/src/state/leases.ts +26 -0
  425. package/src/state/paged/cache.ts +689 -0
  426. package/src/state/paged/codec.ts +36 -0
  427. package/src/state/paged/rows.ts +223 -0
  428. package/src/state/persistence-types.ts +173 -0
  429. package/src/state/persistence-worker.ts +83 -0
  430. package/src/state/persistence.ts +1036 -0
  431. package/src/state/persistence.worker.d.ts +3 -0
  432. package/src/state/persistence.worker.js +264 -0
  433. package/src/state/prefix-cache.ts +685 -0
  434. package/src/state/quantized-kv.ts +369 -0
  435. package/src/state/quantized-tensor.ts +26 -0
  436. package/src/state/request-policy.ts +48 -0
  437. package/src/state/retention-policy.ts +37 -0
  438. package/src/state/rollback.ts +131 -0
  439. package/src/state/rotating-kv-donor.ts +35 -0
  440. package/src/state/rotating-kv-layout.ts +341 -0
  441. package/src/state/rotating-kv.ts +529 -0
  442. package/src/state/rotating-quantized-kv.ts +616 -0
  443. package/src/state/rotating-row-transaction.ts +35 -0
  444. package/src/state/rows.ts +104 -0
  445. package/src/state/softcap-attention.ts +118 -0
  446. package/src/state/speculative/glm52-mtp-state.ts +38 -0
  447. package/src/state/speculative/qwen-mtp-state.ts +34 -0
  448. package/src/state/speculative-rotating-kv.ts +113 -0
  449. package/src/state/ssd-cache.ts +434 -0
  450. package/src/state/ssm-prefill-padding.ts +39 -0
  451. package/src/state/ssm.ts +422 -0
  452. package/src/state/target-layout.ts +7 -0
  453. package/src/state/tiered-prefix-cache.ts +45 -0
  454. package/src/state/training-cache.ts +300 -0
  455. package/src/state/transitioning-kv-rows.ts +267 -0
  456. package/src/state/turboquant-attention.ts +97 -0
  457. package/src/state/turboquant-codec.ts +90 -0
  458. package/src/state/turboquant-kv.ts +442 -0
  459. package/src/state/views.ts +26 -0
  460. package/src/transcription/index.ts +3 -0
  461. package/src/transcription/silero-vad.ts +379 -0
  462. package/src/transcription/whisper/decode.ts +773 -0
  463. package/src/transcription/whisper/format.ts +74 -0
  464. package/src/transcription/whisper/index.ts +4 -0
  465. package/src/transcription/whisper/timing.ts +274 -0
  466. package/src/transcription/whisper/transcribe.ts +394 -0
  467. package/src/transcription/whisper/types.ts +22 -0
@@ -0,0 +1,152 @@
1
+ // Exact logical tensor contract for the direct Colibri GLM-5.2 container.
2
+
3
+ import type { Glm52Config } from "../glm52-config";
4
+ import {
5
+ ColibriGlm52Container,
6
+ type ColibriTensorInfo,
7
+ } from "./container";
8
+
9
+ export interface Glm52LayoutValidation {
10
+ readonly quantizedTensors: number;
11
+ readonly floatTensors: number;
12
+ readonly routedExperts: number;
13
+ readonly hasDsa: boolean;
14
+ readonly hasMtp: boolean;
15
+ }
16
+
17
+ function sameShape(actual: readonly number[], expected: readonly number[]): boolean {
18
+ return actual.length === expected.length &&
19
+ actual.every((dimension, index) => dimension === expected[index]);
20
+ }
21
+
22
+ function validateFloatTensor(
23
+ container: ColibriGlm52Container,
24
+ name: string,
25
+ shape: number[],
26
+ ): ColibriTensorInfo {
27
+ const tensor = container.info(name);
28
+ if (tensor.dtype !== "F32")
29
+ throw new Error(`${name}: resident tensor must be F32 (got ${tensor.dtype})`);
30
+ if (!sameShape(tensor.shape, shape)) {
31
+ throw new Error(
32
+ `${name}: shape ${JSON.stringify(tensor.shape)} != ${JSON.stringify(shape)}`,
33
+ );
34
+ }
35
+ return tensor;
36
+ }
37
+
38
+ export function validateGlm52ContainerLayout(
39
+ container: ColibriGlm52Container,
40
+ config: Glm52Config,
41
+ ): Glm52LayoutValidation {
42
+ let quantizedTensors = 0;
43
+ let floatTensors = 0;
44
+ let routedExperts = 0;
45
+ const q = (name: string, output: number, input: number): void => {
46
+ container.quantized(name, output, input);
47
+ quantizedTensors++;
48
+ };
49
+ const f = (name: string, shape: number[]): void => {
50
+ validateFloatTensor(container, name, shape);
51
+ floatTensors++;
52
+ };
53
+ const D = config.hiddenSize;
54
+ const H = config.numAttentionHeads;
55
+ const indexCapabilities = container.capabilities(config);
56
+
57
+ q("model.embed_tokens.weight", config.vocabSize, D);
58
+ q("lm_head.weight", config.vocabSize, D);
59
+ f("model.norm.weight", [D]);
60
+
61
+ const validateAttention = (layer: number): void => {
62
+ const p = `model.layers.${layer}`;
63
+ f(`${p}.input_layernorm.weight`, [D]);
64
+ f(`${p}.post_attention_layernorm.weight`, [D]);
65
+ q(`${p}.self_attn.q_a_proj.weight`, config.qLoraRank, D);
66
+ f(`${p}.self_attn.q_a_layernorm.weight`, [config.qLoraRank]);
67
+ q(
68
+ `${p}.self_attn.q_b_proj.weight`,
69
+ H * config.qkHeadDim,
70
+ config.qLoraRank,
71
+ );
72
+ q(
73
+ `${p}.self_attn.kv_a_proj_with_mqa.weight`,
74
+ config.kvLoraRank + config.qkRopeHeadDim,
75
+ D,
76
+ );
77
+ f(`${p}.self_attn.kv_a_layernorm.weight`, [config.kvLoraRank]);
78
+ q(
79
+ `${p}.self_attn.kv_b_proj.weight`,
80
+ H * (config.qkNopeHeadDim + config.vHeadDim),
81
+ config.kvLoraRank,
82
+ );
83
+ q(`${p}.self_attn.o_proj.weight`, D, H * config.vHeadDim);
84
+ };
85
+
86
+ const validateSparseMlp = (layer: number): void => {
87
+ const p = `model.layers.${layer}.mlp`;
88
+ f(`${p}.gate.weight`, [config.numRoutedExperts, D]);
89
+ f(`${p}.gate.e_score_correction_bias`, [config.numRoutedExperts]);
90
+ const sharedIntermediate =
91
+ config.moeIntermediateSize * config.numSharedExperts;
92
+ if (sharedIntermediate > 0) {
93
+ q(`${p}.shared_experts.gate_proj.weight`, sharedIntermediate, D);
94
+ q(`${p}.shared_experts.up_proj.weight`, sharedIntermediate, D);
95
+ q(`${p}.shared_experts.down_proj.weight`, D, sharedIntermediate);
96
+ }
97
+ for (let expert = 0; expert < config.numRoutedExperts; expert++) {
98
+ const e = `${p}.experts.${expert}`;
99
+ q(`${e}.gate_proj.weight`, config.moeIntermediateSize, D);
100
+ q(`${e}.up_proj.weight`, config.moeIntermediateSize, D);
101
+ q(`${e}.down_proj.weight`, D, config.moeIntermediateSize);
102
+ routedExperts++;
103
+ }
104
+ };
105
+
106
+ for (let layer = 0; layer < config.numHiddenLayers; layer++) {
107
+ validateAttention(layer);
108
+ const p = `model.layers.${layer}.mlp`;
109
+ if (layer < config.firstKDenseReplace) {
110
+ q(`${p}.gate_proj.weight`, config.intermediateSize, D);
111
+ q(`${p}.up_proj.weight`, config.intermediateSize, D);
112
+ q(`${p}.down_proj.weight`, D, config.intermediateSize);
113
+ } else {
114
+ validateSparseMlp(layer);
115
+ }
116
+ }
117
+
118
+ if (indexCapabilities.hasDsa) {
119
+ for (let layer = 0; layer < config.numHiddenLayers; layer++) {
120
+ if (config.indexerTypes[layer] !== "full") continue;
121
+ const p = `model.layers.${layer}.self_attn.indexer`;
122
+ q(
123
+ `${p}.wq_b.weight`,
124
+ config.indexNumHeads * config.indexHeadDim,
125
+ config.qLoraRank,
126
+ );
127
+ q(`${p}.wk.weight`, config.indexHeadDim, D);
128
+ q(`${p}.weights_proj.weight`, config.indexNumHeads, D);
129
+ f(`${p}.k_norm.weight`, [config.indexHeadDim]);
130
+ f(`${p}.k_norm.bias`, [config.indexHeadDim]);
131
+ }
132
+ }
133
+
134
+ if (indexCapabilities.hasMtp) {
135
+ const layer = config.numHiddenLayers;
136
+ const p = `model.layers.${layer}`;
137
+ validateAttention(layer);
138
+ validateSparseMlp(layer);
139
+ q(`${p}.eh_proj.weight`, D, 2 * D);
140
+ f(`${p}.enorm.weight`, [D]);
141
+ f(`${p}.hnorm.weight`, [D]);
142
+ f(`${p}.shared_head.norm.weight`, [D]);
143
+ }
144
+
145
+ return {
146
+ quantizedTensors,
147
+ floatTensors,
148
+ routedExperts,
149
+ hasDsa: indexCapabilities.hasDsa,
150
+ hasMtp: indexCapabilities.hasMtp,
151
+ };
152
+ }
@@ -0,0 +1,500 @@
1
+ import { buildGlm52ExpertSlotLayout } from "./expert-layout";
2
+ import { positiveInteger } from "../../runtime/integers";
3
+ import {
4
+ loadGlm52Config,
5
+ type Glm52Config,
6
+ } from "../glm52-config";
7
+ import { ColibriGlm52Container } from "./container";
8
+ import type { MemoryPlan } from "../../contracts/portable/memory-plan";
9
+
10
+ export const GLM52_G5_GIB = 1024 ** 3;
11
+ export const GLM52_G5_DEFAULT_MACHINE_BYTES = 32 * GLM52_G5_GIB;
12
+ export const GLM52_G5_DEFAULT_PROCESS_LIMIT_BYTES = 25 * GLM52_G5_GIB;
13
+ export const GLM52_G5_DEFAULT_CONTEXT_TOKENS = 4096;
14
+ export const GLM52_G5_DEFAULT_MAX_GENERATION_TOKENS = 128;
15
+ export const GLM52_G5_DEFAULT_ALLOCATOR_RESERVE_BYTES = 4 * GLM52_G5_GIB;
16
+ export const GLM52_G5_DEFAULT_BUN_NATIVE_RESERVE_BYTES = 512 * 1024 ** 2;
17
+ export const GLM52_G5_DEFAULT_SAFETY_MARGIN_BYTES = 512 * 1024 ** 2;
18
+ /** Curated G5 quality-preserving warm result on the M1 Max 32 GB, native MTP
19
+ * on, 128 generated tokens. This is operator guidance, not a prediction. */
20
+ export const GLM52_G5_MEASURED_WARM_DECODE_TPS = 0.149;
21
+ /** Same-machine direct Colibri/Metal warm median with native MTP on. */
22
+ export const GLM52_G5_DIRECT_ORACLE_WARM_DECODE_TPS = 0.27;
23
+ export const GLM52_G5_ASPIRATIONAL_DECODE_TPS = 2;
24
+ export const GLM52_G5_MEASURED_AT = Date.UTC(2026, 7, 15);
25
+
26
+ const ROUTED_EXPERT = /\.mlp\.experts\.\d+\./;
27
+
28
+ export interface Glm52MemoryGeometry {
29
+ readonly residentWeightBytes: number;
30
+ readonly mainExpertSlotBytes: number;
31
+ readonly mtpExpertSlotBytes?: number;
32
+ readonly hiddenSize: number;
33
+ readonly vocabSize: number;
34
+ readonly numHiddenLayers: number;
35
+ readonly sparseLayers: number;
36
+ readonly numAttentionHeads: number;
37
+ readonly kvLoraRank: number;
38
+ readonly qkRopeHeadDim: number;
39
+ readonly qkHeadDim: number;
40
+ readonly vHeadDim: number;
41
+ readonly numRoutedExperts: number;
42
+ readonly numExpertsPerToken: number;
43
+ readonly dsaLayers?: number;
44
+ readonly dsaHeadDim?: number;
45
+ readonly mtpLayers?: number;
46
+ }
47
+
48
+ export interface Glm52MemoryPlanOptions {
49
+ readonly machineBytes?: number;
50
+ readonly processLimitBytes?: number;
51
+ readonly contextTokens?: number;
52
+ readonly maxGenerationTokens?: number;
53
+ readonly batchSize?: number;
54
+ readonly enableMtp?: boolean;
55
+ readonly mtpDraftTokens?: number;
56
+ readonly workingSlots?: number;
57
+ readonly pinnedExpertSlots?: number;
58
+ readonly slotsPerSparseLayer?: number;
59
+ readonly mtpSlotsPerLayer?: number;
60
+ readonly allocatorReserveBytes?: number;
61
+ readonly bunNativeReserveBytes?: number;
62
+ readonly safetyMarginBytes?: number;
63
+ }
64
+
65
+ export interface Glm52MemoryLineItems {
66
+ readonly residentWeightsBytes: number;
67
+ readonly mainExpertSlabBytes: number;
68
+ readonly mtpExpertSlabBytes: number;
69
+ readonly targetKvBytes: number;
70
+ readonly mtpKvBytes: number;
71
+ readonly reconstructedKvTransientBytes: number;
72
+ readonly verifyBatchTransientBytes: number;
73
+ readonly allocatorReserveBytes: number;
74
+ readonly bunNativeReserveBytes: number;
75
+ readonly safetyMarginBytes: number;
76
+ }
77
+
78
+ export interface Glm52MemoryPlan extends MemoryPlan {
79
+ readonly schemaVersion: 1;
80
+ readonly strategy: "glm52-colibri";
81
+ readonly preset: "g5-32gb-quality";
82
+ readonly machineBytes: number;
83
+ readonly processLimitBytes: number;
84
+ /** Deliberately unavailable to this process. */
85
+ readonly osReserveBytes: number;
86
+ readonly contextTokens: number;
87
+ readonly maxGenerationTokens: number;
88
+ readonly batchSize: number;
89
+ readonly enableMtp: boolean;
90
+ readonly mtpDraftTokens: number;
91
+ readonly verifyRows: number;
92
+ readonly mainWorkingSlots: number;
93
+ readonly mainRequiredUnionSlots: number;
94
+ readonly mainResidentSlots: number;
95
+ readonly mainTotalSlots: number;
96
+ readonly mtpWorkingSlots: number;
97
+ readonly mtpResidentSlots: number;
98
+ readonly mtpTotalSlots: number;
99
+ readonly lineItems: Glm52MemoryLineItems;
100
+ readonly runtimeReserveBytes: number;
101
+ readonly plannedProcessBytes: number;
102
+ readonly processHeadroomBytes: number;
103
+ readonly plannedMachineBytes: number;
104
+ readonly machineHeadroomBytes: number;
105
+ }
106
+
107
+ function nonNegativeSafeInteger(value: number, label: string): void {
108
+ if (!Number.isSafeInteger(value) || value < 0)
109
+ throw new RangeError(`${label} must be a non-negative safe integer`);
110
+ }
111
+
112
+ function sum(label: string, values: readonly number[]): number {
113
+ let result = 0;
114
+ for (const value of values) {
115
+ nonNegativeSafeInteger(value, label);
116
+ result += value;
117
+ if (!Number.isSafeInteger(result))
118
+ throw new RangeError(`${label} exceeds the safe integer range`);
119
+ }
120
+ return result;
121
+ }
122
+
123
+ function product(label: string, values: readonly number[]): number {
124
+ let result = 1;
125
+ for (const value of values) {
126
+ nonNegativeSafeInteger(value, label);
127
+ result *= value;
128
+ if (!Number.isSafeInteger(result))
129
+ throw new RangeError(`${label} exceeds the safe integer range`);
130
+ }
131
+ return result;
132
+ }
133
+
134
+ function bytesWithoutRoutedExperts(
135
+ container: ColibriGlm52Container,
136
+ includeMtp: boolean,
137
+ ): number {
138
+ return sum(
139
+ "GLM resident weights",
140
+ [...container.tensors.values()]
141
+ .filter((tensor) =>
142
+ (includeMtp || tensor.family !== "mtp") &&
143
+ !ROUTED_EXPERT.test(tensor.name))
144
+ .map((tensor) => tensor.byteLength),
145
+ );
146
+ }
147
+
148
+ /**
149
+ * Full G5 process equation. The 32 GB preset reserves one persistent slot per
150
+ * sparse layer, keeps the global 64-slot miss bank, and treats the OS reserve
151
+ * as memory that the process is never allowed to claim.
152
+ */
153
+ export function planGlm52Memory(
154
+ geometry: Glm52MemoryGeometry,
155
+ options: Glm52MemoryPlanOptions = {},
156
+ ): Glm52MemoryPlan {
157
+ const machineBytes =
158
+ options.machineBytes ?? GLM52_G5_DEFAULT_MACHINE_BYTES;
159
+ const processLimitBytes =
160
+ options.processLimitBytes ?? GLM52_G5_DEFAULT_PROCESS_LIMIT_BYTES;
161
+ const contextTokens =
162
+ options.contextTokens ?? GLM52_G5_DEFAULT_CONTEXT_TOKENS;
163
+ const maxGenerationTokens =
164
+ options.maxGenerationTokens ?? GLM52_G5_DEFAULT_MAX_GENERATION_TOKENS;
165
+ const batchSize = options.batchSize ?? 1;
166
+ const enableMtp = options.enableMtp !== false;
167
+ const mtpDraftTokens = options.mtpDraftTokens ?? 3;
168
+ const mainWorkingSlots = options.workingSlots ?? 64;
169
+ const pinnedExpertSlots = options.pinnedExpertSlots ?? 0;
170
+ const slotsPerSparseLayer = options.slotsPerSparseLayer ?? 1;
171
+ const mtpSlotsPerLayer = options.mtpSlotsPerLayer ?? 1;
172
+ const allocatorReserveBytes =
173
+ options.allocatorReserveBytes ??
174
+ GLM52_G5_DEFAULT_ALLOCATOR_RESERVE_BYTES;
175
+ const bunNativeReserveBytes =
176
+ options.bunNativeReserveBytes ??
177
+ GLM52_G5_DEFAULT_BUN_NATIVE_RESERVE_BYTES;
178
+ const safetyMarginBytes =
179
+ options.safetyMarginBytes ??
180
+ GLM52_G5_DEFAULT_SAFETY_MARGIN_BYTES;
181
+
182
+ for (const [label, value] of [
183
+ ["GLM machineBytes", machineBytes],
184
+ ["GLM processLimitBytes", processLimitBytes],
185
+ ["GLM contextTokens", contextTokens],
186
+ ["GLM maxGenerationTokens", maxGenerationTokens],
187
+ ["GLM batchSize", batchSize],
188
+ ["GLM mtpDraftTokens", mtpDraftTokens],
189
+ ["GLM workingSlots", mainWorkingSlots],
190
+ ["GLM slotsPerSparseLayer", slotsPerSparseLayer],
191
+ ["GLM mtpSlotsPerLayer", mtpSlotsPerLayer],
192
+ ["GLM residentWeightBytes", geometry.residentWeightBytes],
193
+ ["GLM mainExpertSlotBytes", geometry.mainExpertSlotBytes],
194
+ ["GLM hiddenSize", geometry.hiddenSize],
195
+ ["GLM vocabSize", geometry.vocabSize],
196
+ ["GLM numHiddenLayers", geometry.numHiddenLayers],
197
+ ["GLM sparseLayers", geometry.sparseLayers],
198
+ ["GLM numAttentionHeads", geometry.numAttentionHeads],
199
+ ["GLM kvLoraRank", geometry.kvLoraRank],
200
+ ["GLM qkRopeHeadDim", geometry.qkRopeHeadDim],
201
+ ["GLM qkHeadDim", geometry.qkHeadDim],
202
+ ["GLM vHeadDim", geometry.vHeadDim],
203
+ ["GLM numRoutedExperts", geometry.numRoutedExperts],
204
+ ["GLM numExpertsPerToken", geometry.numExpertsPerToken],
205
+ ] as const) {
206
+ positiveInteger(value, label);
207
+ }
208
+ for (const [label, value] of [
209
+ ["GLM pinnedExpertSlots", pinnedExpertSlots],
210
+ ["GLM allocatorReserveBytes", allocatorReserveBytes],
211
+ ["GLM bunNativeReserveBytes", bunNativeReserveBytes],
212
+ ["GLM safetyMarginBytes", safetyMarginBytes],
213
+ ["GLM dsaLayers", geometry.dsaLayers ?? 0],
214
+ ["GLM dsaHeadDim", geometry.dsaHeadDim ?? 0],
215
+ ["GLM mtpLayers", geometry.mtpLayers ?? 0],
216
+ ] as const) {
217
+ nonNegativeSafeInteger(value, label);
218
+ }
219
+ if (processLimitBytes > machineBytes) {
220
+ throw new Error(
221
+ `GLM memory contract cannot start: process limit ${processLimitBytes} ` +
222
+ `exceeds machine memory ${machineBytes}`,
223
+ );
224
+ }
225
+ if (maxGenerationTokens > contextTokens) {
226
+ throw new Error(
227
+ `GLM memory contract cannot start: ${maxGenerationTokens} generated ` +
228
+ `tokens exceed the ${contextTokens}-token context preset`,
229
+ );
230
+ }
231
+
232
+ const mtpLayers = enableMtp ? (geometry.mtpLayers ?? 0) : 0;
233
+ if (enableMtp && mtpLayers === 0)
234
+ throw new Error("GLM memory contract cannot enable MTP without an MTP layer");
235
+ if (enableMtp && geometry.mtpExpertSlotBytes === undefined) {
236
+ throw new Error(
237
+ "GLM memory contract cannot enable MTP without an MTP expert slot",
238
+ );
239
+ }
240
+ const dsaLayers = geometry.dsaLayers ?? 0;
241
+ if (dsaLayers > geometry.numHiddenLayers)
242
+ throw new Error("GLM DSA layer count exceeds the target layer count");
243
+ if (dsaLayers > 0 && (geometry.dsaHeadDim ?? 0) === 0)
244
+ throw new Error("GLM DSA layers require a positive DSA head width");
245
+
246
+ const verifyRows = enableMtp ? mtpDraftTokens + 1 : 1;
247
+ const mainRequiredUnionSlots = Math.min(
248
+ geometry.numRoutedExperts,
249
+ geometry.numExpertsPerToken * verifyRows,
250
+ );
251
+ if (mainRequiredUnionSlots > mainWorkingSlots) {
252
+ throw new Error(
253
+ `GLM memory contract cannot start: ${mainRequiredUnionSlots} target ` +
254
+ `verify-union slots exceed the ${mainWorkingSlots}-slot working bank`,
255
+ );
256
+ }
257
+ const mainResidentSlots = geometry.sparseLayers * slotsPerSparseLayer;
258
+ const mainTotalSlots =
259
+ mainWorkingSlots + pinnedExpertSlots + mainResidentSlots;
260
+ const mtpWorkingSlots = enableMtp
261
+ ? Math.min(
262
+ geometry.numRoutedExperts,
263
+ geometry.numExpertsPerToken * mtpDraftTokens,
264
+ )
265
+ : 0;
266
+ const mtpResidentSlots = enableMtp ? mtpLayers * mtpSlotsPerLayer : 0;
267
+ const mtpTotalSlots = mtpWorkingSlots + mtpResidentSlots;
268
+
269
+ const mainExpertSlabBytes = product(
270
+ "GLM main expert slab",
271
+ [mainTotalSlots, geometry.mainExpertSlotBytes],
272
+ );
273
+ const mtpExpertSlabBytes = enableMtp
274
+ ? product(
275
+ "GLM MTP expert slab",
276
+ [mtpTotalSlots, geometry.mtpExpertSlotBytes!],
277
+ )
278
+ : 0;
279
+ const baseKvWidth = geometry.kvLoraRank + geometry.qkRopeHeadDim;
280
+ const targetKvBytes = product(
281
+ "GLM target KV",
282
+ [
283
+ batchSize,
284
+ contextTokens,
285
+ (
286
+ geometry.numHiddenLayers * baseKvWidth +
287
+ dsaLayers * (geometry.dsaHeadDim ?? 0)
288
+ ),
289
+ 4,
290
+ ],
291
+ );
292
+ const mtpKvBytes = enableMtp
293
+ ? product(
294
+ "GLM MTP KV",
295
+ [batchSize, contextTokens, mtpLayers, baseKvWidth, 4],
296
+ )
297
+ : 0;
298
+ // MLA reconstructs full per-head K/V for the largest target verification
299
+ // graph. Count the cache horizon plus that graph's appended rows.
300
+ const reconstructedKvTransientBytes = product(
301
+ "GLM reconstructed KV transient",
302
+ [
303
+ batchSize,
304
+ geometry.numAttentionHeads,
305
+ contextTokens + verifyRows,
306
+ geometry.qkHeadDim + geometry.vHeadDim,
307
+ 4,
308
+ ],
309
+ );
310
+ // Explicit user-visible rows at the verify boundary. Internal graph
311
+ // temporaries and allocator cache remain covered by the separate allocator
312
+ // reserve rather than being guessed tensor-by-tensor.
313
+ const targetVerifyBytes = product(
314
+ "GLM target verify batch",
315
+ [
316
+ batchSize,
317
+ verifyRows,
318
+ geometry.hiddenSize + geometry.vocabSize,
319
+ 4,
320
+ ],
321
+ );
322
+ const mtpDraftBytes = enableMtp
323
+ ? product(
324
+ "GLM MTP draft batch",
325
+ [
326
+ batchSize,
327
+ mtpDraftTokens,
328
+ geometry.hiddenSize + geometry.vocabSize,
329
+ 4,
330
+ ],
331
+ )
332
+ : 0;
333
+ const verifyBatchTransientBytes =
334
+ sum("GLM verify batch transient", [targetVerifyBytes, mtpDraftBytes]);
335
+
336
+ const lineItems: Glm52MemoryLineItems = {
337
+ residentWeightsBytes: geometry.residentWeightBytes,
338
+ mainExpertSlabBytes,
339
+ mtpExpertSlabBytes,
340
+ targetKvBytes,
341
+ mtpKvBytes,
342
+ reconstructedKvTransientBytes,
343
+ verifyBatchTransientBytes,
344
+ allocatorReserveBytes,
345
+ bunNativeReserveBytes,
346
+ safetyMarginBytes,
347
+ };
348
+ const runtimeReserveBytes = sum("GLM runtime reserve", [
349
+ targetKvBytes,
350
+ mtpKvBytes,
351
+ reconstructedKvTransientBytes,
352
+ verifyBatchTransientBytes,
353
+ allocatorReserveBytes,
354
+ bunNativeReserveBytes,
355
+ safetyMarginBytes,
356
+ ]);
357
+ const plannedProcessBytes = sum(
358
+ "GLM planned process bytes",
359
+ Object.values(lineItems),
360
+ );
361
+ if (plannedProcessBytes > processLimitBytes) {
362
+ throw new Error(
363
+ `GLM memory contract cannot start: planned ${plannedProcessBytes} bytes ` +
364
+ `exceed the ${processLimitBytes}-byte process limit`,
365
+ );
366
+ }
367
+ const osReserveBytes = machineBytes - processLimitBytes;
368
+ const plannedMachineBytes = sum(
369
+ "GLM planned machine bytes",
370
+ [plannedProcessBytes, osReserveBytes],
371
+ );
372
+
373
+ return {
374
+ schemaVersion: 1,
375
+ strategy: "glm52-colibri",
376
+ fits: true,
377
+ preset: "g5-32gb-quality",
378
+ machineBytes,
379
+ processLimitBytes,
380
+ osReserveBytes,
381
+ contextTokens,
382
+ maxSafeContext: contextTokens,
383
+ maxGenerationTokens,
384
+ batchSize,
385
+ enableMtp,
386
+ mtpDraftTokens,
387
+ verifyRows,
388
+ mainWorkingSlots,
389
+ mainRequiredUnionSlots,
390
+ mainResidentSlots,
391
+ mainTotalSlots,
392
+ mtpWorkingSlots,
393
+ mtpResidentSlots,
394
+ mtpTotalSlots,
395
+ lineItems,
396
+ weightsBytes: sum("GLM planned weights", [
397
+ lineItems.residentWeightsBytes,
398
+ lineItems.mainExpertSlabBytes,
399
+ lineItems.mtpExpertSlabBytes,
400
+ ]),
401
+ kvBytes: sum("GLM planned KV", [
402
+ lineItems.targetKvBytes,
403
+ lineItems.mtpKvBytes,
404
+ ]),
405
+ transientBytes: sum("GLM planned transient", [
406
+ lineItems.reconstructedKvTransientBytes,
407
+ lineItems.verifyBatchTransientBytes,
408
+ ]),
409
+ reserveBytes: sum("GLM planned reserves", [
410
+ lineItems.allocatorReserveBytes,
411
+ lineItems.bunNativeReserveBytes,
412
+ lineItems.safetyMarginBytes,
413
+ ]),
414
+ totalBytes: plannedProcessBytes,
415
+ usableBytes: processLimitBytes,
416
+ predictedDecodeTps: null,
417
+ allocatorLimitBytes: lineItems.allocatorReserveBytes,
418
+ streamedWeights: {
419
+ "main expert slab": lineItems.mainExpertSlabBytes,
420
+ "MTP expert slab": lineItems.mtpExpertSlabBytes,
421
+ },
422
+ runtimeReserveBytes,
423
+ plannedProcessBytes,
424
+ processHeadroomBytes: processLimitBytes - plannedProcessBytes,
425
+ plannedMachineBytes,
426
+ machineHeadroomBytes: machineBytes - plannedMachineBytes,
427
+ };
428
+ }
429
+
430
+ export function describeGlm52MemoryGeometry(
431
+ container: ColibriGlm52Container,
432
+ config: Glm52Config,
433
+ enableMtp = true,
434
+ ): Glm52MemoryGeometry {
435
+ const sparseLayers = config.numHiddenLayers - config.firstKDenseReplace;
436
+ if (sparseLayers <= 0)
437
+ throw new Error("GLM memory contract requires at least one sparse layer");
438
+ const capabilities = container.capabilities(config);
439
+ if (enableMtp && !capabilities.hasMtp) {
440
+ throw new Error(
441
+ "GLM memory contract cannot enable MTP: artifact is missing the " +
442
+ `complete MTP tensor family (${capabilities.missingMtpTensors.join(", ")})`,
443
+ );
444
+ }
445
+ const mainSlot = buildGlm52ExpertSlotLayout(
446
+ container,
447
+ config,
448
+ config.firstKDenseReplace,
449
+ 0,
450
+ );
451
+ const mtpSlot = enableMtp
452
+ ? buildGlm52ExpertSlotLayout(
453
+ container,
454
+ config,
455
+ config.numHiddenLayers,
456
+ 0,
457
+ )
458
+ : null;
459
+ const dsaLayers = capabilities.hasDsa
460
+ ? config.indexerTypes.filter((kind) => kind === "full").length
461
+ : 0;
462
+ return {
463
+ residentWeightBytes: bytesWithoutRoutedExperts(container, enableMtp),
464
+ mainExpertSlotBytes: mainSlot.slotBytes,
465
+ ...(mtpSlot ? { mtpExpertSlotBytes: mtpSlot.slotBytes } : {}),
466
+ hiddenSize: config.hiddenSize,
467
+ vocabSize: config.vocabSize,
468
+ numHiddenLayers: config.numHiddenLayers,
469
+ sparseLayers,
470
+ numAttentionHeads: config.numAttentionHeads,
471
+ kvLoraRank: config.kvLoraRank,
472
+ qkRopeHeadDim: config.qkRopeHeadDim,
473
+ qkHeadDim: config.qkHeadDim,
474
+ vHeadDim: config.vHeadDim,
475
+ numRoutedExperts: config.numRoutedExperts,
476
+ numExpertsPerToken: config.numExpertsPerToken,
477
+ dsaLayers,
478
+ dsaHeadDim: config.indexHeadDim,
479
+ mtpLayers: enableMtp ? config.numNextnPredictLayers : 0,
480
+ };
481
+ }
482
+
483
+ /**
484
+ * Header-only preflight for the direct Colibri artifact. This is intentionally
485
+ * safe to call before any resident weight mmap or native expert slab exists.
486
+ */
487
+ export async function planGlm52MemoryForArtifact(
488
+ modelDir: string,
489
+ options: Glm52MemoryPlanOptions = {},
490
+ ): Promise<Glm52MemoryPlan> {
491
+ const enableMtp = options.enableMtp !== false;
492
+ const [config, container] = await Promise.all([
493
+ loadGlm52Config(modelDir),
494
+ Promise.resolve().then(() => ColibriGlm52Container.open(modelDir)),
495
+ ]);
496
+ return planGlm52Memory(
497
+ describeGlm52MemoryGeometry(container, config, enableMtp),
498
+ { ...options, enableMtp },
499
+ );
500
+ }