doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -0,0 +1,564 @@
1
+
2
+ import { log } from '../../debug/index.js';
3
+
4
+ // =============================================================================
5
+ // TurboQuant Codebook & Rotation Matrix Module
6
+ //
7
+ // Precomputes Max-Lloyd scalar quantizer codebooks for the Beta distribution
8
+ // arising from random orthogonal rotation of unit-norm vectors, plus
9
+ // deterministic rotation matrix generation via Householder QR.
10
+ // =============================================================================
11
+
12
+ // -- Seeded PRNG (xoshiro128**) -----------------------------------------------
13
+
14
+ function createPRNG(seed) {
15
+ const s = new Uint32Array(4);
16
+ let z = seed | 0;
17
+ for (let i = 0; i < 4; i++) {
18
+ z = (z + 0x9e3779b9) | 0;
19
+ let t = z ^ (z >>> 16);
20
+ t = Math.imul(t, 0x85ebca6b);
21
+ t = t ^ (t >>> 13);
22
+ t = Math.imul(t, 0xc2b2ae35);
23
+ s[i] = (t ^ (t >>> 16)) >>> 0;
24
+ }
25
+
26
+ function rotl(x, k) {
27
+ return ((x << k) | (x >>> (32 - k))) >>> 0;
28
+ }
29
+
30
+ function nextU32() {
31
+ const result = (Math.imul(rotl(Math.imul(s[1], 5), 7), 9)) >>> 0;
32
+ const t = (s[1] << 9) >>> 0;
33
+ s[2] ^= s[0];
34
+ s[3] ^= s[1];
35
+ s[1] ^= s[2];
36
+ s[0] ^= s[3];
37
+ s[2] ^= t;
38
+ s[3] = rotl(s[3], 11);
39
+ return result;
40
+ }
41
+
42
+ function nextFloat() {
43
+ return (nextU32() >>> 0) / 4294967296;
44
+ }
45
+
46
+ function nextGaussian() {
47
+ const u1 = nextFloat() || 1e-10;
48
+ const u2 = nextFloat();
49
+ return Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
50
+ }
51
+
52
+ return { nextFloat, nextGaussian };
53
+ }
54
+
55
+ // -- Sphere marginal PDF ------------------------------------------------------
56
+ // The marginal distribution of a single coordinate of a uniform point on
57
+ // S^{d-1} is proportional to (1 - x^2)^((d-3)/2) on [-1, 1].
58
+
59
+ function sphereMarginalPDF(x, d) {
60
+ if (d < 3) {
61
+ throw new Error('TurboQuant requires headDim >= 3.');
62
+ }
63
+ const t = 1.0 - x * x;
64
+ if (t <= 0) return 0;
65
+ const exponent = (d - 3) / 2;
66
+ return Math.pow(t, exponent);
67
+ }
68
+
69
+ // -- Numerical integration (Simpson's rule) -----------------------------------
70
+
71
+ function integrateWeighted(weightFn, valueFn, a, b, n) {
72
+ if (n % 2 !== 0) n++;
73
+ const h = (b - a) / n;
74
+ let sumW = 0;
75
+ let sumV = 0;
76
+ for (let i = 0; i <= n; i++) {
77
+ const x = a + i * h;
78
+ const coeff = (i === 0 || i === n) ? 1 : (i % 2 === 0 ? 2 : 4);
79
+ const w = weightFn(x) * coeff;
80
+ sumW += w;
81
+ sumV += w * valueFn(x);
82
+ }
83
+ return { integral: sumW * h / 3, moment: sumV * h / 3 };
84
+ }
85
+
86
+ function integrate(fn, a, b, n) {
87
+ if (n % 2 !== 0) n++;
88
+ const h = (b - a) / n;
89
+ let sum = 0;
90
+ for (let i = 0; i <= n; i++) {
91
+ const x = a + i * h;
92
+ const coeff = (i === 0 || i === n) ? 1 : (i % 2 === 0 ? 2 : 4);
93
+ sum += fn(x) * coeff;
94
+ }
95
+ return sum * h / 3;
96
+ }
97
+
98
+ // -- Max-Lloyd quantizer for sphere marginal ----------------------------------
99
+
100
+ const QUADRATURE_POINTS = 2000;
101
+ const MAX_LLOYD_ITERATIONS = 200;
102
+ const LLOYD_CONVERGENCE_EPS = 1e-12;
103
+
104
+ /**
105
+ * Compute the Max-Lloyd optimal scalar quantizer for the sphere marginal
106
+ * distribution at a given dimension and bit-width.
107
+ *
108
+ * @param {number} d - Dimension (headDim).
109
+ * @param {number} bitWidth - Bits per coordinate (1, 2, 3, or 4).
110
+ * @returns {{ centroids: Float32Array, boundaries: Float32Array }}
111
+ */
112
+ export function computeMaxLloydCodebook(d, bitWidth) {
113
+ if (bitWidth < 1 || bitWidth > 4) {
114
+ throw new Error(`TurboQuant bit-width must be 1-4; got ${bitWidth}.`);
115
+ }
116
+ const numCentroids = 1 << bitWidth;
117
+ const numBoundaries = numCentroids - 1;
118
+ const pdf = (x) => sphereMarginalPDF(x, d);
119
+
120
+ // Initialize boundaries uniformly on [-1, 1]
121
+ const boundaries = new Float64Array(numBoundaries);
122
+ for (let i = 0; i < numBoundaries; i++) {
123
+ boundaries[i] = -1.0 + (2.0 * (i + 1)) / numCentroids;
124
+ }
125
+ const centroids = new Float64Array(numCentroids);
126
+
127
+ for (let iter = 0; iter < MAX_LLOYD_ITERATIONS; iter++) {
128
+ // Update centroids: c_i = E[X | b_{i-1} <= X < b_i]
129
+ let prevBound = -1.0;
130
+ for (let i = 0; i < numCentroids; i++) {
131
+ const nextBound = i < numBoundaries ? boundaries[i] : 1.0;
132
+ const { integral: mass, moment } = integrateWeighted(
133
+ pdf, (x) => x, prevBound, nextBound, QUADRATURE_POINTS
134
+ );
135
+ centroids[i] = mass > 0 ? moment / mass : (prevBound + nextBound) / 2;
136
+ prevBound = nextBound;
137
+ }
138
+
139
+ // Update boundaries: b_i = (c_i + c_{i+1}) / 2
140
+ let maxShift = 0;
141
+ for (let i = 0; i < numBoundaries; i++) {
142
+ const newBound = (centroids[i] + centroids[i + 1]) / 2;
143
+ maxShift = Math.max(maxShift, Math.abs(newBound - boundaries[i]));
144
+ boundaries[i] = newBound;
145
+ }
146
+
147
+ if (maxShift < LLOYD_CONVERGENCE_EPS) {
148
+ break;
149
+ }
150
+ }
151
+
152
+ return {
153
+ centroids: new Float32Array(centroids),
154
+ boundaries: new Float32Array(boundaries),
155
+ };
156
+ }
157
+
158
+ // -- Codebook cache (per dimension × bit-width) -------------------------------
159
+
160
+ const codebookCache = new Map();
161
+ const rotationMatrixCache = new Map();
162
+ const qjlMatrixCache = new Map();
163
+ const turboQuantSharedBufferCache = new WeakMap();
164
+
165
+ function codebookKey(d, bitWidth) {
166
+ return `${d}:${bitWidth}`;
167
+ }
168
+
169
+ function matrixKey(d, seed) {
170
+ return `${d}:${seed}`;
171
+ }
172
+
173
+ /**
174
+ * Get or compute the Max-Lloyd codebook for a given dimension and bit-width.
175
+ *
176
+ * @param {number} d - Dimension (headDim).
177
+ * @param {number} bitWidth - Bits per coordinate.
178
+ * @returns {{ centroids: Float32Array, boundaries: Float32Array }}
179
+ */
180
+ export function getCodebook(d, bitWidth) {
181
+ const key = codebookKey(d, bitWidth);
182
+ let cb = codebookCache.get(key);
183
+ if (!cb) {
184
+ cb = computeMaxLloydCodebook(d, bitWidth);
185
+ codebookCache.set(key, cb);
186
+ log.info('TurboQuant', `Computed codebook: d=${d}, b=${bitWidth}, centroids=${cb.centroids.length}`);
187
+ }
188
+ return cb;
189
+ }
190
+
191
+ export function getRotationMatrix(d, seed) {
192
+ const key = matrixKey(d, seed);
193
+ let matrix = rotationMatrixCache.get(key);
194
+ if (!matrix) {
195
+ matrix = generateRotationMatrix(d, seed);
196
+ rotationMatrixCache.set(key, matrix);
197
+ log.info('TurboQuant', `Computed rotation matrix: d=${d}, seed=${seed}`);
198
+ }
199
+ return matrix;
200
+ }
201
+
202
+ export function getQJLMatrix(d, seed) {
203
+ const key = matrixKey(d, seed);
204
+ let matrix = qjlMatrixCache.get(key);
205
+ if (!matrix) {
206
+ matrix = generateQJLMatrix(d, seed);
207
+ qjlMatrixCache.set(key, matrix);
208
+ log.info('TurboQuant', `Computed QJL matrix: d=${d}, seed=${seed}`);
209
+ }
210
+ return matrix;
211
+ }
212
+
213
+ // -- Rotation matrix generation -----------------------------------------------
214
+ // Generates a deterministic d×d orthogonal matrix via QR decomposition
215
+ // of a random Gaussian matrix using Householder reflections.
216
+
217
+ /**
218
+ * Generate a deterministic d×d orthogonal rotation matrix.
219
+ *
220
+ * @param {number} d - Dimension (headDim).
221
+ * @param {number} seed - PRNG seed for reproducibility.
222
+ * @returns {Float32Array} - Flattened d×d orthogonal matrix (row-major).
223
+ */
224
+ export function generateRotationMatrix(d, seed) {
225
+ const rng = createPRNG(seed);
226
+
227
+ // Generate random Gaussian matrix A (d × d)
228
+ const A = new Float64Array(d * d);
229
+ for (let i = 0; i < d * d; i++) {
230
+ A[i] = rng.nextGaussian();
231
+ }
232
+
233
+ // Householder QR decomposition → extract Q
234
+ const R = new Float64Array(A);
235
+ const Q = new Float64Array(d * d);
236
+
237
+ // Initialize Q as identity
238
+ for (let i = 0; i < d; i++) {
239
+ Q[i * d + i] = 1.0;
240
+ }
241
+
242
+ for (let k = 0; k < d; k++) {
243
+ // Extract column k below diagonal
244
+ const v = new Float64Array(d - k);
245
+ for (let i = k; i < d; i++) {
246
+ v[i - k] = R[i * d + k];
247
+ }
248
+
249
+ // Compute Householder vector
250
+ let normV = 0;
251
+ for (let i = 0; i < v.length; i++) {
252
+ normV += v[i] * v[i];
253
+ }
254
+ normV = Math.sqrt(normV);
255
+
256
+ if (normV < 1e-15) continue;
257
+
258
+ const sign = v[0] >= 0 ? 1 : -1;
259
+ v[0] += sign * normV;
260
+
261
+ // Normalize v
262
+ let normV2 = 0;
263
+ for (let i = 0; i < v.length; i++) {
264
+ normV2 += v[i] * v[i];
265
+ }
266
+ if (normV2 < 1e-30) continue;
267
+ const invNorm = 2.0 / normV2;
268
+
269
+ // Apply H = I - 2vv^T/||v||^2 to R columns k..d-1
270
+ for (let j = k; j < d; j++) {
271
+ let dot = 0;
272
+ for (let i = 0; i < v.length; i++) {
273
+ dot += v[i] * R[(i + k) * d + j];
274
+ }
275
+ dot *= invNorm;
276
+ for (let i = 0; i < v.length; i++) {
277
+ R[(i + k) * d + j] -= v[i] * dot;
278
+ }
279
+ }
280
+
281
+ // Apply H to Q columns
282
+ for (let j = 0; j < d; j++) {
283
+ let dot = 0;
284
+ for (let i = 0; i < v.length; i++) {
285
+ dot += v[i] * Q[(i + k) * d + j];
286
+ }
287
+ dot *= invNorm;
288
+ for (let i = 0; i < v.length; i++) {
289
+ Q[(i + k) * d + j] -= v[i] * dot;
290
+ }
291
+ }
292
+ }
293
+
294
+ // Ensure det(Q) = +1 (proper rotation) by flipping first row if needed
295
+ let det = 1.0;
296
+ for (let i = 0; i < d; i++) {
297
+ det *= R[i * d + i];
298
+ }
299
+ if (det < 0) {
300
+ for (let j = 0; j < d; j++) {
301
+ Q[j] = -Q[j]; // Flip first row
302
+ }
303
+ }
304
+
305
+ return new Float32Array(Q);
306
+ }
307
+
308
+ // -- QJL projection matrix for TURBOQUANTprod --------------------------------
309
+ // The QJL (Quantized Johnson-Lindenstrauss) transform uses a random sign
310
+ // matrix for 1-bit residual quantization.
311
+
312
+ /**
313
+ * Generate a deterministic d×d random sign matrix for QJL projection.
314
+ * Each entry is +1 or -1 with equal probability, scaled by 1/sqrt(d).
315
+ *
316
+ * @param {number} d - Dimension (headDim).
317
+ * @param {number} seed - PRNG seed (should differ from rotation seed).
318
+ * @returns {Float32Array} - Flattened d×d sign matrix (row-major).
319
+ */
320
+ export function generateQJLMatrix(d, seed) {
321
+ const rng = createPRNG(seed);
322
+ const P = new Float32Array(d * d);
323
+ const scale = 1.0 / Math.sqrt(d);
324
+ for (let i = 0; i < d * d; i++) {
325
+ P[i] = rng.nextFloat() < 0.5 ? -scale : scale;
326
+ }
327
+ return P;
328
+ }
329
+
330
+ // -- Outlier fraction for non-integer bit-widths ------------------------------
331
+
332
+ /**
333
+ * Compute the fraction of channels that should use higher precision
334
+ * for a given effective bit-width.
335
+ *
336
+ * @param {number} effectiveBits - Target effective bits (e.g., 2.5, 3.5).
337
+ * @param {number} bitsHigh - Bits for outlier channels.
338
+ * @param {number} bitsLow - Bits for non-outlier channels.
339
+ * @returns {number} - Fraction of channels using bitsHigh (0 to 1).
340
+ */
341
+ export function computeOutlierFraction(effectiveBits, bitsHigh, bitsLow) {
342
+ if (bitsHigh <= bitsLow) {
343
+ throw new Error(`bitsHigh (${bitsHigh}) must be > bitsLow (${bitsLow}).`);
344
+ }
345
+ // effectiveBits = fraction * bitsHigh + (1 - fraction) * bitsLow
346
+ const fraction = (effectiveBits - bitsLow) / (bitsHigh - bitsLow);
347
+ if (fraction < 0 || fraction > 1) {
348
+ throw new Error(
349
+ `Effective bits ${effectiveBits} out of range [${bitsLow}, ${bitsHigh}].`
350
+ );
351
+ }
352
+ return fraction;
353
+ }
354
+
355
+ /**
356
+ * Resolve outlier bit-width config from effective bit-width.
357
+ *
358
+ * @param {number} effectiveBits - Target (e.g., 2.5, 3.5, or integer 2/3/4).
359
+ * @returns {{ bitsHigh: number, bitsLow: number, outlierFraction: number, isNonInteger: boolean }}
360
+ */
361
+ export function resolveOutlierConfig(effectiveBits) {
362
+ if (Number.isInteger(effectiveBits) && effectiveBits >= 1 && effectiveBits <= 4) {
363
+ return { bitsHigh: effectiveBits, bitsLow: effectiveBits, outlierFraction: 0, isNonInteger: false };
364
+ }
365
+ const bitsLow = Math.floor(effectiveBits);
366
+ const bitsHigh = bitsLow + 1;
367
+ if (bitsLow < 1 || bitsHigh > 4) {
368
+ throw new Error(`Effective bits ${effectiveBits} out of supported range [1, 4].`);
369
+ }
370
+ const outlierFraction = computeOutlierFraction(effectiveBits, bitsHigh, bitsLow);
371
+ return { bitsHigh, bitsLow, outlierFraction, isNonInteger: true };
372
+ }
373
+
374
+ // -- GPU buffer upload helpers ------------------------------------------------
375
+
376
+ /**
377
+ * Upload rotation matrix to a GPU buffer.
378
+ *
379
+ * @param {GPUDevice} device - WebGPU device.
380
+ * @param {Float32Array} matrix - Flattened d×d rotation matrix.
381
+ * @param {string} label - Buffer label.
382
+ * @returns {GPUBuffer}
383
+ */
384
+ export function uploadRotationMatrix(device, matrix, label) {
385
+ const buf = device.createBuffer({
386
+ label,
387
+ size: matrix.byteLength,
388
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
389
+ });
390
+ device.queue.writeBuffer(buf, 0, matrix);
391
+ return buf;
392
+ }
393
+
394
+ /**
395
+ * Upload codebook (centroids + boundaries) to GPU buffers.
396
+ *
397
+ * @param {GPUDevice} device - WebGPU device.
398
+ * @param {{ centroids: Float32Array, boundaries: Float32Array }} codebook
399
+ * @param {string} prefix - Buffer label prefix.
400
+ * @returns {{ centroidsBuffer: GPUBuffer, boundariesBuffer: GPUBuffer }}
401
+ */
402
+ export function uploadCodebook(device, codebook, prefix) {
403
+ const centroidsBuffer = device.createBuffer({
404
+ label: `${prefix}_centroids`,
405
+ size: Math.max(4, codebook.centroids.byteLength),
406
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
407
+ });
408
+ device.queue.writeBuffer(centroidsBuffer, 0, codebook.centroids);
409
+
410
+ const boundariesBuffer = device.createBuffer({
411
+ label: `${prefix}_boundaries`,
412
+ size: Math.max(4, codebook.boundaries.byteLength),
413
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
414
+ });
415
+ device.queue.writeBuffer(boundariesBuffer, 0, codebook.boundaries);
416
+
417
+ return { centroidsBuffer, boundariesBuffer };
418
+ }
419
+
420
+ function getDeviceSharedBufferCache(device) {
421
+ let cache = turboQuantSharedBufferCache.get(device);
422
+ if (!cache) {
423
+ cache = new Map();
424
+ turboQuantSharedBufferCache.set(device, cache);
425
+ }
426
+ return cache;
427
+ }
428
+
429
+ function retainSharedBuffer(cache, key, factory) {
430
+ let entry = cache.get(key);
431
+ if (!entry) {
432
+ entry = factory();
433
+ entry.refCount = 0;
434
+ cache.set(key, entry);
435
+ }
436
+ entry.refCount += 1;
437
+
438
+ let released = false;
439
+ return {
440
+ value: entry.value,
441
+ release() {
442
+ if (released) {
443
+ return;
444
+ }
445
+ released = true;
446
+ entry.refCount -= 1;
447
+ if (entry.refCount <= 0) {
448
+ entry.destroy();
449
+ cache.delete(key);
450
+ }
451
+ },
452
+ };
453
+ }
454
+
455
+ export function retainTurboQuantSharedBuffers(device, options = {}) {
456
+ const headDim = Number(options.headDim);
457
+ const bitWidth = Number(options.bitWidth ?? 4);
458
+ const prodMode = options.prodMode === true;
459
+ if (!device) {
460
+ throw new Error('TurboQuant shared buffers require a GPU device.');
461
+ }
462
+ if (!Number.isInteger(headDim) || headDim <= 0) {
463
+ throw new Error(`TurboQuant shared buffers require headDim > 0; got "${options.headDim}".`);
464
+ }
465
+ if (!Number.isInteger(bitWidth) || bitWidth < 1 || bitWidth > 4) {
466
+ throw new Error(`TurboQuant shared buffers require bitWidth in [1, 4]; got "${options.bitWidth}".`);
467
+ }
468
+
469
+ const cache = getDeviceSharedBufferCache(device);
470
+ const releasers = [];
471
+
472
+ const rotation = retainSharedBuffer(cache, `rotation:${headDim}`, () => {
473
+ const buffer = uploadRotationMatrix(device, getRotationMatrix(headDim, ROTATION_SEED), 'turboquant_rotation');
474
+ return {
475
+ value: buffer,
476
+ destroy() {
477
+ buffer.destroy();
478
+ },
479
+ };
480
+ });
481
+ releasers.push(rotation.release);
482
+
483
+ const codebook = retainSharedBuffer(cache, `codebook:${headDim}:${bitWidth}`, () => {
484
+ const uploaded = uploadCodebook(device, getCodebook(headDim, bitWidth), 'turboquant_codebook');
485
+ return {
486
+ value: uploaded,
487
+ destroy() {
488
+ uploaded.centroidsBuffer.destroy();
489
+ uploaded.boundariesBuffer.destroy();
490
+ },
491
+ };
492
+ });
493
+ releasers.push(codebook.release);
494
+
495
+ let qjl = null;
496
+ if (prodMode) {
497
+ qjl = retainSharedBuffer(cache, `qjl:${headDim}`, () => {
498
+ const qjlData = getQJLMatrix(headDim, QJL_SEED);
499
+ const buffer = device.createBuffer({
500
+ label: 'turboquant_qjl_matrix',
501
+ size: qjlData.byteLength,
502
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
503
+ });
504
+ device.queue.writeBuffer(buffer, 0, qjlData);
505
+ return {
506
+ value: buffer,
507
+ destroy() {
508
+ buffer.destroy();
509
+ },
510
+ };
511
+ });
512
+ releasers.push(qjl.release);
513
+ }
514
+
515
+ let released = false;
516
+ return {
517
+ rotationMatrixBuffer: rotation.value,
518
+ codebookCentroidsBuffer: codebook.value.centroidsBuffer,
519
+ codebookBoundariesBuffer: codebook.value.boundariesBuffer,
520
+ qjlMatrixBuffer: qjl ? qjl.value : null,
521
+ release() {
522
+ if (released) {
523
+ return;
524
+ }
525
+ released = true;
526
+ for (let idx = releasers.length - 1; idx >= 0; idx--) {
527
+ releasers[idx]();
528
+ }
529
+ },
530
+ };
531
+ }
532
+
533
+ // -- TurboQuant packing helpers -----------------------------------------------
534
+
535
+ /**
536
+ * Compute packed stride for a given headDim and bit-width.
537
+ * packedStride = ceil(headDim / packFactor) where packFactor = floor(32 / bitWidth).
538
+ *
539
+ * @param {number} headDim
540
+ * @param {number} bitWidth
541
+ * @returns {number}
542
+ */
543
+ export function computePackedStride(headDim, bitWidth) {
544
+ const packFactor = Math.floor(32 / bitWidth);
545
+ return Math.ceil(headDim / packFactor);
546
+ }
547
+
548
+ /**
549
+ * Compute the pack factor for a given bit-width.
550
+ *
551
+ * @param {number} bitWidth
552
+ * @returns {number}
553
+ */
554
+ export function computePackFactor(bitWidth) {
555
+ return Math.floor(32 / bitWidth);
556
+ }
557
+
558
+ // -- Default seeds ------------------------------------------------------------
559
+
560
+ /** Default seed for rotation matrix Π. */
561
+ export const ROTATION_SEED = 0x54515545; // "TQUE"
562
+
563
+ /** Default seed for QJL projection matrix P (must differ from rotation seed). */
564
+ export const QJL_SEED = 0x514A4C50; // "QJLP"
@@ -40,6 +40,32 @@ export function writeUniformsFromObject(view, config, values) {
40
40
  }
41
41
  }
42
42
 
43
+ export function getUniformByteLength(config) {
44
+ const uniforms = config?.uniforms;
45
+ if (!uniforms) {
46
+ return 0;
47
+ }
48
+
49
+ const declaredSize = Number.isFinite(uniforms.size) ? uniforms.size : 0;
50
+ let requiredSize = 0;
51
+ for (const field of uniforms.fields ?? []) {
52
+ if (!Number.isFinite(field?.offset)) {
53
+ continue;
54
+ }
55
+ switch (field.type) {
56
+ case 'u32':
57
+ case 'i32':
58
+ case 'f32':
59
+ requiredSize = Math.max(requiredSize, field.offset + 4);
60
+ break;
61
+ default:
62
+ break;
63
+ }
64
+ }
65
+
66
+ return Math.max(declaredSize, requiredSize);
67
+ }
68
+
43
69
 
44
70
  export function createUniformBufferFromData(
45
71
  label,
@@ -62,6 +62,7 @@ export {
62
62
  export {
63
63
  createUniformBufferFromData,
64
64
  createUniformBufferWithView,
65
+ getUniformByteLength,
65
66
  writeUniformsFromObject,
66
67
  } from './uniform-utils.js';
67
68
 
@@ -74,6 +75,7 @@ import { getPipelineFast } from './pipeline-cache.js';
74
75
  import { getDevice } from '../device.js';
75
76
  import { dispatchKernel, dispatchIndirect, recordDispatchIndirect } from './dispatch.js';
76
77
  import { createUniformBufferWithView as createUniformBuffer } from './uniform-utils.js';
78
+ import { getUniformByteLength } from './uniform-utils.js';
77
79
  import { writeUniformsFromObject } from './uniform-utils.js';
78
80
 
79
81
  export async function unifiedKernelWrapper(opName, target, variant, bindings, uniforms, workgroups, constants = null, extraBindings = null) {
@@ -84,7 +86,7 @@ export async function unifiedKernelWrapper(opName, target, variant, bindings, un
84
86
 
85
87
  const uniformBuffer = createUniformBuffer(
86
88
  `${opName}_uniforms`,
87
- config.uniforms.size,
89
+ getUniformByteLength(config),
88
90
  (view) => writeUniformsFromObject(view, config, uniforms),
89
91
  recorder,
90
92
  device
@@ -248,10 +250,13 @@ import { validateAttentionLimits } from './feature-check.js';
248
250
  // This avoids circular dependencies between configs and validation
249
251
  setKernelValidator('attention', 'prefill', validateAttentionLimits);
250
252
  setKernelValidator('attention', 'prefill_small', validateAttentionLimits);
253
+ setKernelValidator('attention', 'decode_small', validateAttentionLimits);
251
254
  setKernelValidator('attention', 'prefill_streaming', validateAttentionLimits);
252
255
  setKernelValidator('attention', 'prefill_f16', validateAttentionLimits);
253
256
  setKernelValidator('attention', 'prefill_small_f16', validateAttentionLimits);
257
+ setKernelValidator('attention', 'decode_small_f16', validateAttentionLimits);
254
258
  setKernelValidator('attention', 'prefill_streaming_f16', validateAttentionLimits);
255
259
  setKernelValidator('attention', 'prefill_f16kv', validateAttentionLimits);
256
260
  setKernelValidator('attention', 'prefill_small_f16kv', validateAttentionLimits);
261
+ setKernelValidator('attention', 'decode_small_f16kv', validateAttentionLimits);
257
262
  setKernelValidator('attention', 'prefill_streaming_f16kv', validateAttentionLimits);
@@ -0,0 +1,12 @@
1
+ /**
2
+ * GPU submit latency probe.
3
+ *
4
+ * Dispatches a trivial compute shader, submits, waits for the GPU fence
5
+ * plus a staging-buffer readback, and returns the wall-clock roundtrip in ms.
6
+ */
7
+
8
+ /**
9
+ * Run a single submit+readback roundtrip and return the elapsed time in ms.
10
+ * Returns `null` if the probe cannot run.
11
+ */
12
+ export function probeSubmitLatency(device: GPUDevice): Promise<number | null>;