doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -3,7 +3,19 @@
3
3
  import { parseModelConfig } from './config.js';
4
4
  import { getDevice, getDeviceLimits, getKernelCapabilities } from '../../../gpu/device.js';
5
5
  import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
6
- import { KVCache, SlidingWindowKVCache, TieredKVCache, BasisDecomposedPagedCache } from '../../kv-cache.js';
6
+ import {
7
+ KVCache,
8
+ SlidingWindowKVCache,
9
+ TieredKVCache,
10
+ BasisDecomposedPagedCache,
11
+ QuantizedKVCache,
12
+ MixedGeometryKVCache,
13
+ } from '../../kv-cache.js';
14
+ import {
15
+ retainTurboQuantSharedBuffers,
16
+ } from '../../../gpu/kernels/turboquant-codebook.js';
17
+ import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
18
+ import { selectRuleValue as selectKernelRuleValue } from '../../../gpu/kernels/rule-registry.js';
7
19
  import { Tokenizer } from '../../tokenizer.js';
8
20
  import { MoERouter } from '../../moe-router.js';
9
21
  import { SpeculativeDecoder } from '../../speculative.js';
@@ -11,13 +23,16 @@ import { getDopplerLoader } from '../../../loader/doppler-loader.js';
11
23
  import { log, setGPUDevice, trace as debugTrace } from '../../../debug/index.js';
12
24
  import { getRuntimeConfig } from '../../../config/runtime.js';
13
25
  import { PAGED_LAYOUT_SEQ_LEN_THRESHOLD } from '../../../config/schema/index.js';
14
- import { isKernelPathFusedQ4K, kernelPathRequiresF32MatmulWeights } from '../../../config/kernel-path-loader.js';
15
- import { createWeightBuffer, getWeightDtype, isWeightBuffer } from '../../../gpu/weight-buffer.js';
26
+ import {
27
+ getKernelPathMatmulVariant,
28
+ kernelPathRequiresF32MatmulWeights,
29
+ } from '../../../config/kernel-path-loader.js';
30
+ import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
16
31
  import { selectRuleValue } from '../../../rules/rule-registry.js';
17
32
  import {
18
- createSourceStorageContext,
19
- getSourceRuntimeMetadata,
20
- } from '../../../tooling/source-runtime-bundle.js';
33
+ createHttpArtifactStorageContext,
34
+ createNodeFileArtifactStorageContext,
35
+ } from '../../../storage/artifact-storage-context.js';
21
36
 
22
37
  function resolveErrorMessage(error) {
23
38
  if (error && typeof error === 'object' && typeof error.message === 'string') {
@@ -29,6 +44,48 @@ function resolveErrorMessage(error) {
29
44
  return String(error);
30
45
  }
31
46
 
47
+ function isPlainObject(value) {
48
+ return value != null && typeof value === 'object' && !Array.isArray(value);
49
+ }
50
+
51
+ function isRuntimeInferenceConfigInput(value) {
52
+ if (!isPlainObject(value)) {
53
+ return false;
54
+ }
55
+ return Object.prototype.hasOwnProperty.call(value, 'session')
56
+ || Object.prototype.hasOwnProperty.call(value, 'kvcache')
57
+ || Object.prototype.hasOwnProperty.call(value, 'compute')
58
+ || Object.prototype.hasOwnProperty.call(value, 'batching')
59
+ || Object.prototype.hasOwnProperty.call(value, 'generation')
60
+ || Object.prototype.hasOwnProperty.call(value, 'kernelPath')
61
+ || Object.prototype.hasOwnProperty.call(value, 'pipeline');
62
+ }
63
+
64
+ function resolveRuntimeKVConfig(runtimeInput) {
65
+ if (runtimeInput != null && !isRuntimeInferenceConfigInput(runtimeInput)) {
66
+ return runtimeInput;
67
+ }
68
+ const runtimeInference = runtimeInput ?? getRuntimeConfig().inference;
69
+ if (runtimeInference?.session?.kvcache) {
70
+ return runtimeInference.session.kvcache;
71
+ }
72
+ throw new Error(
73
+ 'runtime.inference.session.kvcache is required for live KV cache creation. ' +
74
+ 'Top-level runtime.inference.kvcache is not supported on the live path.'
75
+ );
76
+ }
77
+
78
+ function assertSupportedTurboQuantMode(mode, label) {
79
+ if (String(mode ?? '').trim().toLowerCase() !== 'turboquant_outlier') {
80
+ return;
81
+ }
82
+ throw new Error(
83
+ `${label}="turboquant_outlier" is not supported yet. ` +
84
+ 'TurboQuant outlier high-precision buffers and decode kernels are not wired end to end; ' +
85
+ 'use "turboquant" or "turboquant_prod".'
86
+ );
87
+ }
88
+
32
89
  function toArrayBuffer(value, label) {
33
90
  if (value instanceof ArrayBuffer) {
34
91
  return value;
@@ -53,95 +110,81 @@ function isRDRRManifest(manifest) {
53
110
  return manifest !== null && typeof manifest === 'object' && Array.isArray( (manifest).shards);
54
111
  }
55
112
 
56
- // NOTE: This normalizeBaseUrl is duplicated in src/inference/pipelines/diffusion/weights.js
57
- // (simpler variant) and src/tooling/node-browser-command-runner.js (URL-validating variant).
58
- // This copy in text/init.js is the canonical version for inference pipeline shard loading.
59
- // The duplication is intentional to avoid cross-pipeline imports; do not merge without a
60
- // coordinated refactor of both pipeline families.
61
- function normalizeBaseUrl(baseUrl) {
62
- if (typeof baseUrl !== 'string' || baseUrl.trim().length === 0) {
63
- return null;
64
- }
65
- return baseUrl.replace(/\/$/, '');
113
+ export function createNodeFileShardStorageContext(baseUrl, manifest) {
114
+ return createNodeFileArtifactStorageContext(baseUrl, manifest);
66
115
  }
67
116
 
68
- async function fetchBytes(url, offset = null, length = null) {
69
- const headers = {};
70
- if (Number.isFinite(offset) && Number.isFinite(length) && length > 0) {
71
- const start = Math.max(0, Math.floor(offset));
72
- const end = start + Math.max(0, Math.floor(length)) - 1;
73
- headers.Range = `bytes=${start}-${end}`;
74
- }
75
- const response = await fetch(url, { headers });
76
- if (!response.ok) {
77
- throw new Error(`Failed to fetch ${url}: ${response.status}`);
117
+ function createRemoteStorageContext(baseUrl, manifest) {
118
+ const nodeFileStorageContext = createNodeFileArtifactStorageContext(baseUrl, manifest);
119
+ if (nodeFileStorageContext) {
120
+ return nodeFileStorageContext;
78
121
  }
79
- return new Uint8Array(await response.arrayBuffer());
122
+ return createHttpArtifactStorageContext(baseUrl, manifest);
80
123
  }
81
124
 
82
- function createRemoteStorageContext(baseUrl, manifest) {
83
- const root = normalizeBaseUrl(baseUrl);
84
- if (!root || !isRDRRManifest(manifest)) {
85
- return null;
86
- }
87
125
 
88
- if (!Array.isArray(manifest.shards) || manifest.shards.length === 0) {
89
- throw new Error(
90
- `Manifest "${manifest?.modelId ?? 'unknown'}" is missing or has empty shards array. ` +
91
- 'Re-convert the model to produce a valid manifest with shard entries.'
92
- );
126
+ /** Allowed quantizationInfo.layout values for Q4K models. */
127
+ const Q4K_LAYOUT_ALLOWLIST = new Set(['row', 'col']);
128
+ const Q4K_PROJECTION_OPS = new Set([
129
+ 'q_proj', 'k_proj', 'v_proj', 'o_proj',
130
+ 'gate_proj', 'up_proj', 'down_proj',
131
+ 'qkv_proj',
132
+ 'linear_qkv_proj',
133
+ 'linear_z_proj',
134
+ 'linear_a_proj',
135
+ 'linear_b_proj',
136
+ 'linear_out_proj',
137
+ ]);
138
+
139
+ function summarizeQ4KProjectionKernelKinds(kernelPath) {
140
+ const summary = {
141
+ denseProjectionKernels: [],
142
+ fusedProjectionKernels: [],
143
+ };
144
+ if (!kernelPath || typeof kernelPath !== 'object') {
145
+ return summary;
93
146
  }
94
147
 
95
- const sourceRuntime = getSourceRuntimeMetadata(manifest);
96
- if (sourceRuntime) {
97
- const readRange = async (relativePath, offset, length) => {
98
- const filename = String(relativePath || '').replace(/^\/+/, '');
99
- if (!filename) {
100
- throw new Error('Direct-source artifact path is required.');
101
- }
102
- const url = `${root}/${filename}`;
103
- return fetchBytes(url, offset, length);
104
- };
105
- const readText = async (relativePath) => {
106
- const filename = String(relativePath || '').replace(/^\/+/, '');
107
- if (!filename) return null;
108
- const response = await fetch(`${root}/${filename}`);
109
- if (!response.ok) {
110
- throw new Error(`Failed to fetch ${filename} from ${root}: ${response.status}`);
148
+ const appendPhase = (phase, steps) => {
149
+ for (const step of steps ?? []) {
150
+ if (!Q4K_PROJECTION_OPS.has(step?.op)) {
151
+ continue;
111
152
  }
112
- return response.text();
113
- };
114
- const readBinary = async (relativePath) => {
115
- const filename = String(relativePath || '').replace(/^\/+/, '');
116
- if (!filename) {
117
- throw new Error('Direct-source binary asset path is required.');
153
+ const kernel = String(step?.kernel ?? '');
154
+ if (!kernel) {
155
+ continue;
118
156
  }
119
- return fetchBytes(`${root}/${filename}`);
120
- };
121
- return createSourceStorageContext({
122
- manifest,
123
- readRange,
124
- readText,
125
- readBinary,
126
- verifyHashes: true,
127
- });
128
- }
129
-
130
- return {
131
- async loadShard(index) {
132
- const shard = manifest.shards[index];
133
- const filename = shard?.filename;
134
- if (!filename) {
135
- throw new Error(`Manifest shard ${index} is missing filename.`);
157
+ const descriptor = `${phase}.${step.op}:${kernel}#${String(step?.entry ?? 'main')}`;
158
+ if (kernel.startsWith('fused_matmul_q4')) {
159
+ summary.fusedProjectionKernels.push(descriptor);
160
+ } else if (kernel.startsWith('matmul_')) {
161
+ summary.denseProjectionKernels.push(descriptor);
136
162
  }
137
- return fetchBytes(`${root}/${filename.replace(/^\/+/, '')}`);
138
- },
163
+ }
139
164
  };
140
- }
141
165
 
166
+ appendPhase('decode', kernelPath.decode?.steps);
167
+ appendPhase('prefill', kernelPath.prefill?.steps);
168
+ return summary;
169
+ }
142
170
 
143
- /** Allowed quantizationInfo.layout values for Q4K models. */
144
- const Q4K_LAYOUT_ALLOWLIST = new Set(['row', 'col']);
171
+ function resolveQ4KProjectionMaterializationMode(
172
+ manifest,
173
+ kernelPath,
174
+ kernelPathSource = 'none'
175
+ ) {
176
+ const summary = summarizeQ4KProjectionKernelKinds(kernelPath);
177
+ const mode = selectKernelRuleValue('matmul', 'q4kMaterializationMode', {
178
+ hasFusedProjections: summary.fusedProjectionKernels.length > 0,
179
+ hasDenseProjections: summary.denseProjectionKernels.length > 0,
180
+ });
181
+ debugTrace.loader(
182
+ `Q4K materialization: model=${manifest?.modelId ?? 'unknown'}, mode=${mode}, ` +
183
+ `source=${kernelPathSource}, dense=${summary.denseProjectionKernels.length}, ` +
184
+ `fused=${summary.fusedProjectionKernels.length}`
185
+ );
186
+ return mode;
187
+ }
145
188
 
146
189
  export function resolveQ4KConfig(
147
190
  manifest,
@@ -165,7 +208,19 @@ export function resolveQ4KConfig(
165
208
  `Allowed values: ${[...Q4K_LAYOUT_ALLOWLIST].join(', ')}.`
166
209
  );
167
210
  }
168
- let useFused = kernelPath ? isKernelPathFusedQ4K(kernelPath) : hasSubgroups;
211
+ const q4kMaterializationMode = isQ4KModel
212
+ ? resolveQ4KProjectionMaterializationMode(manifest, kernelPath, kernelPathSource)
213
+ : 'dense';
214
+ if (isQ4KModel) {
215
+ debugTrace.loader(
216
+ `Q4K projection materialization: model=${manifest?.modelId ?? 'unknown'}, ` +
217
+ `mode=${q4kMaterializationMode}, source=${kernelPathSource}`
218
+ );
219
+ }
220
+ const hasExplicitKernelPath = kernelPath != null;
221
+ let useFused = hasExplicitKernelPath
222
+ ? q4kMaterializationMode !== 'dense'
223
+ : hasSubgroups;
169
224
  const kernelPathKeepsF32Weights = kernelPathRequiresF32MatmulWeights(kernelPath);
170
225
  if (q4kLayout === 'col') {
171
226
  useFused = false;
@@ -176,13 +231,15 @@ export function resolveQ4KConfig(
176
231
  const layoutLabel = q4kLayout ?? 'none';
177
232
  debugTrace.loader(
178
233
  `Q4K config: fused=${useFused}, kernelPath=${pathLabel}, source=${kernelPathSource}, ` +
179
- `layout=${layoutLabel}, keepF32Weights=${resolvedKeepF32Weights}, subgroups=${hasSubgroups}`
234
+ `layout=${layoutLabel}, materialization=${q4kMaterializationMode}, ` +
235
+ `keepF32Weights=${resolvedKeepF32Weights}, subgroups=${hasSubgroups}`
180
236
  );
181
237
 
182
238
  return {
183
239
  useFusedQ4K: useFused,
184
240
  q4kLayout,
185
241
  keepF32Weights: resolvedKeepF32Weights,
242
+ q4kMaterializationMode,
186
243
  };
187
244
  }
188
245
 
@@ -191,13 +248,21 @@ export function resolveQ4KConfig(
191
248
  // ============================================================================
192
249
 
193
250
 
194
- function computeRoPEFreqsForTheta(theta, headDim, maxSeqLen, ropeScale, ropeScalingType, ropeScaling) {
195
- const halfDim = headDim / 2;
251
+ function computeRoPEFreqsForTheta(
252
+ theta,
253
+ rotaryDim,
254
+ frequencyBaseDim,
255
+ maxSeqLen,
256
+ ropeScale,
257
+ ropeScalingType,
258
+ ropeScaling
259
+ ) {
260
+ const halfDim = rotaryDim / 2;
196
261
 
197
262
  // Compute base frequencies: theta_i = 1 / (base^(2i/d))
198
263
  const freqs = new Float32Array(halfDim);
199
264
  for (let i = 0; i < halfDim; i++) {
200
- freqs[i] = 1.0 / Math.pow(theta, (2 * i) / headDim);
265
+ freqs[i] = 1.0 / Math.pow(theta, (2 * i) / frequencyBaseDim);
201
266
  }
202
267
 
203
268
  // Compute per-dimension scaling factors
@@ -283,6 +348,33 @@ function isSameRoPEScalingConfig(
283
348
  === (rightScaling?.original_max_position_embeddings ?? null);
284
349
  }
285
350
 
351
+ const GPU_ROPE_BUFFER_CACHE = new WeakMap();
352
+ const CPU_ROPE_BUFFER_CACHE = new Map();
353
+
354
+ function buildRoPECacheKey(config) {
355
+ return JSON.stringify({
356
+ headDim: config.headDim,
357
+ localHeadDim: config.localHeadDim ?? null,
358
+ rotaryDim: config.rotaryDim ?? null,
359
+ ropeLocalRotaryDim: config.ropeLocalRotaryDim ?? null,
360
+ ropeFrequencyBaseDim: config.ropeFrequencyBaseDim ?? null,
361
+ ropeLocalFrequencyBaseDim: config.ropeLocalFrequencyBaseDim ?? null,
362
+ maxSeqLen: config.maxSeqLen,
363
+ ropeTheta: config.ropeTheta,
364
+ ropeLocalTheta: config.ropeLocalTheta ?? null,
365
+ mropeInterleaved: config.mropeInterleaved === true,
366
+ mropeSection: Array.isArray(config.mropeSection) ? [...config.mropeSection] : null,
367
+ partialRotaryFactor: config.partialRotaryFactor ?? null,
368
+ ropeLocalPartialRotaryFactor: config.ropeLocalPartialRotaryFactor ?? null,
369
+ ropeScale: config.ropeScale,
370
+ ropeLocalScale: config.ropeLocalScale ?? null,
371
+ ropeScalingType: config.ropeScalingType ?? null,
372
+ ropeLocalScalingType: config.ropeLocalScalingType ?? null,
373
+ ropeScaling: config.ropeScaling ?? null,
374
+ ropeLocalScaling: config.ropeLocalScaling ?? null,
375
+ });
376
+ }
377
+
286
378
  function resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor) {
287
379
  if (rotaryDim != null) {
288
380
  if (!Number.isFinite(rotaryDim) || rotaryDim <= 0 || (rotaryDim % 2) !== 0) {
@@ -311,17 +403,37 @@ function resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor) {
311
403
  return resolved;
312
404
  }
313
405
 
406
+ function resolveFrequencyBaseDim(rotaryDim, frequencyBaseDim, label) {
407
+ if (frequencyBaseDim == null) {
408
+ return rotaryDim;
409
+ }
410
+ if (!Number.isFinite(frequencyBaseDim) || frequencyBaseDim <= 0 || (Math.trunc(frequencyBaseDim) % 2) !== 0) {
411
+ throw new Error(`${label} must be a positive even integer; got "${frequencyBaseDim}".`);
412
+ }
413
+ const resolved = Math.trunc(frequencyBaseDim);
414
+ if (resolved < rotaryDim) {
415
+ throw new Error(`${label} ${resolved} cannot be smaller than rotaryDim ${rotaryDim}.`);
416
+ }
417
+ return resolved;
418
+ }
419
+
314
420
 
315
421
  export async function initRoPEFrequencies(config, useGPU) {
422
+ const cacheKey = buildRoPECacheKey(config);
316
423
  const {
317
424
  headDim,
425
+ localHeadDim,
318
426
  rotaryDim,
427
+ ropeLocalRotaryDim,
428
+ ropeFrequencyBaseDim,
429
+ ropeLocalFrequencyBaseDim,
319
430
  maxSeqLen,
320
431
  ropeTheta,
321
432
  ropeLocalTheta,
322
433
  mropeInterleaved,
323
434
  mropeSection,
324
435
  partialRotaryFactor,
436
+ ropeLocalPartialRotaryFactor,
325
437
  ropeScale,
326
438
  ropeLocalScale,
327
439
  ropeScalingType,
@@ -347,7 +459,23 @@ export async function initRoPEFrequencies(config, useGPU) {
347
459
  ? ropeScaling
348
460
  : ropeLocalScaling
349
461
  );
462
+ const resolvedLocalHeadDim = localHeadDim ?? headDim;
350
463
  const resolvedRotaryDim = resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor);
464
+ const resolvedLocalRotaryDim = resolveRotaryDim(
465
+ resolvedLocalHeadDim,
466
+ ropeLocalRotaryDim,
467
+ ropeLocalPartialRotaryFactor
468
+ );
469
+ const resolvedFrequencyBaseDim = resolveFrequencyBaseDim(
470
+ resolvedRotaryDim,
471
+ ropeFrequencyBaseDim,
472
+ 'RoPE frequency base dim'
473
+ );
474
+ const resolvedLocalFrequencyBaseDim = resolveFrequencyBaseDim(
475
+ resolvedLocalRotaryDim,
476
+ ropeLocalFrequencyBaseDim,
477
+ 'Local RoPE frequency base dim'
478
+ );
351
479
  const halfDim = resolvedRotaryDim / 2;
352
480
  if (mropeInterleaved === true && Array.isArray(mropeSection)) {
353
481
  const expandedDim = mropeSection.reduce((sum, entry) => sum + entry, 0) * 2;
@@ -363,7 +491,13 @@ export async function initRoPEFrequencies(config, useGPU) {
363
491
 
364
492
  // Compute global (full_attention) frequencies
365
493
  const globalFreqs = computeRoPEFreqsForTheta(
366
- ropeTheta, resolvedRotaryDim, maxSeqLen, ropeScale, ropeScalingType, ropeScaling
494
+ ropeTheta,
495
+ resolvedRotaryDim,
496
+ resolvedFrequencyBaseDim,
497
+ maxSeqLen,
498
+ ropeScale,
499
+ ropeScalingType,
500
+ ropeScaling
367
501
  );
368
502
 
369
503
  // Compute local (sliding_attention) frequencies if different from global.
@@ -371,6 +505,7 @@ export async function initRoPEFrequencies(config, useGPU) {
371
505
 
372
506
  let localFreqs = null;
373
507
  const hasDistinctLocalTheta = resolvedLocalTheta !== ropeTheta;
508
+ const hasDistinctLocalDim = resolvedLocalRotaryDim !== resolvedRotaryDim;
374
509
  const hasDistinctLocalScaling = !isSameRoPEScalingConfig(
375
510
  ropeScalingType,
376
511
  ropeScale,
@@ -379,10 +514,11 @@ export async function initRoPEFrequencies(config, useGPU) {
379
514
  resolvedLocalScale,
380
515
  resolvedLocalScaling
381
516
  );
382
- if (hasDistinctLocalTheta || hasDistinctLocalScaling) {
517
+ if (hasDistinctLocalTheta || hasDistinctLocalScaling || hasDistinctLocalDim) {
383
518
  localFreqs = computeRoPEFreqsForTheta(
384
519
  resolvedLocalTheta,
385
- resolvedRotaryDim,
520
+ resolvedLocalRotaryDim,
521
+ resolvedLocalFrequencyBaseDim,
386
522
  maxSeqLen,
387
523
  resolvedLocalScale,
388
524
  resolvedLocalScalingType,
@@ -391,6 +527,8 @@ export async function initRoPEFrequencies(config, useGPU) {
391
527
  log.debug(
392
528
  'Pipeline',
393
529
  `Dual RoPE: local theta=${resolvedLocalTheta}, global theta=${ropeTheta}, ` +
530
+ `localRotaryDim=${resolvedLocalRotaryDim}, globalRotaryDim=${resolvedRotaryDim}, ` +
531
+ `localFrequencyBaseDim=${resolvedLocalFrequencyBaseDim}, globalFrequencyBaseDim=${resolvedFrequencyBaseDim}, ` +
394
532
  `localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}, ` +
395
533
  `globalScaling=${ropeScalingType ?? 'none'}:${ropeScale}`
396
534
  );
@@ -411,6 +549,15 @@ export async function initRoPEFrequencies(config, useGPU) {
411
549
  // Upload to GPU if available
412
550
  const device = getDevice();
413
551
  if (device && useGPU) {
552
+ let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
553
+ if (!perDeviceCache) {
554
+ perDeviceCache = new Map();
555
+ GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
556
+ }
557
+ const cachedBuffers = perDeviceCache.get(cacheKey);
558
+ if (cachedBuffers) {
559
+ return cachedBuffers;
560
+ }
414
561
  let cosBuffer = null;
415
562
  let sinBuffer = null;
416
563
  let localCosBuffer = null;
@@ -440,38 +587,49 @@ export async function initRoPEFrequencies(config, useGPU) {
440
587
  'Pipeline',
441
588
  `RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
442
589
  `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
590
+ `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
443
591
  `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
444
592
  `interleaved=${mropeInterleaved === true}`
445
593
  );
446
594
 
447
- return {
595
+ const buffers = {
448
596
  cos: cosBuffer,
449
597
  sin: sinBuffer,
450
598
  localCos: localCosBuffer,
451
599
  localSin: localSinBuffer,
452
600
  };
601
+ perDeviceCache.set(cacheKey, buffers);
602
+ return buffers;
603
+ }
604
+
605
+ const cachedCpuBuffers = CPU_ROPE_BUFFER_CACHE.get(cacheKey);
606
+ if (cachedCpuBuffers) {
607
+ return cachedCpuBuffers;
453
608
  }
454
609
 
455
610
  log.debug(
456
611
  'Pipeline',
457
612
  `RoPE frequencies initialized (CPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
458
613
  `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
614
+ `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
459
615
  `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
460
616
  `interleaved=${mropeInterleaved === true}`
461
617
  );
462
618
 
463
- return {
619
+ const buffers = {
464
620
  cos: globalFreqs.cos,
465
621
  sin: globalFreqs.sin,
466
622
  localCos: localFreqs?.cos,
467
623
  localSin: localFreqs?.sin,
468
624
  };
625
+ CPU_ROPE_BUFFER_CACHE.set(cacheKey, buffers);
626
+ return buffers;
469
627
  }
470
628
 
471
629
 
472
630
  export function isGPURoPEBuffers(buffers) {
473
631
  if (typeof GPUBuffer === 'undefined') return false;
474
- return !!buffers?.cos && buffers.cos instanceof GPUBuffer;
632
+ return !!buffers?.cos && isGpuBufferInstance(buffers.cos);
475
633
  }
476
634
 
477
635
 
@@ -494,6 +652,128 @@ function hasFullAttentionLayers(layerTypes) {
494
652
  return layerTypes.some((layerType) => !isSlidingLayerType(layerType));
495
653
  }
496
654
 
655
+ function resolveContiguousKVPolicy(modelConfig) {
656
+ if (hasFullAttentionLayers(modelConfig.layerTypes)) {
657
+ return {
658
+ forceContiguousKVCache: true,
659
+ reason: 'layerPattern',
660
+ };
661
+ }
662
+ if (modelConfig.layerTypes == null && modelConfig.slidingWindow == null) {
663
+ return {
664
+ forceContiguousKVCache: true,
665
+ reason: 'slidingWindow',
666
+ };
667
+ }
668
+ return {
669
+ forceContiguousKVCache: false,
670
+ reason: 'none',
671
+ };
672
+ }
673
+
674
+ function usesMixedGeometryKVCache(modelConfig) {
675
+ if (!modelConfig || typeof modelConfig !== 'object') {
676
+ return false;
677
+ }
678
+ const hasMixedAttentionGeometry = Number.isFinite(modelConfig.globalHeadDim)
679
+ && modelConfig.globalHeadDim > 0
680
+ && modelConfig.globalHeadDim !== modelConfig.headDim;
681
+ const hasSharedKvLayers = Number.isFinite(modelConfig.numKvSharedLayers)
682
+ && modelConfig.numKvSharedLayers > 0;
683
+ const hasExplicitLayerTypes = Array.isArray(modelConfig.layerTypes)
684
+ && modelConfig.layerTypes.length === modelConfig.numLayers;
685
+ return hasExplicitLayerTypes && (hasMixedAttentionGeometry || hasSharedKvLayers);
686
+ }
687
+
688
+ function getRequiredKernelMaxKVLen(operation, variant, label) {
689
+ const config = getKernelConfig(operation, variant);
690
+ const maxKVLen = config.variantMetadata?.maxKVLen;
691
+ if (!Number.isFinite(maxKVLen) || maxKVLen <= 0) {
692
+ throw new Error(`${label} kernel "${variant}" is missing variantMetadata.maxKVLen.`);
693
+ }
694
+ return maxKVLen;
695
+ }
696
+
697
+ function resolveTieredRequestedQuantMode(runtimeKV) {
698
+ const tieringMode = String(runtimeKV?.tiering?.mode ?? 'off').trim().toLowerCase();
699
+ assertSupportedTurboQuantMode(tieringMode, 'runtime.inference.session.kvcache.tiering.mode');
700
+ const compressionMode = String(
701
+ runtimeKV?.tiering?.compression?.mode
702
+ ?? (tieringMode === 'int8'
703
+ || tieringMode === 'int4'
704
+ || tieringMode === 'turboquant'
705
+ || tieringMode === 'turboquant_prod'
706
+ ? tieringMode
707
+ : 'none')
708
+ ).trim().toLowerCase();
709
+ assertSupportedTurboQuantMode(
710
+ compressionMode,
711
+ 'runtime.inference.session.kvcache.tiering.compression.mode'
712
+ );
713
+ const gatingMode = String(runtimeKV?.tiering?.gating?.mode ?? 'auto').trim().toLowerCase();
714
+ if (gatingMode === 'force_off') {
715
+ return 'none';
716
+ }
717
+ return compressionMode;
718
+ }
719
+
720
+ function resolveContiguousRequestedQuantMode(runtimeKV) {
721
+ const quantMode = String(runtimeKV?.quantization?.mode ?? 'none').trim().toLowerCase();
722
+ assertSupportedTurboQuantMode(
723
+ quantMode,
724
+ 'runtime.inference.session.kvcache.quantization.mode'
725
+ );
726
+ return quantMode;
727
+ }
728
+
729
+ function assertQuantizedKVKernelSupport(modelConfig, cacheLayout, cacheMaxSeqLen, runtimeKV) {
730
+ if (cacheLayout === 'contiguous_quantized') {
731
+ if (modelConfig.headDim > 256) {
732
+ throw new Error(
733
+ `Contiguous quantized KV cache requires headDim <= 256; got ${modelConfig.headDim}.`
734
+ );
735
+ }
736
+ const quantMode = resolveContiguousRequestedQuantMode(runtimeKV);
737
+ const variant = selectKernelRuleValue('attention', 'contiguousQuantVariant', { mode: quantMode });
738
+ const maxKVLen = getRequiredKernelMaxKVLen(
739
+ 'attention_contiguous_quant',
740
+ variant,
741
+ 'Contiguous quant attention'
742
+ );
743
+ if (cacheMaxSeqLen > maxKVLen) {
744
+ throw new Error(
745
+ `Contiguous quantized KV cache requires maxSeqLen <= ${maxKVLen}; got ${cacheMaxSeqLen}.`
746
+ );
747
+ }
748
+ return;
749
+ }
750
+
751
+ if (cacheLayout !== 'tiered') {
752
+ return;
753
+ }
754
+
755
+ const coldQuantMode = resolveTieredRequestedQuantMode(runtimeKV);
756
+ if (coldQuantMode === 'none') {
757
+ return;
758
+ }
759
+ if (modelConfig.headDim > 256) {
760
+ throw new Error(
761
+ `Tiered quantized KV cache requires headDim <= 256; got ${modelConfig.headDim}.`
762
+ );
763
+ }
764
+ const variant = selectKernelRuleValue('attention', 'tieredQuantVariant', { mode: coldQuantMode });
765
+ const maxKVLen = getRequiredKernelMaxKVLen(
766
+ 'attention_tiered_quant',
767
+ variant,
768
+ 'Tiered quant attention'
769
+ );
770
+ if (cacheMaxSeqLen > maxKVLen) {
771
+ throw new Error(
772
+ `Tiered quantized KV cache requires maxSeqLen <= ${maxKVLen}; got ${cacheMaxSeqLen}.`
773
+ );
774
+ }
775
+ }
776
+
497
777
 
498
778
  // ============================================================================
499
779
  // KV Cache Setup
@@ -501,8 +781,16 @@ function hasFullAttentionLayers(layerTypes) {
501
781
 
502
782
 
503
783
  export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig) {
504
- const runtimeKV = runtimeConfig ?? getRuntimeConfig().inference.kvcache;
505
- const forceContiguousKVCache = hasFullAttentionLayers(modelConfig.layerTypes);
784
+ if (modelConfig?.decodeStrategy === 'replay_prefill') {
785
+ throw new Error(
786
+ 'Live KV cache creation is not supported for models that require replay-prefill decode. ' +
787
+ 'Skip createKVCache() when the model config does not resolve explicit layerTypes for mixed-geometry/shared-KV decode.'
788
+ );
789
+ }
790
+ const runtimeKV = resolveRuntimeKVConfig(runtimeConfig);
791
+ const requiresMixedGeometryKVCache = usesMixedGeometryKVCache(modelConfig);
792
+ const contiguousKVPolicy = resolveContiguousKVPolicy(modelConfig);
793
+ const forceContiguousKVCache = contiguousKVPolicy.forceContiguousKVCache;
506
794
  const modelMaxSeqLen = modelConfig.maxSeqLen;
507
795
  if (!Number.isFinite(modelMaxSeqLen) || modelMaxSeqLen <= 0) {
508
796
  throw new Error('Model config is missing maxSeqLen.');
@@ -517,19 +805,19 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
517
805
 
518
806
  let cacheLayout = runtimeKV.layout;
519
807
  if (!cacheLayout) {
520
- throw new Error('runtime.inference.kvcache.layout is required.');
808
+ throw new Error('runtime.inference.session.kvcache.layout is required.');
521
809
  }
522
810
  if (cacheLayout === 'tiered' && !runtimeKV.tiering) {
523
- throw new Error('runtime.inference.kvcache.tiering is required for tiered layout.');
811
+ throw new Error('runtime.inference.session.kvcache.tiering is required for tiered layout.');
524
812
  }
525
813
  const tieringMode = runtimeKV.tiering?.mode;
526
814
  if (tieringMode == null) {
527
- throw new Error('runtime.inference.kvcache.tiering.mode is required.');
815
+ throw new Error('runtime.inference.session.kvcache.tiering.mode is required.');
528
816
  }
529
817
  let layoutSource = 'runtime';
530
818
  if (tieringMode !== 'off' && cacheLayout !== 'tiered') {
531
819
  if (cacheLayout !== 'contiguous') {
532
- throw new Error('runtime.inference.kvcache.layout must be "tiered" when tiering.mode is enabled.');
820
+ throw new Error('runtime.inference.session.kvcache.layout must be "tiered" when tiering.mode is enabled.');
533
821
  }
534
822
  cacheLayout = 'tiered';
535
823
  layoutSource = 'tiering';
@@ -538,12 +826,42 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
538
826
  cacheLayout = 'paged';
539
827
  layoutSource = 'threshold';
540
828
  }
829
+ const quantMode = resolveContiguousRequestedQuantMode(runtimeKV);
830
+ if (forceContiguousKVCache && cacheLayout === 'contiguous' && quantMode !== 'none') {
831
+ cacheLayout = 'contiguous_quantized';
832
+ layoutSource = 'quantization';
833
+ }
541
834
  if (forceContiguousKVCache && cacheLayout === 'paged') {
542
835
  throw new Error(
543
836
  'Paged KV cache layout is not supported for models with full-attention layers. ' +
544
- 'Set runtime.inference.kvcache.layout to "contiguous" instead.'
837
+ 'Set runtime.inference.session.kvcache.layout to "contiguous" instead.'
545
838
  );
546
839
  }
840
+ if (requiresMixedGeometryKVCache) {
841
+ if (cacheLayout !== 'contiguous') {
842
+ throw new Error(
843
+ `Mixed-geometry incremental KV cache requires layout="contiguous"; got "${cacheLayout}". ` +
844
+ 'Disable tiering and quantized KV cache for this model.'
845
+ );
846
+ }
847
+ if (runtimeKV.tiering?.mode != null && runtimeKV.tiering.mode !== 'off') {
848
+ throw new Error(
849
+ `Mixed-geometry incremental KV cache requires tiering.mode="off"; got "${runtimeKV.tiering.mode}".`
850
+ );
851
+ }
852
+ if (resolveContiguousRequestedQuantMode(runtimeKV) !== 'none') {
853
+ throw new Error(
854
+ 'Mixed-geometry incremental KV cache does not support contiguous quantization yet. ' +
855
+ 'Set runtime.inference.session.kvcache.quantization.mode="none".'
856
+ );
857
+ }
858
+ if (!useGPU) {
859
+ throw new Error(
860
+ 'Mixed-geometry incremental KV cache requires GPU execution. ' +
861
+ 'Use a WebGPU-capable surface.'
862
+ );
863
+ }
864
+ }
547
865
  if (debug && cacheLayout !== runtimeKV.layout) {
548
866
  log.debug('Pipeline', `KV cache layout override: ${runtimeKV.layout} -> ${cacheLayout} (${layoutSource})`);
549
867
  }
@@ -581,11 +899,23 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
581
899
  if (cacheLayout === 'tiered' && kvDtype !== 'f16') {
582
900
  throw new Error('Tiered KV cache requires kvDtype="f16" (no f32 tiered kernels yet).');
583
901
  }
902
+ if (cacheLayout === 'contiguous_quantized' && kvDtype !== 'f16') {
903
+ throw new Error('Contiguous quantized KV cache requires kvDtype="f16".');
904
+ }
905
+ if (cacheLayout === 'contiguous_quantized' && !useGPU) {
906
+ throw new Error('Contiguous quantized KV cache requires GPU.');
907
+ }
584
908
 
585
- if (useGPU && (cacheLayout === 'paged' || cacheLayout === 'tiered' || cacheLayout === 'bdpa')) {
909
+ if (useGPU && (cacheLayout === 'paged' || cacheLayout === 'tiered' || cacheLayout === 'bdpa' || cacheLayout === 'contiguous_quantized')) {
586
910
  const limits = getDeviceLimits();
587
911
  if (limits) {
588
- const bytesPerToken = modelConfig.numKVHeads * modelConfig.headDim * (kvDtype === 'f16' ? 2 : 4);
912
+ const maxLayerHeadDim = Math.max(
913
+ modelConfig.headDim,
914
+ Number.isFinite(modelConfig.globalHeadDim) && modelConfig.globalHeadDim > 0
915
+ ? modelConfig.globalHeadDim
916
+ : modelConfig.headDim
917
+ );
918
+ const bytesPerToken = modelConfig.numKVHeads * maxLayerHeadDim * (kvDtype === 'f16' ? 2 : 4);
589
919
  const maxByBinding = Math.floor(limits.maxStorageBufferBindingSize / bytesPerToken);
590
920
  const maxByBuffer = Math.floor(limits.maxBufferSize / bytesPerToken);
591
921
  const fallbackMax = Number.isFinite(runtimeKV.gpuPagedFallbackMaxSeqLen) && runtimeKV.gpuPagedFallbackMaxSeqLen > 0
@@ -605,6 +935,8 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
605
935
  }
606
936
  }
607
937
 
938
+ assertQuantizedKVKernelSupport(modelConfig, cacheLayout, cacheMaxSeqLen, runtimeKV);
939
+
608
940
 
609
941
  const cacheConfig = {
610
942
  numLayers: modelConfig.numLayers,
@@ -621,7 +953,15 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
621
953
 
622
954
  let kvCache;
623
955
 
624
- if (modelConfig.slidingWindow && !forceContiguousKVCache && cacheLayout !== 'paged' && cacheLayout !== 'tiered' && cacheLayout !== 'bdpa') {
956
+ if (requiresMixedGeometryKVCache) {
957
+ kvCache = new MixedGeometryKVCache({
958
+ ...cacheConfig,
959
+ numHeads: modelConfig.numKVHeads,
960
+ globalHeadDim: modelConfig.globalHeadDim ?? null,
961
+ slidingWindow,
962
+ layerTypes: modelConfig.layerTypes,
963
+ });
964
+ } else if (modelConfig.slidingWindow && !forceContiguousKVCache && cacheLayout !== 'paged' && cacheLayout !== 'tiered' && cacheLayout !== 'bdpa') {
625
965
  kvCache = new SlidingWindowKVCache({
626
966
  ...cacheConfig,
627
967
  windowSize: slidingWindow ?? modelConfig.slidingWindow,
@@ -635,6 +975,21 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
635
975
  ...cacheConfig,
636
976
  tiering: runtimeKV.tiering,
637
977
  });
978
+ } else if (cacheLayout === 'contiguous_quantized') {
979
+ const quantCfg = runtimeKV.quantization ?? {};
980
+ const qCache = new QuantizedKVCache({
981
+ ...cacheConfig,
982
+ quantMode,
983
+ bitWidth: quantCfg.bitWidth ?? 4,
984
+ prodMode: quantCfg.prodMode === true,
985
+ });
986
+ const device = getDevice();
987
+ qCache.setSharedBuffers(retainTurboQuantSharedBuffers(device, {
988
+ headDim: modelConfig.headDim,
989
+ bitWidth: quantCfg.bitWidth ?? 4,
990
+ prodMode: quantCfg.prodMode === true,
991
+ }));
992
+ kvCache = qCache;
638
993
  } else {
639
994
  kvCache = new KVCache(cacheConfig);
640
995
  }
@@ -667,8 +1022,10 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
667
1022
  );
668
1023
 
669
1024
  if (debug) {
670
- if (forceContiguousKVCache && modelConfig.layerTypes) {
1025
+ if (contiguousKVPolicy.reason === 'layerPattern') {
671
1026
  log.debug('Pipeline', 'Layer pattern includes full-attention layers; paged layout blocked, contiguous enforced.');
1027
+ } else if (contiguousKVPolicy.reason === 'slidingWindow') {
1028
+ log.debug('Pipeline', 'Model declares attention.slidingWindow=null without explicit layerTypes; treating KV layout as contiguous-compatible.');
672
1029
  }
673
1030
  const isSliding = kvCache instanceof SlidingWindowKVCache;
674
1031
  log.debug('Pipeline', `KV cache: type=${kvCache?.constructor?.name || 'unknown'}, kvDtype=${kvCache.kvDtype}, layout=${kvCache.layout}, maxSeqLen=${kvCache.maxSeqLen}, windowSize=${isSliding ? kvCache.windowSize : null}`);
@@ -709,6 +1066,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
709
1066
  loadingConfig,
710
1067
  baseUrl,
711
1068
  loaderDebug,
1069
+ perLayerInputSession,
712
1070
  } = options;
713
1071
  const runtimeStorageContext = options.storageContext
714
1072
  ?? createRemoteStorageContext(baseUrl, manifest);
@@ -732,6 +1090,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
732
1090
  )
733
1091
  );
734
1092
  dopplerLoader.setLoaderDebugConfig(loaderDebug ?? null);
1093
+ dopplerLoader.setPerLayerInputSession(perLayerInputSession ?? modelConfig.perLayerInputsSession);
735
1094
 
736
1095
  const tensorsFile = isRDRRManifest(manifest) ? manifest.tensorsFile : null;
737
1096
  if (baseUrl && tensorsFile) {
@@ -832,6 +1191,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
832
1191
  lmHead: dopplerLoader.lmHead,
833
1192
  finalNorm: dopplerLoader.finalNorm,
834
1193
  embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
1194
+ perLayerInputWeights: dopplerLoader.perLayerInputWeights,
835
1195
  layerRouterWeights,
836
1196
  };
837
1197
  }
@@ -846,7 +1206,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
846
1206
  function applyTurnBasedTemplate(prompt) {
847
1207
  // Turn-based format: <start_of_turn>role\ncontent<end_of_turn>
848
1208
  const userTurn = `<start_of_turn>user\n${prompt}<end_of_turn>\n`;
849
- const modelTurn = `<start_of_turn>model\n`;
1209
+ const modelTurn = '<start_of_turn>model\n';
850
1210
  return userTurn + modelTurn;
851
1211
  }
852
1212
 
@@ -865,8 +1225,18 @@ function applyChatMLTemplate(prompt) {
865
1225
  return `<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n`;
866
1226
  }
867
1227
 
868
- function applyQwenTemplate(prompt) {
869
- return `<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n`;
1228
+ function getQwenAssistantPrefix(options) {
1229
+ return options?.thinking === true
1230
+ ? '<|im_start|>assistant\n<think>\n'
1231
+ : '<|im_start|>assistant\n<think>\n\n</think>\n\n';
1232
+ }
1233
+
1234
+ function applyQwenTemplate(prompt, options) {
1235
+ return `<|im_start|>user\n${prompt}<|im_end|>\n${getQwenAssistantPrefix(options)}`;
1236
+ }
1237
+
1238
+ function applyGemma4Template(prompt) {
1239
+ return `<bos><|turn>user\n${prompt}<turn|>\n<|turn>model\n`;
870
1240
  }
871
1241
 
872
1242
  function applyTranslateGemmaTemplate() {
@@ -880,6 +1250,7 @@ function applyTranslateGemmaTemplate() {
880
1250
  // Add new template types here rather than adding switch cases.
881
1251
  const PROMPT_TEMPLATES = {
882
1252
  'gemma': applyTurnBasedTemplate,
1253
+ 'gemma4': applyGemma4Template,
883
1254
  'llama3': applyHeaderBasedTemplate,
884
1255
  'gpt-oss': applyChannelBasedTemplate,
885
1256
  'chatml': applyChatMLTemplate,
@@ -887,19 +1258,20 @@ const PROMPT_TEMPLATES = {
887
1258
  'translategemma': applyTranslateGemmaTemplate,
888
1259
  };
889
1260
 
890
- export function applyChatTemplate(prompt, templateType) {
1261
+ export function applyChatTemplate(prompt, templateType, options = undefined) {
891
1262
  if (templateType == null) {
892
1263
  return prompt;
893
1264
  }
894
1265
  const formatter = PROMPT_TEMPLATES[templateType];
895
1266
  if (formatter) {
896
- return formatter(prompt);
1267
+ return formatter(prompt, options);
897
1268
  }
898
1269
  throw new Error(`Unrecognized chat template type: ${templateType}`);
899
1270
  }
900
1271
 
901
1272
  // Exports preserved for existing external imports.
902
1273
  export const applyGemmaChatTemplate = applyTurnBasedTemplate;
1274
+ export const applyGemma4ChatTemplate = applyGemma4Template;
903
1275
  export const applyLlama3ChatTemplate = applyHeaderBasedTemplate;
904
1276
  export const applyGptOssChatTemplate = applyChannelBasedTemplate;
905
1277
  export const applyQwenChatTemplate = applyQwenTemplate;
@@ -974,12 +1346,48 @@ export function initSpeculativeDecoder(manifest, speculativeConfig) {
974
1346
  // ============================================================================
975
1347
 
976
1348
 
977
- export function fuseQKVWeights(layerWeights, modelConfig) {
1349
+ function kernelPathVariantRequiresQ4KWeights(variant) {
1350
+ if (typeof variant !== 'string' || variant.length === 0) {
1351
+ return false;
1352
+ }
1353
+ let config;
1354
+ try {
1355
+ config = getKernelConfig('matmul', variant);
1356
+ } catch {
1357
+ return false;
1358
+ }
1359
+ const shaderFile = String(config?.shaderFile ?? config?.wgsl ?? '');
1360
+ return shaderFile.startsWith('fused_matmul_q4');
1361
+ }
1362
+
1363
+ function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
1364
+ if (!kernelPath) {
1365
+ return false;
1366
+ }
1367
+ const prefillVariant = getKernelPathMatmulVariant('qkv_proj', 'prefill', layerIdx, kernelPath);
1368
+ if (kernelPathVariantRequiresQ4KWeights(prefillVariant)) {
1369
+ return true;
1370
+ }
1371
+ const decodeVariant = getKernelPathMatmulVariant('qkv_proj', 'decode', layerIdx, kernelPath);
1372
+ return kernelPathVariantRequiresQ4KWeights(decodeVariant);
1373
+ }
1374
+
1375
+ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
978
1376
  const device = getDevice();
979
1377
  if (!device) {
980
1378
  log.debug('QKV Fusion', 'No GPU device, skipping fusion');
981
1379
  return;
982
1380
  }
1381
+ if (
1382
+ Number.isFinite(modelConfig?.globalHeadDim)
1383
+ && modelConfig.globalHeadDim !== modelConfig.headDim
1384
+ ) {
1385
+ log.debug(
1386
+ 'QKV Fusion',
1387
+ 'Skipping QKV fusion for mixed-head-dim model; per-layer attention geometry must stay explicit.'
1388
+ );
1389
+ return;
1390
+ }
983
1391
 
984
1392
  const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
985
1393
  const qSize = numHeads * headDim;
@@ -996,7 +1404,7 @@ export function fuseQKVWeights(layerWeights, modelConfig) {
996
1404
  shape: Array.isArray(value.shape) ? value.shape : null,
997
1405
  };
998
1406
  }
999
- if (value instanceof GPUBuffer) {
1407
+ if (isGpuBufferInstance(value)) {
1000
1408
  return {
1001
1409
  buffer: value,
1002
1410
  dtype: getWeightDtype(value),
@@ -1029,6 +1437,13 @@ export function fuseQKVWeights(layerWeights, modelConfig) {
1029
1437
 
1030
1438
  // Skip if already fused or if weights are not GPUBuffers
1031
1439
  if (weights.qkvProj) continue;
1440
+ if (shouldSkipQKVFusionForLayer(l, kernelPath)) {
1441
+ log.debug(
1442
+ 'QKV Fusion',
1443
+ `Layer ${l}: skipped because active kernel path requires Q4K weights for qkv_proj`
1444
+ );
1445
+ continue;
1446
+ }
1032
1447
  const qProj = resolveWeight(weights.qProj);
1033
1448
  const kProj = resolveWeight(weights.kProj);
1034
1449
  const vProj = resolveWeight(weights.vProj);