doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,8 +1,6 @@
1
1
 
2
2
 
3
- import { parseModelConfig } from './config.js';
4
3
  import { getDevice, getDeviceLimits, getKernelCapabilities } from '../../../gpu/device.js';
5
- import { acquireBuffer, isBufferActive, releaseBuffer } from '../../../memory/buffer-pool.js';
6
4
  import {
7
5
  KVCache,
8
6
  SlidingWindowKVCache,
@@ -20,14 +18,13 @@ import { Tokenizer } from '../../tokenizer.js';
20
18
  import { MoERouter } from '../../moe-router.js';
21
19
  import { SpeculativeDecoder } from '../../speculative.js';
22
20
  import { getDopplerLoader } from '../../../loader/doppler-loader.js';
23
- import { log, setGPUDevice, trace as debugTrace } from '../../../debug/index.js';
21
+ import { log, trace as debugTrace } from '../../../debug/index.js';
24
22
  import { getRuntimeConfig } from '../../../config/runtime.js';
25
23
  import { PAGED_LAYOUT_SEQ_LEN_THRESHOLD } from '../../../config/schema/index.js';
26
24
  import {
27
- getKernelPathMatmulVariant,
28
25
  kernelPathRequiresF32MatmulWeights,
29
26
  } from '../../../config/kernel-path-loader.js';
30
- import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
27
+ import { resolveCapabilityTransforms } from '../../../config/transforms/capability-transform-resolver.js';
31
28
  import { selectRuleValue } from '../../../rules/rule-registry.js';
32
29
  import { resolvePerLayerInputsSession } from './generator-helpers.js';
33
30
  import {
@@ -35,6 +32,34 @@ import {
35
32
  createNodeFileArtifactStorageContext,
36
33
  } from '../../../storage/artifact-storage-context.js';
37
34
 
35
+ // Extracted standalone modules
36
+ import { initRoPEFrequencies, isGPURoPEBuffers } from './init-rope.js';
37
+ import {
38
+ applyChatTemplate,
39
+ applyGemmaChatTemplate,
40
+ applyGemma4ChatTemplate,
41
+ applyLlama3ChatTemplate,
42
+ applyGptOssChatTemplate,
43
+ applyQwenChatTemplate,
44
+ isStopToken,
45
+ } from './init-chat-templates.js';
46
+ import { fuseQKVWeights } from './init-qkv-fusion.js';
47
+ import { rewriteWeightLoadError } from './load-errors.js';
48
+ import { toArrayBuffer } from '../../../utils/array-buffer.js';
49
+
50
+ // Re-exports for backwards compatibility
51
+ export { initRoPEFrequencies, isGPURoPEBuffers } from './init-rope.js';
52
+ export {
53
+ applyChatTemplate,
54
+ applyGemmaChatTemplate,
55
+ applyGemma4ChatTemplate,
56
+ applyLlama3ChatTemplate,
57
+ applyGptOssChatTemplate,
58
+ applyQwenChatTemplate,
59
+ isStopToken,
60
+ } from './init-chat-templates.js';
61
+ export { fuseQKVWeights } from './init-qkv-fusion.js';
62
+
38
63
  function resolveErrorMessage(error) {
39
64
  if (error && typeof error === 'object' && typeof error.message === 'string') {
40
65
  return error.message;
@@ -87,16 +112,6 @@ function assertSupportedTurboQuantMode(mode, label) {
87
112
  );
88
113
  }
89
114
 
90
- function toArrayBuffer(value, label) {
91
- if (value instanceof ArrayBuffer) {
92
- return value;
93
- }
94
- if (value instanceof Uint8Array) {
95
- return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
96
- }
97
- throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
98
- }
99
-
100
115
  function toUint8Array(value, label) {
101
116
  if (value instanceof Uint8Array) {
102
117
  return value;
@@ -107,6 +122,25 @@ function toUint8Array(value, label) {
107
122
  throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
108
123
  }
109
124
 
125
+ function createStorageContextLoadError(error, operation, index, context = {}) {
126
+ const message = error?.message || String(error);
127
+ const wrapped = new Error(
128
+ `storageContext.${operation} failed for shard ${index}: ${message}`,
129
+ error instanceof Error ? { cause: error } : undefined
130
+ );
131
+ wrapped.name = error?.name || 'Error';
132
+ if (error?.code !== undefined) {
133
+ wrapped.code = error.code;
134
+ }
135
+ wrapped.details = {
136
+ ...(error?.details && typeof error.details === 'object' ? error.details : {}),
137
+ storageOperation: operation,
138
+ shardIndex: index,
139
+ ...context,
140
+ };
141
+ return wrapped;
142
+ }
143
+
110
144
  function isRDRRManifest(manifest) {
111
145
  return manifest !== null && typeof manifest === 'object' && Array.isArray( (manifest).shards);
112
146
  }
@@ -137,6 +171,23 @@ const Q4K_PROJECTION_OPS = new Set([
137
171
  'linear_out_proj',
138
172
  ]);
139
173
 
174
+ const Q4K_MATMUL_ROLE_OPS = new Map([
175
+ ['q_proj', 'q_proj'],
176
+ ['k_proj', 'k_proj'],
177
+ ['v_proj', 'v_proj'],
178
+ ['o_proj', 'o_proj'],
179
+ ['gate_proj', 'ffn_gate'],
180
+ ['up_proj', 'ffn_up'],
181
+ ['down_proj', 'ffn_down'],
182
+ ['qkv_proj', 'qkv_proj'],
183
+ ['linear_qkv_proj', 'linear_qkv_proj'],
184
+ ['linear_z_proj', 'linear_z_proj'],
185
+ ['linear_a_proj', 'linear_a_proj'],
186
+ ['linear_b_proj', 'linear_b_proj'],
187
+ ['linear_out_proj', 'linear_out_proj'],
188
+ ['lm_head', 'lm_head'],
189
+ ]);
190
+
140
191
  function summarizeQ4KProjectionKernelKinds(kernelPath) {
141
192
  const summary = {
142
193
  denseProjectionKernels: [],
@@ -148,14 +199,15 @@ function summarizeQ4KProjectionKernelKinds(kernelPath) {
148
199
 
149
200
  const appendPhase = (phase, steps) => {
150
201
  for (const step of steps ?? []) {
151
- if (!Q4K_PROJECTION_OPS.has(step?.op)) {
202
+ const op = step?.op;
203
+ if (!Q4K_PROJECTION_OPS.has(op)) {
152
204
  continue;
153
205
  }
154
206
  const kernel = String(step?.kernel ?? '');
155
207
  if (!kernel) {
156
208
  continue;
157
209
  }
158
- const descriptor = `${phase}.${step.op}:${kernel}#${String(step?.entry ?? 'main')}`;
210
+ const descriptor = `${phase}.${op}:${kernel}#${String(step?.entry ?? 'main')}`;
159
211
  if (kernel.startsWith('fused_matmul_q4')) {
160
212
  summary.fusedProjectionKernels.push(descriptor);
161
213
  } else if (kernel.startsWith('matmul_')) {
@@ -169,6 +221,31 @@ function summarizeQ4KProjectionKernelKinds(kernelPath) {
169
221
  return summary;
170
222
  }
171
223
 
224
+ function collectQ4KFusedRoles(kernelPath) {
225
+ const roles = new Set();
226
+ if (!kernelPath || typeof kernelPath !== 'object') {
227
+ return [];
228
+ }
229
+
230
+ const appendSteps = (steps) => {
231
+ for (const step of steps ?? []) {
232
+ const role = Q4K_MATMUL_ROLE_OPS.get(step?.op);
233
+ if (!role) {
234
+ continue;
235
+ }
236
+ const kernel = String(step?.kernel ?? '');
237
+ if (kernel.startsWith('fused_matmul_q4')) {
238
+ roles.add(role);
239
+ }
240
+ }
241
+ };
242
+
243
+ appendSteps(kernelPath.decode?.steps);
244
+ appendSteps(kernelPath.prefill?.steps);
245
+ appendSteps(kernelPath.postLayer);
246
+ return [...roles].sort();
247
+ }
248
+
172
249
  function resolveQ4KProjectionMaterializationMode(
173
250
  manifest,
174
251
  kernelPath,
@@ -187,6 +264,25 @@ function resolveQ4KProjectionMaterializationMode(
187
264
  return mode;
188
265
  }
189
266
 
267
+ function isRetainQ4KMaterializationDisabledByCapability(manifest, caps) {
268
+ if (!caps) return false;
269
+ const adapterInfo = caps.adapterInfo ?? {};
270
+ const platform = {
271
+ id: adapterInfo.device ?? 'unknown',
272
+ vendor: adapterInfo.vendor ?? 'unknown',
273
+ architecture: adapterInfo.architecture ?? 'unknown',
274
+ };
275
+ const runtimeSession = getRuntimeConfig().inference?.session ?? {};
276
+ const graphContext = {
277
+ modelId: manifest?.modelId ?? 'unknown',
278
+ activationDtype: runtimeSession.compute?.defaults?.activationDtype ?? null,
279
+ kvDtype: runtimeSession.kvcache?.kvDtype ?? null,
280
+ retainQ4KMaterialization: true,
281
+ };
282
+ const resolved = resolveCapabilityTransforms(caps, platform, graphContext);
283
+ return resolved.names.includes('disableRetainQ4KMaterialization');
284
+ }
285
+
190
286
  export function resolveQ4KConfig(
191
287
  manifest,
192
288
  kernelPath,
@@ -198,6 +294,7 @@ export function resolveQ4KConfig(
198
294
  // Layout in quantizationInfo: 'row' (fused) or 'col' (dequant)
199
295
  const q4kLayout = manifest?.quantizationInfo?.layout ?? null;
200
296
  const isQ4KModel = manifest?.quantization === 'Q4_K_M';
297
+ const q4kFusedRoles = isQ4KModel ? collectQ4KFusedRoles(kernelPath) : [];
201
298
  if (isQ4KModel && q4kLayout == null) {
202
299
  throw new Error(
203
300
  `Manifest "${manifest?.modelId ?? 'unknown'}" is missing quantizationInfo.layout for Q4_K_M. Re-convert the model.`
@@ -209,9 +306,33 @@ export function resolveQ4KConfig(
209
306
  `Allowed values: ${[...Q4K_LAYOUT_ALLOWLIST].join(', ')}.`
210
307
  );
211
308
  }
212
- const q4kMaterializationMode = isQ4KModel
309
+ let q4kMaterializationMode = isQ4KModel
213
310
  ? resolveQ4KProjectionMaterializationMode(manifest, kernelPath, kernelPathSource)
214
311
  : 'dense';
312
+ // Runtime override: retainQ4KMaterialization forces "mixed" mode so the
313
+ // loader keeps the Q4_K packed buffer alongside the dequantized dense
314
+ // buffer. Unlocks `hasQ4KMaterialization=true` in the FFN fusion rule when
315
+ // the execution graph doesn't declare a fused projection kernel.
316
+ if (isQ4KModel && hasSubgroups && q4kMaterializationMode === 'dense') {
317
+ // Runtime-over-manifest precedence per config-style-guide §Category Rules:
318
+ // getRuntimeConfig() returns the merged session (manifest base + runtime
319
+ // overrides via merge.js); consumers should not re-implement precedence.
320
+ const runtimeRetain = getRuntimeConfig().inference?.session?.retainQ4KMaterialization === true;
321
+ if (runtimeRetain) {
322
+ if (isRetainQ4KMaterializationDisabledByCapability(manifest, caps)) {
323
+ debugTrace.loader(
324
+ 'Q4K materialization retain request disabled by capability rule: ' +
325
+ `model=${manifest?.modelId ?? 'unknown'}, adapter=${caps.adapterInfo?.vendor ?? 'unknown'}/` +
326
+ `${caps.adapterInfo?.architecture ?? 'unknown'}, mode=${q4kMaterializationMode}`
327
+ );
328
+ } else {
329
+ q4kMaterializationMode = 'mixed';
330
+ debugTrace.loader(
331
+ `Q4K materialization overridden by runtime flag retainQ4KMaterialization=true: mode=mixed`
332
+ );
333
+ }
334
+ }
335
+ }
215
336
  if (isQ4KModel) {
216
337
  debugTrace.loader(
217
338
  `Q4K projection materialization: model=${manifest?.modelId ?? 'unknown'}, ` +
@@ -241,414 +362,10 @@ export function resolveQ4KConfig(
241
362
  q4kLayout,
242
363
  keepF32Weights: resolvedKeepF32Weights,
243
364
  q4kMaterializationMode,
365
+ q4kFusedRoles,
244
366
  };
245
367
  }
246
368
 
247
- // ============================================================================
248
- // RoPE Initialization
249
- // ============================================================================
250
-
251
-
252
- function computeRoPEFreqsForTheta(
253
- theta,
254
- rotaryDim,
255
- frequencyBaseDim,
256
- maxSeqLen,
257
- ropeScale,
258
- ropeScalingType,
259
- ropeScaling
260
- ) {
261
- const halfDim = rotaryDim / 2;
262
-
263
- // Compute base frequencies: theta_i = 1 / (base^(2i/d))
264
- const freqs = new Float32Array(halfDim);
265
- for (let i = 0; i < halfDim; i++) {
266
- freqs[i] = 1.0 / Math.pow(theta, (2 * i) / frequencyBaseDim);
267
- }
268
-
269
- // Compute per-dimension scaling factors
270
- const scales = new Float32Array(halfDim);
271
- const isYarn = ropeScalingType === 'yarn';
272
- if (isYarn) {
273
- // YARN scaling - validate ALL required params (fail fast on incomplete manifest)
274
- if (ropeScaling?.factor == null ||
275
- ropeScaling?.beta_fast == null || ropeScaling?.beta_slow == null ||
276
- ropeScaling?.original_max_position_embeddings == null) {
277
- throw new Error(
278
- `RoPE scaling type is 'yarn' but YARN params missing. ` +
279
- `Manifest must provide factor, beta_fast, beta_slow, and original_max_position_embeddings. ` +
280
- `Got: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}, ` +
281
- `original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
282
- );
283
- }
284
- // Extract validated YARN params (no hidden defaults - all guaranteed non-null)
285
- const yarnFactor = ropeScaling.factor;
286
- const yarnBetaFast = ropeScaling.beta_fast;
287
- const yarnBetaSlow = ropeScaling.beta_slow;
288
- const originalMaxPos = ropeScaling.original_max_position_embeddings;
289
-
290
- // YARN: wavelength-based interpolation
291
- for (let i = 0; i < halfDim; i++) {
292
- const wavelength = (2 * Math.PI) / freqs[i];
293
- const lowThresh = originalMaxPos / yarnBetaSlow;
294
- const highThresh = originalMaxPos / yarnBetaFast;
295
-
296
- if (wavelength < highThresh) {
297
- scales[i] = 1.0;
298
- } else if (wavelength > lowThresh) {
299
- scales[i] = yarnFactor;
300
- } else {
301
- const t = (wavelength - highThresh) / (lowThresh - highThresh);
302
- scales[i] = 1.0 + (yarnFactor - 1.0) * t;
303
- }
304
- }
305
- } else {
306
- // Linear scaling: uniform across all dimensions
307
- if (ropeScalingType != null && ropeScalingType !== 'linear') {
308
- log.warn(
309
- 'Pipeline',
310
- `Unrecognized RoPE scaling type "${ropeScalingType}"; falling back to linear scaling. ` +
311
- 'Known types: "linear", "yarn".'
312
- );
313
- }
314
- for (let i = 0; i < halfDim; i++) {
315
- scales[i] = ropeScale;
316
- }
317
- }
318
-
319
- // Compute cos/sin for each position
320
- const cosValues = new Float32Array(maxSeqLen * halfDim);
321
- const sinValues = new Float32Array(maxSeqLen * halfDim);
322
-
323
- for (let pos = 0; pos < maxSeqLen; pos++) {
324
- for (let i = 0; i < halfDim; i++) {
325
- const scaledPos = pos / scales[i];
326
- const angle = scaledPos * freqs[i];
327
- cosValues[pos * halfDim + i] = Math.cos(angle);
328
- sinValues[pos * halfDim + i] = Math.sin(angle);
329
- }
330
- }
331
-
332
- return { cos: cosValues, sin: sinValues };
333
- }
334
-
335
- function isSameRoPEScalingConfig(
336
- leftType,
337
- leftScale,
338
- leftScaling,
339
- rightType,
340
- rightScale,
341
- rightScaling
342
- ) {
343
- if (leftType !== rightType) return false;
344
- if (leftScale !== rightScale) return false;
345
- if (leftType !== 'yarn') return true;
346
- return (leftScaling?.beta_fast ?? null) === (rightScaling?.beta_fast ?? null)
347
- && (leftScaling?.beta_slow ?? null) === (rightScaling?.beta_slow ?? null)
348
- && (leftScaling?.original_max_position_embeddings ?? null)
349
- === (rightScaling?.original_max_position_embeddings ?? null);
350
- }
351
-
352
- const GPU_ROPE_BUFFER_CACHE = new WeakMap();
353
- const CPU_ROPE_BUFFER_CACHE = new Map();
354
-
355
- function isLiveCachedRopeBuffer(buffer) {
356
- return buffer == null || isBufferActive(buffer);
357
- }
358
-
359
- function hasLiveCachedGpuRopeBuffers(buffers) {
360
- return !!buffers
361
- && isLiveCachedRopeBuffer(buffers.cos)
362
- && isLiveCachedRopeBuffer(buffers.sin)
363
- && isLiveCachedRopeBuffer(buffers.localCos)
364
- && isLiveCachedRopeBuffer(buffers.localSin);
365
- }
366
-
367
- function buildRoPECacheKey(config) {
368
- return JSON.stringify({
369
- headDim: config.headDim,
370
- localHeadDim: config.localHeadDim ?? null,
371
- rotaryDim: config.rotaryDim ?? null,
372
- ropeLocalRotaryDim: config.ropeLocalRotaryDim ?? null,
373
- ropeFrequencyBaseDim: config.ropeFrequencyBaseDim ?? null,
374
- ropeLocalFrequencyBaseDim: config.ropeLocalFrequencyBaseDim ?? null,
375
- maxSeqLen: config.maxSeqLen,
376
- ropeTheta: config.ropeTheta,
377
- ropeLocalTheta: config.ropeLocalTheta ?? null,
378
- mropeInterleaved: config.mropeInterleaved === true,
379
- mropeSection: Array.isArray(config.mropeSection) ? [...config.mropeSection] : null,
380
- partialRotaryFactor: config.partialRotaryFactor ?? null,
381
- ropeLocalPartialRotaryFactor: config.ropeLocalPartialRotaryFactor ?? null,
382
- ropeScale: config.ropeScale,
383
- ropeLocalScale: config.ropeLocalScale ?? null,
384
- ropeScalingType: config.ropeScalingType ?? null,
385
- ropeLocalScalingType: config.ropeLocalScalingType ?? null,
386
- ropeScaling: config.ropeScaling ?? null,
387
- ropeLocalScaling: config.ropeLocalScaling ?? null,
388
- });
389
- }
390
-
391
- function resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor) {
392
- if (rotaryDim != null) {
393
- if (!Number.isFinite(rotaryDim) || rotaryDim <= 0 || (rotaryDim % 2) !== 0) {
394
- throw new Error(`RoPE rotary dim must be a positive even integer; got "${rotaryDim}".`);
395
- }
396
- if (rotaryDim > headDim) {
397
- throw new Error(`RoPE rotary dim ${rotaryDim} cannot exceed headDim ${headDim}.`);
398
- }
399
- return rotaryDim;
400
- }
401
- if (partialRotaryFactor == null) {
402
- return headDim;
403
- }
404
- if (!Number.isFinite(partialRotaryFactor) || partialRotaryFactor <= 0 || partialRotaryFactor > 1) {
405
- throw new Error(
406
- `RoPE partialRotaryFactor must be a number in (0, 1]; got "${partialRotaryFactor}".`
407
- );
408
- }
409
- const resolved = Math.trunc(headDim * partialRotaryFactor);
410
- if (resolved <= 0 || (resolved % 2) !== 0) {
411
- throw new Error(
412
- `RoPE partialRotaryFactor=${partialRotaryFactor} with headDim=${headDim} resolves ` +
413
- `to rotaryDim=${resolved}, but rotaryDim must be a positive even integer.`
414
- );
415
- }
416
- return resolved;
417
- }
418
-
419
- function resolveFrequencyBaseDim(rotaryDim, frequencyBaseDim, label) {
420
- if (frequencyBaseDim == null) {
421
- return rotaryDim;
422
- }
423
- if (!Number.isFinite(frequencyBaseDim) || frequencyBaseDim <= 0 || (Math.trunc(frequencyBaseDim) % 2) !== 0) {
424
- throw new Error(`${label} must be a positive even integer; got "${frequencyBaseDim}".`);
425
- }
426
- const resolved = Math.trunc(frequencyBaseDim);
427
- if (resolved < rotaryDim) {
428
- throw new Error(`${label} ${resolved} cannot be smaller than rotaryDim ${rotaryDim}.`);
429
- }
430
- return resolved;
431
- }
432
-
433
-
434
- export async function initRoPEFrequencies(config, useGPU) {
435
- const cacheKey = buildRoPECacheKey(config);
436
- const {
437
- headDim,
438
- localHeadDim,
439
- rotaryDim,
440
- ropeLocalRotaryDim,
441
- ropeFrequencyBaseDim,
442
- ropeLocalFrequencyBaseDim,
443
- maxSeqLen,
444
- ropeTheta,
445
- ropeLocalTheta,
446
- mropeInterleaved,
447
- mropeSection,
448
- partialRotaryFactor,
449
- ropeLocalPartialRotaryFactor,
450
- ropeScale,
451
- ropeLocalScale,
452
- ropeScalingType,
453
- ropeLocalScalingType,
454
- ropeScaling,
455
- ropeLocalScaling,
456
- } = config;
457
- if (!Number.isFinite(ropeScale) || ropeScale <= 0) {
458
- throw new Error(`RoPE scale must be a positive number; got "${ropeScale}".`);
459
- }
460
- const resolvedLocalScale = ropeLocalScale;
461
- if (resolvedLocalScale != null && (!Number.isFinite(resolvedLocalScale) || resolvedLocalScale <= 0)) {
462
- throw new Error(`Local RoPE scale must be a positive number; got "${resolvedLocalScale}".`);
463
- }
464
- const resolvedLocalTheta = ropeLocalTheta ?? ropeTheta;
465
- const resolvedLocalScalingType = (
466
- ropeLocalScalingType === undefined
467
- ? ropeScalingType
468
- : ropeLocalScalingType
469
- );
470
- const resolvedLocalScaling = (
471
- ropeLocalScalingType === undefined
472
- ? ropeScaling
473
- : ropeLocalScaling
474
- );
475
- const resolvedLocalHeadDim = localHeadDim ?? headDim;
476
- const resolvedRotaryDim = resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor);
477
- const resolvedLocalRotaryDim = resolveRotaryDim(
478
- resolvedLocalHeadDim,
479
- ropeLocalRotaryDim,
480
- ropeLocalPartialRotaryFactor
481
- );
482
- const resolvedFrequencyBaseDim = resolveFrequencyBaseDim(
483
- resolvedRotaryDim,
484
- ropeFrequencyBaseDim,
485
- 'RoPE frequency base dim'
486
- );
487
- const resolvedLocalFrequencyBaseDim = resolveFrequencyBaseDim(
488
- resolvedLocalRotaryDim,
489
- ropeLocalFrequencyBaseDim,
490
- 'Local RoPE frequency base dim'
491
- );
492
- const halfDim = resolvedRotaryDim / 2;
493
- if (mropeInterleaved === true && Array.isArray(mropeSection)) {
494
- const expandedDim = mropeSection.reduce((sum, entry) => sum + entry, 0) * 2;
495
- if (expandedDim !== resolvedRotaryDim) {
496
- throw new Error(
497
- `RoPE mropeSection expands to ${expandedDim} dims, but rotaryDim is ${resolvedRotaryDim}.`
498
- );
499
- }
500
- }
501
-
502
- const isYarn = ropeScalingType === 'yarn';
503
- const isLocalYarn = resolvedLocalScalingType === 'yarn';
504
-
505
- // Compute global (full_attention) frequencies
506
- const globalFreqs = computeRoPEFreqsForTheta(
507
- ropeTheta,
508
- resolvedRotaryDim,
509
- resolvedFrequencyBaseDim,
510
- maxSeqLen,
511
- ropeScale,
512
- ropeScalingType,
513
- ropeScaling
514
- );
515
-
516
- // Compute local (sliding_attention) frequencies if different from global.
517
- // Models with dual RoPE use different theta for local vs global attention layers.
518
-
519
- let localFreqs = null;
520
- const hasDistinctLocalTheta = resolvedLocalTheta !== ropeTheta;
521
- const hasDistinctLocalDim = resolvedLocalRotaryDim !== resolvedRotaryDim;
522
- const hasDistinctLocalScaling = !isSameRoPEScalingConfig(
523
- ropeScalingType,
524
- ropeScale,
525
- ropeScaling,
526
- resolvedLocalScalingType,
527
- resolvedLocalScale,
528
- resolvedLocalScaling
529
- );
530
- if (hasDistinctLocalTheta || hasDistinctLocalScaling || hasDistinctLocalDim) {
531
- localFreqs = computeRoPEFreqsForTheta(
532
- resolvedLocalTheta,
533
- resolvedLocalRotaryDim,
534
- resolvedLocalFrequencyBaseDim,
535
- maxSeqLen,
536
- resolvedLocalScale,
537
- resolvedLocalScalingType,
538
- resolvedLocalScaling
539
- );
540
- log.debug(
541
- 'Pipeline',
542
- `Dual RoPE: local theta=${resolvedLocalTheta}, global theta=${ropeTheta}, ` +
543
- `localRotaryDim=${resolvedLocalRotaryDim}, globalRotaryDim=${resolvedRotaryDim}, ` +
544
- `localFrequencyBaseDim=${resolvedLocalFrequencyBaseDim}, globalFrequencyBaseDim=${resolvedFrequencyBaseDim}, ` +
545
- `localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}, ` +
546
- `globalScaling=${ropeScalingType ?? 'none'}:${ropeScale}`
547
- );
548
- }
549
-
550
- if (isYarn) {
551
- // Log YARN params (already validated in computeRoPEFreqs)
552
- log.debug('Pipeline', `YARN RoPE: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}`);
553
- }
554
- if (isLocalYarn && hasDistinctLocalScaling) {
555
- log.debug(
556
- 'Pipeline',
557
- `Local YARN RoPE: factor=${resolvedLocalScaling?.factor}, ` +
558
- `beta_fast=${resolvedLocalScaling?.beta_fast}, beta_slow=${resolvedLocalScaling?.beta_slow}`
559
- );
560
- }
561
-
562
- // Upload to GPU if available
563
- const device = getDevice();
564
- if (device && useGPU) {
565
- let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
566
- if (!perDeviceCache) {
567
- perDeviceCache = new Map();
568
- GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
569
- }
570
- const cachedBuffers = perDeviceCache.get(cacheKey);
571
- if (cachedBuffers) {
572
- if (hasLiveCachedGpuRopeBuffers(cachedBuffers)) {
573
- return cachedBuffers;
574
- }
575
- perDeviceCache.delete(cacheKey);
576
- }
577
- let cosBuffer = null;
578
- let sinBuffer = null;
579
- let localCosBuffer = null;
580
- let localSinBuffer = null;
581
- try {
582
- cosBuffer = acquireBuffer(globalFreqs.cos.byteLength, undefined, 'rope_cos');
583
- sinBuffer = acquireBuffer(globalFreqs.sin.byteLength, undefined, 'rope_sin');
584
- device.queue.writeBuffer(cosBuffer, 0, globalFreqs.cos.buffer, globalFreqs.cos.byteOffset, globalFreqs.cos.byteLength);
585
- device.queue.writeBuffer(sinBuffer, 0, globalFreqs.sin.buffer, globalFreqs.sin.byteOffset, globalFreqs.sin.byteLength);
586
-
587
- if (localFreqs) {
588
- localCosBuffer = acquireBuffer(localFreqs.cos.byteLength, undefined, 'rope_local_cos');
589
- localSinBuffer = acquireBuffer(localFreqs.sin.byteLength, undefined, 'rope_local_sin');
590
- device.queue.writeBuffer(localCosBuffer, 0, localFreqs.cos.buffer, localFreqs.cos.byteOffset, localFreqs.cos.byteLength);
591
- device.queue.writeBuffer(localSinBuffer, 0, localFreqs.sin.buffer, localFreqs.sin.byteOffset, localFreqs.sin.byteLength);
592
- }
593
- } catch (error) {
594
- for (const buffer of [cosBuffer, sinBuffer, localCosBuffer, localSinBuffer]) {
595
- if (buffer) {
596
- releaseBuffer(buffer);
597
- }
598
- }
599
- throw error;
600
- }
601
-
602
- log.debug(
603
- 'Pipeline',
604
- `RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
605
- `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
606
- `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
607
- `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
608
- `interleaved=${mropeInterleaved === true}`
609
- );
610
-
611
- const buffers = {
612
- cos: cosBuffer,
613
- sin: sinBuffer,
614
- localCos: localCosBuffer,
615
- localSin: localSinBuffer,
616
- };
617
- perDeviceCache.set(cacheKey, buffers);
618
- return buffers;
619
- }
620
-
621
- const cachedCpuBuffers = CPU_ROPE_BUFFER_CACHE.get(cacheKey);
622
- if (cachedCpuBuffers) {
623
- return cachedCpuBuffers;
624
- }
625
-
626
- log.debug(
627
- 'Pipeline',
628
- `RoPE frequencies initialized (CPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
629
- `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
630
- `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
631
- `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
632
- `interleaved=${mropeInterleaved === true}`
633
- );
634
-
635
- const buffers = {
636
- cos: globalFreqs.cos,
637
- sin: globalFreqs.sin,
638
- localCos: localFreqs?.cos,
639
- localSin: localFreqs?.sin,
640
- };
641
- CPU_ROPE_BUFFER_CACHE.set(cacheKey, buffers);
642
- return buffers;
643
- }
644
-
645
-
646
- export function isGPURoPEBuffers(buffers) {
647
- if (typeof GPUBuffer === 'undefined') return false;
648
- return !!buffers?.cos && isGpuBufferInstance(buffers.cos);
649
- }
650
-
651
-
652
369
  function normalizeLayerType(layerType) {
653
370
  return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
654
371
  }
@@ -701,6 +418,20 @@ function usesMixedGeometryKVCache(modelConfig) {
701
418
  return hasExplicitLayerTypes && (hasMixedAttentionGeometry || hasSharedKvLayers);
702
419
  }
703
420
 
421
+ function resolveMixedGeometrySlidingLayerLayout(modelConfig) {
422
+ const diffusionGemmaContract = modelConfig?.diffusionGemma ?? null;
423
+ if (diffusionGemmaContract == null) {
424
+ return 'ring';
425
+ }
426
+ if (diffusionGemmaContract.decoderCacheMode === 'encoder_kv_readonly_canvas_concat') {
427
+ return 'contiguous';
428
+ }
429
+ throw new Error(
430
+ `Unsupported DiffusionGemma decoderCacheMode="${String(diffusionGemmaContract.decoderCacheMode)}" ` +
431
+ 'for mixed-geometry KV cache.'
432
+ );
433
+ }
434
+
704
435
  function getRequiredKernelMaxKVLen(operation, variant, label) {
705
436
  const config = getKernelConfig(operation, variant);
706
437
  const maxKVLen = config.variantMetadata?.maxKVLen;
@@ -973,8 +704,10 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
973
704
  kvCache = new MixedGeometryKVCache({
974
705
  ...cacheConfig,
975
706
  numHeads: modelConfig.numKVHeads,
707
+ globalNumHeads: modelConfig.numGlobalKVHeads ?? modelConfig.numKVHeads,
976
708
  globalHeadDim: modelConfig.globalHeadDim ?? null,
977
709
  slidingWindow,
710
+ slidingLayerLayout: resolveMixedGeometrySlidingLayerLayout(modelConfig),
978
711
  layerTypes: modelConfig.layerTypes,
979
712
  });
980
713
  } else if (modelConfig.slidingWindow && !forceContiguousKVCache && cacheLayout !== 'paged' && cacheLayout !== 'tiered' && cacheLayout !== 'bdpa') {
@@ -1086,6 +819,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1086
819
  } = options;
1087
820
  const runtimeStorageContext = options.storageContext
1088
821
  ?? createRemoteStorageContext(baseUrl, manifest);
822
+ if (typeof runtimeStorageContext?.preflight === 'function') {
823
+ await runtimeStorageContext.preflight();
824
+ }
1089
825
  const verifyHashes = (
1090
826
  typeof runtimeStorageContext?.verifyHashes === 'boolean'
1091
827
  ? runtimeStorageContext.verifyHashes
@@ -1121,6 +857,11 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1121
857
  } else {
1122
858
  dopplerLoader.setTensorsJsonUrl(null);
1123
859
  }
860
+ dopplerLoader.setTensorsJsonLoader(
861
+ typeof runtimeStorageContext?.loadTensorsJson === 'function'
862
+ ? () => runtimeStorageContext.loadTensorsJson()
863
+ : null
864
+ );
1124
865
 
1125
866
  // Configure custom shard loader if provided (Native Bridge or direct-source bundle)
1126
867
  const hasLoadShard = typeof runtimeStorageContext?.loadShard === 'function';
@@ -1130,25 +871,37 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1130
871
  log.debug('Pipeline', 'Using custom shard loader (Native Bridge or external)');
1131
872
 
1132
873
  const loadShard = async (index) => {
1133
- if (hasLoadShard) {
1134
- const data = await runtimeStorageContext.loadShard(index);
1135
- return toUint8Array(data, 'storageContext.loadShard');
874
+ try {
875
+ if (hasLoadShard) {
876
+ const data = await runtimeStorageContext.loadShard(index);
877
+ return toUint8Array(data, 'storageContext.loadShard');
878
+ }
879
+ const rangeData = await runtimeStorageContext.loadShardRange(index, 0, null);
880
+ return toUint8Array(rangeData, 'storageContext.loadShardRange');
881
+ } catch (error) {
882
+ throw createStorageContextLoadError(error, hasLoadShard ? 'loadShard' : 'loadShardRange', index);
1136
883
  }
1137
- const rangeData = await runtimeStorageContext.loadShardRange(index, 0, null);
1138
- return toUint8Array(rangeData, 'storageContext.loadShardRange');
1139
884
  };
1140
885
 
1141
886
  const loadShardRange = hasLoadShardRange
1142
887
  ? async (index, offset, length = null) => {
1143
- const data = await runtimeStorageContext.loadShardRange(index, offset, length);
1144
- return toArrayBuffer(data, 'storageContext.loadShardRange');
888
+ try {
889
+ const data = await runtimeStorageContext.loadShardRange(index, offset, length);
890
+ return toArrayBuffer(data, 'storageContext.loadShardRange');
891
+ } catch (error) {
892
+ throw createStorageContextLoadError(error, 'loadShardRange', index, { offset, length });
893
+ }
1145
894
  }
1146
895
  : null;
1147
896
 
1148
897
  const streamShardRange = hasStreamShardRange
1149
898
  ? async function* (index, offset = 0, length = null, streamOptions = {}) {
1150
- for await (const chunk of runtimeStorageContext.streamShardRange(index, offset, length, streamOptions)) {
1151
- yield toUint8Array(chunk, 'storageContext.streamShardRange');
899
+ try {
900
+ for await (const chunk of runtimeStorageContext.streamShardRange(index, offset, length, streamOptions)) {
901
+ yield toUint8Array(chunk, 'storageContext.streamShardRange');
902
+ }
903
+ } catch (error) {
904
+ throw createStorageContextLoadError(error, 'streamShardRange', index, { offset, length });
1152
905
  }
1153
906
  }
1154
907
  : null;
@@ -1157,6 +910,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1157
910
  verify: verifyHashes,
1158
911
  loadShardRange,
1159
912
  streamShardRange,
913
+ loadAuxiliaryFile: typeof runtimeStorageContext?.loadAuxiliaryFile === 'function'
914
+ ? (path) => runtimeStorageContext.loadAuxiliaryFile(path)
915
+ : null,
1160
916
  });
1161
917
  if (isRDRRManifest(manifest)) {
1162
918
  dopplerLoader.setManifest(manifest);
@@ -1170,15 +926,19 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1170
926
  if (!modelId) {
1171
927
  throw new Error('Manifest is missing modelId. Re-convert the model with modelId set.');
1172
928
  }
1173
- await dopplerLoader.load(modelId, {
1174
- verifyHashes,
1175
- onProgress: onProgress || ((info) => {
1176
- // Shard and layer progress are logged by loader with source info
1177
- if (info.stage !== 'layers' && info.stage !== 'shards') {
1178
- log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%`);
1179
- }
1180
- }),
1181
- });
929
+ try {
930
+ await dopplerLoader.load(modelId, {
931
+ verifyHashes,
932
+ onProgress: onProgress || ((info) => {
933
+ // Shard and layer progress are logged by loader with source info
934
+ if (info.stage !== 'layers' && info.stage !== 'shards') {
935
+ log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%`);
936
+ }
937
+ }),
938
+ });
939
+ } catch (error) {
940
+ throw rewriteWeightLoadError(error, { modelId });
941
+ }
1182
942
 
1183
943
  // Map layer weights
1184
944
 
@@ -1200,6 +960,8 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1200
960
  layerRouterWeights.set(l, {
1201
961
  weight: weights.routerWeight,
1202
962
  bias: weights.routerBias || null,
963
+ scale: weights.routerScale || null,
964
+ perExpertScale: weights.routerPerExpertScale || null,
1203
965
  });
1204
966
  }
1205
967
  }
@@ -1212,98 +974,15 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1212
974
  lmHead: dopplerLoader.lmHead,
1213
975
  finalNorm: dopplerLoader.finalNorm,
1214
976
  embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
977
+ diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
1215
978
  perLayerInputWeights: dopplerLoader.perLayerInputWeights,
1216
979
  layerRouterWeights,
980
+ loadTiming: typeof dopplerLoader.getLoadTiming === 'function'
981
+ ? dopplerLoader.getLoadTiming()
982
+ : null,
1217
983
  };
1218
984
  }
1219
985
 
1220
- // ============================================================================
1221
- // Chat Templates
1222
- // ============================================================================
1223
-
1224
- // Simple prompt templates for single-turn chat.
1225
- // For multi-turn conversations, use formatChatMessages from chat-format.js.
1226
-
1227
- function applyTurnBasedTemplate(prompt) {
1228
- // Turn-based format: <start_of_turn>role\ncontent<end_of_turn>
1229
- const userTurn = `<start_of_turn>user\n${prompt}<end_of_turn>\n`;
1230
- const modelTurn = '<start_of_turn>model\n';
1231
- return userTurn + modelTurn;
1232
- }
1233
-
1234
- function applyHeaderBasedTemplate(prompt) {
1235
- // Header-based format: <|start_header_id|>role<|end_header_id|>\n\ncontent<|eot_id|>
1236
- return `<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n${prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n`;
1237
- }
1238
-
1239
- function applyChannelBasedTemplate(prompt) {
1240
- // Channel-based format: <|start|>role<|message|>content<|end|>
1241
- return `<|start|>user<|message|>${prompt}<|end|><|start|>assistant<|channel|>final<|message|>`;
1242
- }
1243
-
1244
- function applyChatMLTemplate(prompt) {
1245
- // ChatML format: <|im_start|>role\ncontent<|im_end|>
1246
- return `<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n`;
1247
- }
1248
-
1249
- function getQwenAssistantPrefix(options) {
1250
- return options?.thinking === true
1251
- ? '<|im_start|>assistant\n<think>\n'
1252
- : '<|im_start|>assistant\n<think>\n\n</think>\n\n';
1253
- }
1254
-
1255
- function applyQwenTemplate(prompt, options) {
1256
- return `<|im_start|>user\n${prompt}<|im_end|>\n${getQwenAssistantPrefix(options)}`;
1257
- }
1258
-
1259
- function applyGemma4Template(prompt) {
1260
- return `<bos><|turn>user\n${prompt}<turn|>\n<|turn>model\n`;
1261
- }
1262
-
1263
- function applyTranslateGemmaTemplate() {
1264
- throw new Error(
1265
- 'TranslateGemma template requires structured messages. ' +
1266
- 'Use formatChatMessages(messages, "translategemma") instead of applyChatTemplate(prompt, ...).'
1267
- );
1268
- }
1269
-
1270
- // Template type to formatter mapping.
1271
- // Add new template types here rather than adding switch cases.
1272
- const PROMPT_TEMPLATES = {
1273
- 'gemma': applyTurnBasedTemplate,
1274
- 'gemma4': applyGemma4Template,
1275
- 'llama3': applyHeaderBasedTemplate,
1276
- 'gpt-oss': applyChannelBasedTemplate,
1277
- 'chatml': applyChatMLTemplate,
1278
- 'qwen': applyQwenTemplate,
1279
- 'translategemma': applyTranslateGemmaTemplate,
1280
- };
1281
-
1282
- export function applyChatTemplate(prompt, templateType, options = undefined) {
1283
- if (templateType == null) {
1284
- return prompt;
1285
- }
1286
- const formatter = PROMPT_TEMPLATES[templateType];
1287
- if (formatter) {
1288
- return formatter(prompt, options);
1289
- }
1290
- throw new Error(`Unrecognized chat template type: ${templateType}`);
1291
- }
1292
-
1293
- // Exports preserved for existing external imports.
1294
- export const applyGemmaChatTemplate = applyTurnBasedTemplate;
1295
- export const applyGemma4ChatTemplate = applyGemma4Template;
1296
- export const applyLlama3ChatTemplate = applyHeaderBasedTemplate;
1297
- export const applyGptOssChatTemplate = applyChannelBasedTemplate;
1298
- export const applyQwenChatTemplate = applyQwenTemplate;
1299
-
1300
-
1301
- export function isStopToken(token, stopTokenIds, eosTokenId) {
1302
- if (stopTokenIds.includes(token)) return true;
1303
- if (typeof eosTokenId === 'number' && token === eosTokenId) return true;
1304
- return false;
1305
- }
1306
-
1307
986
  // ============================================================================
1308
987
  // MoE Router Setup
1309
988
  // ============================================================================
@@ -1323,7 +1002,12 @@ export function initMoERouter(modelConfig, moeRoutingConfig, layerWeights) {
1323
1002
  for (let l = 0; l < modelConfig.numLayers; l++) {
1324
1003
  const weights = layerWeights.get(`layer_${l}`);
1325
1004
  if (weights?.routerWeight) {
1326
- router.loadWeights(weights.routerWeight, weights.routerBias || null);
1005
+ router.loadWeights(
1006
+ weights.routerWeight,
1007
+ weights.routerBias || null,
1008
+ weights.routerScale || null,
1009
+ weights.routerPerExpertScale || null
1010
+ );
1327
1011
  log.debug('Pipeline', `Loaded MoE router from layer ${l}${weights.routerBias ? ' (with bias)' : ''}`);
1328
1012
  break;
1329
1013
  }
@@ -1361,202 +1045,6 @@ export function initSpeculativeDecoder(manifest, speculativeConfig) {
1361
1045
  randomSeed: speculativeConfig.randomSeed,
1362
1046
  });
1363
1047
  }
1364
-
1365
- // ============================================================================
1366
- // QKV Fusion
1367
- // ============================================================================
1368
-
1369
-
1370
- function kernelPathVariantRequiresQ4KWeights(variant) {
1371
- if (typeof variant !== 'string' || variant.length === 0) {
1372
- return false;
1373
- }
1374
- let config;
1375
- try {
1376
- config = getKernelConfig('matmul', variant);
1377
- } catch {
1378
- return false;
1379
- }
1380
- const shaderFile = String(config?.shaderFile ?? config?.wgsl ?? '');
1381
- return shaderFile.startsWith('fused_matmul_q4');
1382
- }
1383
-
1384
- function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
1385
- if (!kernelPath) {
1386
- return false;
1387
- }
1388
- const prefillVariant = getKernelPathMatmulVariant('qkv_proj', 'prefill', layerIdx, kernelPath);
1389
- if (kernelPathVariantRequiresQ4KWeights(prefillVariant)) {
1390
- return true;
1391
- }
1392
- const decodeVariant = getKernelPathMatmulVariant('qkv_proj', 'decode', layerIdx, kernelPath);
1393
- return kernelPathVariantRequiresQ4KWeights(decodeVariant);
1394
- }
1395
-
1396
- export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
1397
- const device = getDevice();
1398
- if (!device) {
1399
- log.debug('QKV Fusion', 'No GPU device, skipping fusion');
1400
- return;
1401
- }
1402
- if (
1403
- Number.isFinite(modelConfig?.globalHeadDim)
1404
- && modelConfig.globalHeadDim !== modelConfig.headDim
1405
- ) {
1406
- log.debug(
1407
- 'QKV Fusion',
1408
- 'Skipping QKV fusion for mixed-head-dim model; per-layer attention geometry must stay explicit.'
1409
- );
1410
- return;
1411
- }
1412
-
1413
- const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
1414
- const qSize = numHeads * headDim;
1415
- const kSize = numKVHeads * headDim;
1416
- const vSize = numKVHeads * headDim;
1417
- const qkvSize = qSize + kSize + vSize;
1418
-
1419
- const resolveWeight = (value) => {
1420
- if (isWeightBuffer(value)) {
1421
- return {
1422
- buffer: value.buffer,
1423
- dtype: value.dtype ?? null,
1424
- layout: value.layout ?? null,
1425
- shape: Array.isArray(value.shape) ? value.shape : null,
1426
- };
1427
- }
1428
- if (isGpuBufferInstance(value)) {
1429
- return {
1430
- buffer: value,
1431
- dtype: getWeightDtype(value),
1432
- layout: null,
1433
- shape: null,
1434
- };
1435
- }
1436
- return null;
1437
- };
1438
-
1439
- log.debug('QKV Fusion', `Fusing Q/K/V weights for ${numLayers} layers (${qSize}+${kSize}+${vSize}=${qkvSize})`);
1440
-
1441
- const resolveBytesPerElement = (weight, expectedElements) => {
1442
- const dtype = typeof weight?.dtype === 'string'
1443
- ? weight.dtype.toLowerCase()
1444
- : null;
1445
- if (dtype === 'f16' || dtype === 'bf16') return 2;
1446
- if (dtype === 'f32') return 4;
1447
- const minF16Bytes = expectedElements * 2;
1448
- const minF32Bytes = expectedElements * 4;
1449
- if (weight?.buffer?.size >= minF32Bytes) return 4;
1450
- if (weight?.buffer?.size >= minF16Bytes) return 2;
1451
- return 0;
1452
- };
1453
-
1454
- let fusedCount = 0;
1455
- for (let l = 0; l < numLayers; l++) {
1456
- const weights = layerWeights.get(`layer_${l}`);
1457
- if (!weights) continue;
1458
-
1459
- // Skip if already fused or if weights are not GPUBuffers
1460
- if (weights.qkvProj) continue;
1461
- if (shouldSkipQKVFusionForLayer(l, kernelPath)) {
1462
- log.debug(
1463
- 'QKV Fusion',
1464
- `Layer ${l}: skipped because active kernel path requires Q4K weights for qkv_proj`
1465
- );
1466
- continue;
1467
- }
1468
- const qProj = resolveWeight(weights.qProj);
1469
- const kProj = resolveWeight(weights.kProj);
1470
- const vProj = resolveWeight(weights.vProj);
1471
- if (!qProj || !kProj || !vProj) {
1472
- continue;
1473
- }
1474
-
1475
- const qExpectedElements = qSize * hiddenSize;
1476
- const kExpectedElements = kSize * hiddenSize;
1477
- const vExpectedElements = vSize * hiddenSize;
1478
- const bytesPerElement = resolveBytesPerElement(qProj, qExpectedElements);
1479
- const kBytesPerElement = resolveBytesPerElement(kProj, kExpectedElements);
1480
- const vBytesPerElement = resolveBytesPerElement(vProj, vExpectedElements);
1481
-
1482
- // Pool allocation can round GPUBuffer.size up, so infer logical dtype first and
1483
- // only use buffer size as a minimum-size inference.
1484
- if ((bytesPerElement !== 2 && bytesPerElement !== 4)
1485
- || kBytesPerElement !== bytesPerElement
1486
- || vBytesPerElement !== bytesPerElement) {
1487
- log.debug(
1488
- 'QKV Fusion',
1489
- `Layer ${l}: inconsistent projection dtypes (q=${bytesPerElement}, k=${kBytesPerElement}, v=${vBytesPerElement}), skipping`
1490
- );
1491
- continue;
1492
- }
1493
-
1494
- const normalizedDtype = typeof qProj.dtype === 'string'
1495
- ? qProj.dtype.toLowerCase()
1496
- : null;
1497
- const dtype = normalizedDtype === 'bf16'
1498
- ? 'f16'
1499
- : (
1500
- normalizedDtype === 'f16' || normalizedDtype === 'f32'
1501
- ? normalizedDtype
1502
- : selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement })
1503
- );
1504
- const layout = qProj.layout ?? kProj.layout ?? vProj.layout ?? 'row';
1505
- let fusedShape = [qkvSize, hiddenSize];
1506
- if (Array.isArray(qProj.shape) && qProj.shape.length === 2) {
1507
- if (qProj.shape[0] === qSize && qProj.shape[1] === hiddenSize) {
1508
- fusedShape = [qkvSize, hiddenSize];
1509
- } else if (qProj.shape[1] === qSize && qProj.shape[0] === hiddenSize) {
1510
- fusedShape = [hiddenSize, qkvSize];
1511
- }
1512
- }
1513
-
1514
- // Create fused QKV buffer: [qkvSize, hiddenSize] row-major
1515
- // Each row is concatenated: [q_row, k_row, v_row]
1516
- const qkvBuffer = device.createBuffer({
1517
- label: `layer_${l}_qkv_proj`,
1518
- size: qkvSize * hiddenSize * bytesPerElement,
1519
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
1520
- });
1521
-
1522
- // Copy Q, K, V weights into fused buffer
1523
- // Q: [qSize, hiddenSize] -> offset 0
1524
- // K: [kSize, hiddenSize] -> offset qSize * hiddenSize * bytesPerElement
1525
- // V: [vSize, hiddenSize] -> offset (qSize + kSize) * hiddenSize * bytesPerElement
1526
- const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
1527
- encoder.copyBufferToBuffer(
1528
- qProj.buffer, 0,
1529
- qkvBuffer, 0,
1530
- qSize * hiddenSize * bytesPerElement
1531
- );
1532
- encoder.copyBufferToBuffer(
1533
- kProj.buffer, 0,
1534
- qkvBuffer, qSize * hiddenSize * bytesPerElement,
1535
- kSize * hiddenSize * bytesPerElement
1536
- );
1537
- encoder.copyBufferToBuffer(
1538
- vProj.buffer, 0,
1539
- qkvBuffer, (qSize + kSize) * hiddenSize * bytesPerElement,
1540
- vSize * hiddenSize * bytesPerElement
1541
- );
1542
- device.queue.submit([encoder.finish()]);
1543
-
1544
- // Store fused buffer, sizes, and dtype
1545
- weights.qkvProj = createWeightBuffer(
1546
- qkvBuffer,
1547
- dtype,
1548
- layout,
1549
- fusedShape,
1550
- `layer_${l}_qkv_proj`
1551
- );
1552
- weights.qkvSizes = [qSize, kSize, vSize];
1553
- weights.qkvDtype = dtype;
1554
- fusedCount++;
1555
- }
1556
-
1557
- log.debug('QKV Fusion', `Fused ${fusedCount}/${numLayers} layers`);
1558
- }
1559
-
1560
1048
  // ============================================================================
1561
1049
  // Emulation Setup
1562
1050
  // ============================================================================