doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,8 +1,6 @@
1
1
 
2
2
 
3
- import { parseModelConfig } from './config.js';
4
3
  import { getDevice, getDeviceLimits, getKernelCapabilities } from '../../../gpu/device.js';
5
- import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
6
4
  import {
7
5
  KVCache,
8
6
  SlidingWindowKVCache,
@@ -20,20 +18,48 @@ import { Tokenizer } from '../../tokenizer.js';
20
18
  import { MoERouter } from '../../moe-router.js';
21
19
  import { SpeculativeDecoder } from '../../speculative.js';
22
20
  import { getDopplerLoader } from '../../../loader/doppler-loader.js';
23
- import { log, setGPUDevice, trace as debugTrace } from '../../../debug/index.js';
21
+ import { log, trace as debugTrace } from '../../../debug/index.js';
24
22
  import { getRuntimeConfig } from '../../../config/runtime.js';
25
23
  import { PAGED_LAYOUT_SEQ_LEN_THRESHOLD } from '../../../config/schema/index.js';
26
24
  import {
27
- getKernelPathMatmulVariant,
28
25
  kernelPathRequiresF32MatmulWeights,
29
26
  } from '../../../config/kernel-path-loader.js';
30
- import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
27
+ import { resolveCapabilityTransforms } from '../../../config/transforms/capability-transform-resolver.js';
31
28
  import { selectRuleValue } from '../../../rules/rule-registry.js';
29
+ import { resolvePerLayerInputsSession } from './generator-helpers.js';
32
30
  import {
33
31
  createHttpArtifactStorageContext,
34
32
  createNodeFileArtifactStorageContext,
35
33
  } from '../../../storage/artifact-storage-context.js';
36
34
 
35
+ // Extracted standalone modules
36
+ import { initRoPEFrequencies, isGPURoPEBuffers } from './init-rope.js';
37
+ import {
38
+ applyChatTemplate,
39
+ applyGemmaChatTemplate,
40
+ applyGemma4ChatTemplate,
41
+ applyLlama3ChatTemplate,
42
+ applyGptOssChatTemplate,
43
+ applyQwenChatTemplate,
44
+ isStopToken,
45
+ } from './init-chat-templates.js';
46
+ import { fuseQKVWeights } from './init-qkv-fusion.js';
47
+ import { rewriteWeightLoadError } from './load-errors.js';
48
+ import { toArrayBuffer } from '../../../utils/array-buffer.js';
49
+
50
+ // Re-exports for backwards compatibility
51
+ export { initRoPEFrequencies, isGPURoPEBuffers } from './init-rope.js';
52
+ export {
53
+ applyChatTemplate,
54
+ applyGemmaChatTemplate,
55
+ applyGemma4ChatTemplate,
56
+ applyLlama3ChatTemplate,
57
+ applyGptOssChatTemplate,
58
+ applyQwenChatTemplate,
59
+ isStopToken,
60
+ } from './init-chat-templates.js';
61
+ export { fuseQKVWeights } from './init-qkv-fusion.js';
62
+
37
63
  function resolveErrorMessage(error) {
38
64
  if (error && typeof error === 'object' && typeof error.message === 'string') {
39
65
  return error.message;
@@ -86,16 +112,6 @@ function assertSupportedTurboQuantMode(mode, label) {
86
112
  );
87
113
  }
88
114
 
89
- function toArrayBuffer(value, label) {
90
- if (value instanceof ArrayBuffer) {
91
- return value;
92
- }
93
- if (value instanceof Uint8Array) {
94
- return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
95
- }
96
- throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
97
- }
98
-
99
115
  function toUint8Array(value, label) {
100
116
  if (value instanceof Uint8Array) {
101
117
  return value;
@@ -106,6 +122,25 @@ function toUint8Array(value, label) {
106
122
  throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
107
123
  }
108
124
 
125
+ function createStorageContextLoadError(error, operation, index, context = {}) {
126
+ const message = error?.message || String(error);
127
+ const wrapped = new Error(
128
+ `storageContext.${operation} failed for shard ${index}: ${message}`,
129
+ error instanceof Error ? { cause: error } : undefined
130
+ );
131
+ wrapped.name = error?.name || 'Error';
132
+ if (error?.code !== undefined) {
133
+ wrapped.code = error.code;
134
+ }
135
+ wrapped.details = {
136
+ ...(error?.details && typeof error.details === 'object' ? error.details : {}),
137
+ storageOperation: operation,
138
+ shardIndex: index,
139
+ ...context,
140
+ };
141
+ return wrapped;
142
+ }
143
+
109
144
  function isRDRRManifest(manifest) {
110
145
  return manifest !== null && typeof manifest === 'object' && Array.isArray( (manifest).shards);
111
146
  }
@@ -136,6 +171,23 @@ const Q4K_PROJECTION_OPS = new Set([
136
171
  'linear_out_proj',
137
172
  ]);
138
173
 
174
+ const Q4K_MATMUL_ROLE_OPS = new Map([
175
+ ['q_proj', 'q_proj'],
176
+ ['k_proj', 'k_proj'],
177
+ ['v_proj', 'v_proj'],
178
+ ['o_proj', 'o_proj'],
179
+ ['gate_proj', 'ffn_gate'],
180
+ ['up_proj', 'ffn_up'],
181
+ ['down_proj', 'ffn_down'],
182
+ ['qkv_proj', 'qkv_proj'],
183
+ ['linear_qkv_proj', 'linear_qkv_proj'],
184
+ ['linear_z_proj', 'linear_z_proj'],
185
+ ['linear_a_proj', 'linear_a_proj'],
186
+ ['linear_b_proj', 'linear_b_proj'],
187
+ ['linear_out_proj', 'linear_out_proj'],
188
+ ['lm_head', 'lm_head'],
189
+ ]);
190
+
139
191
  function summarizeQ4KProjectionKernelKinds(kernelPath) {
140
192
  const summary = {
141
193
  denseProjectionKernels: [],
@@ -147,14 +199,15 @@ function summarizeQ4KProjectionKernelKinds(kernelPath) {
147
199
 
148
200
  const appendPhase = (phase, steps) => {
149
201
  for (const step of steps ?? []) {
150
- if (!Q4K_PROJECTION_OPS.has(step?.op)) {
202
+ const op = step?.op;
203
+ if (!Q4K_PROJECTION_OPS.has(op)) {
151
204
  continue;
152
205
  }
153
206
  const kernel = String(step?.kernel ?? '');
154
207
  if (!kernel) {
155
208
  continue;
156
209
  }
157
- const descriptor = `${phase}.${step.op}:${kernel}#${String(step?.entry ?? 'main')}`;
210
+ const descriptor = `${phase}.${op}:${kernel}#${String(step?.entry ?? 'main')}`;
158
211
  if (kernel.startsWith('fused_matmul_q4')) {
159
212
  summary.fusedProjectionKernels.push(descriptor);
160
213
  } else if (kernel.startsWith('matmul_')) {
@@ -168,6 +221,31 @@ function summarizeQ4KProjectionKernelKinds(kernelPath) {
168
221
  return summary;
169
222
  }
170
223
 
224
+ function collectQ4KFusedRoles(kernelPath) {
225
+ const roles = new Set();
226
+ if (!kernelPath || typeof kernelPath !== 'object') {
227
+ return [];
228
+ }
229
+
230
+ const appendSteps = (steps) => {
231
+ for (const step of steps ?? []) {
232
+ const role = Q4K_MATMUL_ROLE_OPS.get(step?.op);
233
+ if (!role) {
234
+ continue;
235
+ }
236
+ const kernel = String(step?.kernel ?? '');
237
+ if (kernel.startsWith('fused_matmul_q4')) {
238
+ roles.add(role);
239
+ }
240
+ }
241
+ };
242
+
243
+ appendSteps(kernelPath.decode?.steps);
244
+ appendSteps(kernelPath.prefill?.steps);
245
+ appendSteps(kernelPath.postLayer);
246
+ return [...roles].sort();
247
+ }
248
+
171
249
  function resolveQ4KProjectionMaterializationMode(
172
250
  manifest,
173
251
  kernelPath,
@@ -186,6 +264,25 @@ function resolveQ4KProjectionMaterializationMode(
186
264
  return mode;
187
265
  }
188
266
 
267
+ function isRetainQ4KMaterializationDisabledByCapability(manifest, caps) {
268
+ if (!caps) return false;
269
+ const adapterInfo = caps.adapterInfo ?? {};
270
+ const platform = {
271
+ id: adapterInfo.device ?? 'unknown',
272
+ vendor: adapterInfo.vendor ?? 'unknown',
273
+ architecture: adapterInfo.architecture ?? 'unknown',
274
+ };
275
+ const runtimeSession = getRuntimeConfig().inference?.session ?? {};
276
+ const graphContext = {
277
+ modelId: manifest?.modelId ?? 'unknown',
278
+ activationDtype: runtimeSession.compute?.defaults?.activationDtype ?? null,
279
+ kvDtype: runtimeSession.kvcache?.kvDtype ?? null,
280
+ retainQ4KMaterialization: true,
281
+ };
282
+ const resolved = resolveCapabilityTransforms(caps, platform, graphContext);
283
+ return resolved.names.includes('disableRetainQ4KMaterialization');
284
+ }
285
+
189
286
  export function resolveQ4KConfig(
190
287
  manifest,
191
288
  kernelPath,
@@ -197,6 +294,7 @@ export function resolveQ4KConfig(
197
294
  // Layout in quantizationInfo: 'row' (fused) or 'col' (dequant)
198
295
  const q4kLayout = manifest?.quantizationInfo?.layout ?? null;
199
296
  const isQ4KModel = manifest?.quantization === 'Q4_K_M';
297
+ const q4kFusedRoles = isQ4KModel ? collectQ4KFusedRoles(kernelPath) : [];
200
298
  if (isQ4KModel && q4kLayout == null) {
201
299
  throw new Error(
202
300
  `Manifest "${manifest?.modelId ?? 'unknown'}" is missing quantizationInfo.layout for Q4_K_M. Re-convert the model.`
@@ -208,9 +306,33 @@ export function resolveQ4KConfig(
208
306
  `Allowed values: ${[...Q4K_LAYOUT_ALLOWLIST].join(', ')}.`
209
307
  );
210
308
  }
211
- const q4kMaterializationMode = isQ4KModel
309
+ let q4kMaterializationMode = isQ4KModel
212
310
  ? resolveQ4KProjectionMaterializationMode(manifest, kernelPath, kernelPathSource)
213
311
  : 'dense';
312
+ // Runtime override: retainQ4KMaterialization forces "mixed" mode so the
313
+ // loader keeps the Q4_K packed buffer alongside the dequantized dense
314
+ // buffer. Unlocks `hasQ4KMaterialization=true` in the FFN fusion rule when
315
+ // the execution graph doesn't declare a fused projection kernel.
316
+ if (isQ4KModel && hasSubgroups && q4kMaterializationMode === 'dense') {
317
+ // Runtime-over-manifest precedence per config-style-guide §Category Rules:
318
+ // getRuntimeConfig() returns the merged session (manifest base + runtime
319
+ // overrides via merge.js); consumers should not re-implement precedence.
320
+ const runtimeRetain = getRuntimeConfig().inference?.session?.retainQ4KMaterialization === true;
321
+ if (runtimeRetain) {
322
+ if (isRetainQ4KMaterializationDisabledByCapability(manifest, caps)) {
323
+ debugTrace.loader(
324
+ 'Q4K materialization retain request disabled by capability rule: ' +
325
+ `model=${manifest?.modelId ?? 'unknown'}, adapter=${caps.adapterInfo?.vendor ?? 'unknown'}/` +
326
+ `${caps.adapterInfo?.architecture ?? 'unknown'}, mode=${q4kMaterializationMode}`
327
+ );
328
+ } else {
329
+ q4kMaterializationMode = 'mixed';
330
+ debugTrace.loader(
331
+ `Q4K materialization overridden by runtime flag retainQ4KMaterialization=true: mode=mixed`
332
+ );
333
+ }
334
+ }
335
+ }
214
336
  if (isQ4KModel) {
215
337
  debugTrace.loader(
216
338
  `Q4K projection materialization: model=${manifest?.modelId ?? 'unknown'}, ` +
@@ -240,399 +362,10 @@ export function resolveQ4KConfig(
240
362
  q4kLayout,
241
363
  keepF32Weights: resolvedKeepF32Weights,
242
364
  q4kMaterializationMode,
365
+ q4kFusedRoles,
243
366
  };
244
367
  }
245
368
 
246
- // ============================================================================
247
- // RoPE Initialization
248
- // ============================================================================
249
-
250
-
251
- function computeRoPEFreqsForTheta(
252
- theta,
253
- rotaryDim,
254
- frequencyBaseDim,
255
- maxSeqLen,
256
- ropeScale,
257
- ropeScalingType,
258
- ropeScaling
259
- ) {
260
- const halfDim = rotaryDim / 2;
261
-
262
- // Compute base frequencies: theta_i = 1 / (base^(2i/d))
263
- const freqs = new Float32Array(halfDim);
264
- for (let i = 0; i < halfDim; i++) {
265
- freqs[i] = 1.0 / Math.pow(theta, (2 * i) / frequencyBaseDim);
266
- }
267
-
268
- // Compute per-dimension scaling factors
269
- const scales = new Float32Array(halfDim);
270
- const isYarn = ropeScalingType === 'yarn';
271
- if (isYarn) {
272
- // YARN scaling - validate ALL required params (fail fast on incomplete manifest)
273
- if (ropeScaling?.factor == null ||
274
- ropeScaling?.beta_fast == null || ropeScaling?.beta_slow == null ||
275
- ropeScaling?.original_max_position_embeddings == null) {
276
- throw new Error(
277
- `RoPE scaling type is 'yarn' but YARN params missing. ` +
278
- `Manifest must provide factor, beta_fast, beta_slow, and original_max_position_embeddings. ` +
279
- `Got: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}, ` +
280
- `original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
281
- );
282
- }
283
- // Extract validated YARN params (no hidden defaults - all guaranteed non-null)
284
- const yarnFactor = ropeScaling.factor;
285
- const yarnBetaFast = ropeScaling.beta_fast;
286
- const yarnBetaSlow = ropeScaling.beta_slow;
287
- const originalMaxPos = ropeScaling.original_max_position_embeddings;
288
-
289
- // YARN: wavelength-based interpolation
290
- for (let i = 0; i < halfDim; i++) {
291
- const wavelength = (2 * Math.PI) / freqs[i];
292
- const lowThresh = originalMaxPos / yarnBetaSlow;
293
- const highThresh = originalMaxPos / yarnBetaFast;
294
-
295
- if (wavelength < highThresh) {
296
- scales[i] = 1.0;
297
- } else if (wavelength > lowThresh) {
298
- scales[i] = yarnFactor;
299
- } else {
300
- const t = (wavelength - highThresh) / (lowThresh - highThresh);
301
- scales[i] = 1.0 + (yarnFactor - 1.0) * t;
302
- }
303
- }
304
- } else {
305
- // Linear scaling: uniform across all dimensions
306
- if (ropeScalingType != null && ropeScalingType !== 'linear') {
307
- log.warn(
308
- 'Pipeline',
309
- `Unrecognized RoPE scaling type "${ropeScalingType}"; falling back to linear scaling. ` +
310
- 'Known types: "linear", "yarn".'
311
- );
312
- }
313
- for (let i = 0; i < halfDim; i++) {
314
- scales[i] = ropeScale;
315
- }
316
- }
317
-
318
- // Compute cos/sin for each position
319
- const cosValues = new Float32Array(maxSeqLen * halfDim);
320
- const sinValues = new Float32Array(maxSeqLen * halfDim);
321
-
322
- for (let pos = 0; pos < maxSeqLen; pos++) {
323
- for (let i = 0; i < halfDim; i++) {
324
- const scaledPos = pos / scales[i];
325
- const angle = scaledPos * freqs[i];
326
- cosValues[pos * halfDim + i] = Math.cos(angle);
327
- sinValues[pos * halfDim + i] = Math.sin(angle);
328
- }
329
- }
330
-
331
- return { cos: cosValues, sin: sinValues };
332
- }
333
-
334
- function isSameRoPEScalingConfig(
335
- leftType,
336
- leftScale,
337
- leftScaling,
338
- rightType,
339
- rightScale,
340
- rightScaling
341
- ) {
342
- if (leftType !== rightType) return false;
343
- if (leftScale !== rightScale) return false;
344
- if (leftType !== 'yarn') return true;
345
- return (leftScaling?.beta_fast ?? null) === (rightScaling?.beta_fast ?? null)
346
- && (leftScaling?.beta_slow ?? null) === (rightScaling?.beta_slow ?? null)
347
- && (leftScaling?.original_max_position_embeddings ?? null)
348
- === (rightScaling?.original_max_position_embeddings ?? null);
349
- }
350
-
351
- const GPU_ROPE_BUFFER_CACHE = new WeakMap();
352
- const CPU_ROPE_BUFFER_CACHE = new Map();
353
-
354
- function buildRoPECacheKey(config) {
355
- return JSON.stringify({
356
- headDim: config.headDim,
357
- localHeadDim: config.localHeadDim ?? null,
358
- rotaryDim: config.rotaryDim ?? null,
359
- ropeLocalRotaryDim: config.ropeLocalRotaryDim ?? null,
360
- ropeFrequencyBaseDim: config.ropeFrequencyBaseDim ?? null,
361
- ropeLocalFrequencyBaseDim: config.ropeLocalFrequencyBaseDim ?? null,
362
- maxSeqLen: config.maxSeqLen,
363
- ropeTheta: config.ropeTheta,
364
- ropeLocalTheta: config.ropeLocalTheta ?? null,
365
- mropeInterleaved: config.mropeInterleaved === true,
366
- mropeSection: Array.isArray(config.mropeSection) ? [...config.mropeSection] : null,
367
- partialRotaryFactor: config.partialRotaryFactor ?? null,
368
- ropeLocalPartialRotaryFactor: config.ropeLocalPartialRotaryFactor ?? null,
369
- ropeScale: config.ropeScale,
370
- ropeLocalScale: config.ropeLocalScale ?? null,
371
- ropeScalingType: config.ropeScalingType ?? null,
372
- ropeLocalScalingType: config.ropeLocalScalingType ?? null,
373
- ropeScaling: config.ropeScaling ?? null,
374
- ropeLocalScaling: config.ropeLocalScaling ?? null,
375
- });
376
- }
377
-
378
- function resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor) {
379
- if (rotaryDim != null) {
380
- if (!Number.isFinite(rotaryDim) || rotaryDim <= 0 || (rotaryDim % 2) !== 0) {
381
- throw new Error(`RoPE rotary dim must be a positive even integer; got "${rotaryDim}".`);
382
- }
383
- if (rotaryDim > headDim) {
384
- throw new Error(`RoPE rotary dim ${rotaryDim} cannot exceed headDim ${headDim}.`);
385
- }
386
- return rotaryDim;
387
- }
388
- if (partialRotaryFactor == null) {
389
- return headDim;
390
- }
391
- if (!Number.isFinite(partialRotaryFactor) || partialRotaryFactor <= 0 || partialRotaryFactor > 1) {
392
- throw new Error(
393
- `RoPE partialRotaryFactor must be a number in (0, 1]; got "${partialRotaryFactor}".`
394
- );
395
- }
396
- const resolved = Math.trunc(headDim * partialRotaryFactor);
397
- if (resolved <= 0 || (resolved % 2) !== 0) {
398
- throw new Error(
399
- `RoPE partialRotaryFactor=${partialRotaryFactor} with headDim=${headDim} resolves ` +
400
- `to rotaryDim=${resolved}, but rotaryDim must be a positive even integer.`
401
- );
402
- }
403
- return resolved;
404
- }
405
-
406
- function resolveFrequencyBaseDim(rotaryDim, frequencyBaseDim, label) {
407
- if (frequencyBaseDim == null) {
408
- return rotaryDim;
409
- }
410
- if (!Number.isFinite(frequencyBaseDim) || frequencyBaseDim <= 0 || (Math.trunc(frequencyBaseDim) % 2) !== 0) {
411
- throw new Error(`${label} must be a positive even integer; got "${frequencyBaseDim}".`);
412
- }
413
- const resolved = Math.trunc(frequencyBaseDim);
414
- if (resolved < rotaryDim) {
415
- throw new Error(`${label} ${resolved} cannot be smaller than rotaryDim ${rotaryDim}.`);
416
- }
417
- return resolved;
418
- }
419
-
420
-
421
- export async function initRoPEFrequencies(config, useGPU) {
422
- const cacheKey = buildRoPECacheKey(config);
423
- const {
424
- headDim,
425
- localHeadDim,
426
- rotaryDim,
427
- ropeLocalRotaryDim,
428
- ropeFrequencyBaseDim,
429
- ropeLocalFrequencyBaseDim,
430
- maxSeqLen,
431
- ropeTheta,
432
- ropeLocalTheta,
433
- mropeInterleaved,
434
- mropeSection,
435
- partialRotaryFactor,
436
- ropeLocalPartialRotaryFactor,
437
- ropeScale,
438
- ropeLocalScale,
439
- ropeScalingType,
440
- ropeLocalScalingType,
441
- ropeScaling,
442
- ropeLocalScaling,
443
- } = config;
444
- if (!Number.isFinite(ropeScale) || ropeScale <= 0) {
445
- throw new Error(`RoPE scale must be a positive number; got "${ropeScale}".`);
446
- }
447
- const resolvedLocalScale = ropeLocalScale;
448
- if (resolvedLocalScale != null && (!Number.isFinite(resolvedLocalScale) || resolvedLocalScale <= 0)) {
449
- throw new Error(`Local RoPE scale must be a positive number; got "${resolvedLocalScale}".`);
450
- }
451
- const resolvedLocalTheta = ropeLocalTheta ?? ropeTheta;
452
- const resolvedLocalScalingType = (
453
- ropeLocalScalingType === undefined
454
- ? ropeScalingType
455
- : ropeLocalScalingType
456
- );
457
- const resolvedLocalScaling = (
458
- ropeLocalScalingType === undefined
459
- ? ropeScaling
460
- : ropeLocalScaling
461
- );
462
- const resolvedLocalHeadDim = localHeadDim ?? headDim;
463
- const resolvedRotaryDim = resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor);
464
- const resolvedLocalRotaryDim = resolveRotaryDim(
465
- resolvedLocalHeadDim,
466
- ropeLocalRotaryDim,
467
- ropeLocalPartialRotaryFactor
468
- );
469
- const resolvedFrequencyBaseDim = resolveFrequencyBaseDim(
470
- resolvedRotaryDim,
471
- ropeFrequencyBaseDim,
472
- 'RoPE frequency base dim'
473
- );
474
- const resolvedLocalFrequencyBaseDim = resolveFrequencyBaseDim(
475
- resolvedLocalRotaryDim,
476
- ropeLocalFrequencyBaseDim,
477
- 'Local RoPE frequency base dim'
478
- );
479
- const halfDim = resolvedRotaryDim / 2;
480
- if (mropeInterleaved === true && Array.isArray(mropeSection)) {
481
- const expandedDim = mropeSection.reduce((sum, entry) => sum + entry, 0) * 2;
482
- if (expandedDim !== resolvedRotaryDim) {
483
- throw new Error(
484
- `RoPE mropeSection expands to ${expandedDim} dims, but rotaryDim is ${resolvedRotaryDim}.`
485
- );
486
- }
487
- }
488
-
489
- const isYarn = ropeScalingType === 'yarn';
490
- const isLocalYarn = resolvedLocalScalingType === 'yarn';
491
-
492
- // Compute global (full_attention) frequencies
493
- const globalFreqs = computeRoPEFreqsForTheta(
494
- ropeTheta,
495
- resolvedRotaryDim,
496
- resolvedFrequencyBaseDim,
497
- maxSeqLen,
498
- ropeScale,
499
- ropeScalingType,
500
- ropeScaling
501
- );
502
-
503
- // Compute local (sliding_attention) frequencies if different from global.
504
- // Models with dual RoPE use different theta for local vs global attention layers.
505
-
506
- let localFreqs = null;
507
- const hasDistinctLocalTheta = resolvedLocalTheta !== ropeTheta;
508
- const hasDistinctLocalDim = resolvedLocalRotaryDim !== resolvedRotaryDim;
509
- const hasDistinctLocalScaling = !isSameRoPEScalingConfig(
510
- ropeScalingType,
511
- ropeScale,
512
- ropeScaling,
513
- resolvedLocalScalingType,
514
- resolvedLocalScale,
515
- resolvedLocalScaling
516
- );
517
- if (hasDistinctLocalTheta || hasDistinctLocalScaling || hasDistinctLocalDim) {
518
- localFreqs = computeRoPEFreqsForTheta(
519
- resolvedLocalTheta,
520
- resolvedLocalRotaryDim,
521
- resolvedLocalFrequencyBaseDim,
522
- maxSeqLen,
523
- resolvedLocalScale,
524
- resolvedLocalScalingType,
525
- resolvedLocalScaling
526
- );
527
- log.debug(
528
- 'Pipeline',
529
- `Dual RoPE: local theta=${resolvedLocalTheta}, global theta=${ropeTheta}, ` +
530
- `localRotaryDim=${resolvedLocalRotaryDim}, globalRotaryDim=${resolvedRotaryDim}, ` +
531
- `localFrequencyBaseDim=${resolvedLocalFrequencyBaseDim}, globalFrequencyBaseDim=${resolvedFrequencyBaseDim}, ` +
532
- `localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}, ` +
533
- `globalScaling=${ropeScalingType ?? 'none'}:${ropeScale}`
534
- );
535
- }
536
-
537
- if (isYarn) {
538
- // Log YARN params (already validated in computeRoPEFreqs)
539
- log.debug('Pipeline', `YARN RoPE: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}`);
540
- }
541
- if (isLocalYarn && hasDistinctLocalScaling) {
542
- log.debug(
543
- 'Pipeline',
544
- `Local YARN RoPE: factor=${resolvedLocalScaling?.factor}, ` +
545
- `beta_fast=${resolvedLocalScaling?.beta_fast}, beta_slow=${resolvedLocalScaling?.beta_slow}`
546
- );
547
- }
548
-
549
- // Upload to GPU if available
550
- const device = getDevice();
551
- if (device && useGPU) {
552
- let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
553
- if (!perDeviceCache) {
554
- perDeviceCache = new Map();
555
- GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
556
- }
557
- const cachedBuffers = perDeviceCache.get(cacheKey);
558
- if (cachedBuffers) {
559
- return cachedBuffers;
560
- }
561
- let cosBuffer = null;
562
- let sinBuffer = null;
563
- let localCosBuffer = null;
564
- let localSinBuffer = null;
565
- try {
566
- cosBuffer = acquireBuffer(globalFreqs.cos.byteLength, undefined, 'rope_cos');
567
- sinBuffer = acquireBuffer(globalFreqs.sin.byteLength, undefined, 'rope_sin');
568
- device.queue.writeBuffer(cosBuffer, 0, globalFreqs.cos.buffer, globalFreqs.cos.byteOffset, globalFreqs.cos.byteLength);
569
- device.queue.writeBuffer(sinBuffer, 0, globalFreqs.sin.buffer, globalFreqs.sin.byteOffset, globalFreqs.sin.byteLength);
570
-
571
- if (localFreqs) {
572
- localCosBuffer = acquireBuffer(localFreqs.cos.byteLength, undefined, 'rope_local_cos');
573
- localSinBuffer = acquireBuffer(localFreqs.sin.byteLength, undefined, 'rope_local_sin');
574
- device.queue.writeBuffer(localCosBuffer, 0, localFreqs.cos.buffer, localFreqs.cos.byteOffset, localFreqs.cos.byteLength);
575
- device.queue.writeBuffer(localSinBuffer, 0, localFreqs.sin.buffer, localFreqs.sin.byteOffset, localFreqs.sin.byteLength);
576
- }
577
- } catch (error) {
578
- for (const buffer of [cosBuffer, sinBuffer, localCosBuffer, localSinBuffer]) {
579
- if (buffer) {
580
- releaseBuffer(buffer);
581
- }
582
- }
583
- throw error;
584
- }
585
-
586
- log.debug(
587
- 'Pipeline',
588
- `RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
589
- `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
590
- `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
591
- `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
592
- `interleaved=${mropeInterleaved === true}`
593
- );
594
-
595
- const buffers = {
596
- cos: cosBuffer,
597
- sin: sinBuffer,
598
- localCos: localCosBuffer,
599
- localSin: localSinBuffer,
600
- };
601
- perDeviceCache.set(cacheKey, buffers);
602
- return buffers;
603
- }
604
-
605
- const cachedCpuBuffers = CPU_ROPE_BUFFER_CACHE.get(cacheKey);
606
- if (cachedCpuBuffers) {
607
- return cachedCpuBuffers;
608
- }
609
-
610
- log.debug(
611
- 'Pipeline',
612
- `RoPE frequencies initialized (CPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
613
- `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
614
- `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
615
- `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
616
- `interleaved=${mropeInterleaved === true}`
617
- );
618
-
619
- const buffers = {
620
- cos: globalFreqs.cos,
621
- sin: globalFreqs.sin,
622
- localCos: localFreqs?.cos,
623
- localSin: localFreqs?.sin,
624
- };
625
- CPU_ROPE_BUFFER_CACHE.set(cacheKey, buffers);
626
- return buffers;
627
- }
628
-
629
-
630
- export function isGPURoPEBuffers(buffers) {
631
- if (typeof GPUBuffer === 'undefined') return false;
632
- return !!buffers?.cos && isGpuBufferInstance(buffers.cos);
633
- }
634
-
635
-
636
369
  function normalizeLayerType(layerType) {
637
370
  return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
638
371
  }
@@ -957,6 +690,7 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
957
690
  kvCache = new MixedGeometryKVCache({
958
691
  ...cacheConfig,
959
692
  numHeads: modelConfig.numKVHeads,
693
+ globalNumHeads: modelConfig.numGlobalKVHeads ?? modelConfig.numKVHeads,
960
694
  globalHeadDim: modelConfig.globalHeadDim ?? null,
961
695
  slidingWindow,
962
696
  layerTypes: modelConfig.layerTypes,
@@ -1070,6 +804,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1070
804
  } = options;
1071
805
  const runtimeStorageContext = options.storageContext
1072
806
  ?? createRemoteStorageContext(baseUrl, manifest);
807
+ if (typeof runtimeStorageContext?.preflight === 'function') {
808
+ await runtimeStorageContext.preflight();
809
+ }
1073
810
  const verifyHashes = (
1074
811
  typeof runtimeStorageContext?.verifyHashes === 'boolean'
1075
812
  ? runtimeStorageContext.verifyHashes
@@ -1090,7 +827,12 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1090
827
  )
1091
828
  );
1092
829
  dopplerLoader.setLoaderDebugConfig(loaderDebug ?? null);
1093
- dopplerLoader.setPerLayerInputSession(perLayerInputSession ?? modelConfig.perLayerInputsSession);
830
+ dopplerLoader.setPerLayerInputSession(
831
+ resolvePerLayerInputsSession(
832
+ modelConfig.perLayerInputsSession ?? null,
833
+ perLayerInputSession ?? null
834
+ )
835
+ );
1094
836
 
1095
837
  const tensorsFile = isRDRRManifest(manifest) ? manifest.tensorsFile : null;
1096
838
  if (baseUrl && tensorsFile) {
@@ -1100,6 +842,11 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1100
842
  } else {
1101
843
  dopplerLoader.setTensorsJsonUrl(null);
1102
844
  }
845
+ dopplerLoader.setTensorsJsonLoader(
846
+ typeof runtimeStorageContext?.loadTensorsJson === 'function'
847
+ ? () => runtimeStorageContext.loadTensorsJson()
848
+ : null
849
+ );
1103
850
 
1104
851
  // Configure custom shard loader if provided (Native Bridge or direct-source bundle)
1105
852
  const hasLoadShard = typeof runtimeStorageContext?.loadShard === 'function';
@@ -1109,25 +856,37 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1109
856
  log.debug('Pipeline', 'Using custom shard loader (Native Bridge or external)');
1110
857
 
1111
858
  const loadShard = async (index) => {
1112
- if (hasLoadShard) {
1113
- const data = await runtimeStorageContext.loadShard(index);
1114
- return toUint8Array(data, 'storageContext.loadShard');
859
+ try {
860
+ if (hasLoadShard) {
861
+ const data = await runtimeStorageContext.loadShard(index);
862
+ return toUint8Array(data, 'storageContext.loadShard');
863
+ }
864
+ const rangeData = await runtimeStorageContext.loadShardRange(index, 0, null);
865
+ return toUint8Array(rangeData, 'storageContext.loadShardRange');
866
+ } catch (error) {
867
+ throw createStorageContextLoadError(error, hasLoadShard ? 'loadShard' : 'loadShardRange', index);
1115
868
  }
1116
- const rangeData = await runtimeStorageContext.loadShardRange(index, 0, null);
1117
- return toUint8Array(rangeData, 'storageContext.loadShardRange');
1118
869
  };
1119
870
 
1120
871
  const loadShardRange = hasLoadShardRange
1121
872
  ? async (index, offset, length = null) => {
1122
- const data = await runtimeStorageContext.loadShardRange(index, offset, length);
1123
- return toArrayBuffer(data, 'storageContext.loadShardRange');
873
+ try {
874
+ const data = await runtimeStorageContext.loadShardRange(index, offset, length);
875
+ return toArrayBuffer(data, 'storageContext.loadShardRange');
876
+ } catch (error) {
877
+ throw createStorageContextLoadError(error, 'loadShardRange', index, { offset, length });
878
+ }
1124
879
  }
1125
880
  : null;
1126
881
 
1127
882
  const streamShardRange = hasStreamShardRange
1128
883
  ? async function* (index, offset = 0, length = null, streamOptions = {}) {
1129
- for await (const chunk of runtimeStorageContext.streamShardRange(index, offset, length, streamOptions)) {
1130
- yield toUint8Array(chunk, 'storageContext.streamShardRange');
884
+ try {
885
+ for await (const chunk of runtimeStorageContext.streamShardRange(index, offset, length, streamOptions)) {
886
+ yield toUint8Array(chunk, 'storageContext.streamShardRange');
887
+ }
888
+ } catch (error) {
889
+ throw createStorageContextLoadError(error, 'streamShardRange', index, { offset, length });
1131
890
  }
1132
891
  }
1133
892
  : null;
@@ -1149,15 +908,19 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1149
908
  if (!modelId) {
1150
909
  throw new Error('Manifest is missing modelId. Re-convert the model with modelId set.');
1151
910
  }
1152
- await dopplerLoader.load(modelId, {
1153
- verifyHashes,
1154
- onProgress: onProgress || ((info) => {
1155
- // Shard and layer progress are logged by loader with source info
1156
- if (info.stage !== 'layers' && info.stage !== 'shards') {
1157
- log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%`);
1158
- }
1159
- }),
1160
- });
911
+ try {
912
+ await dopplerLoader.load(modelId, {
913
+ verifyHashes,
914
+ onProgress: onProgress || ((info) => {
915
+ // Shard and layer progress are logged by loader with source info
916
+ if (info.stage !== 'layers' && info.stage !== 'shards') {
917
+ log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%`);
918
+ }
919
+ }),
920
+ });
921
+ } catch (error) {
922
+ throw rewriteWeightLoadError(error, { modelId });
923
+ }
1161
924
 
1162
925
  // Map layer weights
1163
926
 
@@ -1196,93 +959,6 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1196
959
  };
1197
960
  }
1198
961
 
1199
- // ============================================================================
1200
- // Chat Templates
1201
- // ============================================================================
1202
-
1203
- // Simple prompt templates for single-turn chat.
1204
- // For multi-turn conversations, use formatChatMessages from chat-format.js.
1205
-
1206
- function applyTurnBasedTemplate(prompt) {
1207
- // Turn-based format: <start_of_turn>role\ncontent<end_of_turn>
1208
- const userTurn = `<start_of_turn>user\n${prompt}<end_of_turn>\n`;
1209
- const modelTurn = '<start_of_turn>model\n';
1210
- return userTurn + modelTurn;
1211
- }
1212
-
1213
- function applyHeaderBasedTemplate(prompt) {
1214
- // Header-based format: <|start_header_id|>role<|end_header_id|>\n\ncontent<|eot_id|>
1215
- return `<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n${prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n`;
1216
- }
1217
-
1218
- function applyChannelBasedTemplate(prompt) {
1219
- // Channel-based format: <|start|>role<|message|>content<|end|>
1220
- return `<|start|>user<|message|>${prompt}<|end|><|start|>assistant<|channel|>final<|message|>`;
1221
- }
1222
-
1223
- function applyChatMLTemplate(prompt) {
1224
- // ChatML format: <|im_start|>role\ncontent<|im_end|>
1225
- return `<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n`;
1226
- }
1227
-
1228
- function getQwenAssistantPrefix(options) {
1229
- return options?.thinking === true
1230
- ? '<|im_start|>assistant\n<think>\n'
1231
- : '<|im_start|>assistant\n<think>\n\n</think>\n\n';
1232
- }
1233
-
1234
- function applyQwenTemplate(prompt, options) {
1235
- return `<|im_start|>user\n${prompt}<|im_end|>\n${getQwenAssistantPrefix(options)}`;
1236
- }
1237
-
1238
- function applyGemma4Template(prompt) {
1239
- return `<bos><|turn>user\n${prompt}<turn|>\n<|turn>model\n`;
1240
- }
1241
-
1242
- function applyTranslateGemmaTemplate() {
1243
- throw new Error(
1244
- 'TranslateGemma template requires structured messages. ' +
1245
- 'Use formatChatMessages(messages, "translategemma") instead of applyChatTemplate(prompt, ...).'
1246
- );
1247
- }
1248
-
1249
- // Template type to formatter mapping.
1250
- // Add new template types here rather than adding switch cases.
1251
- const PROMPT_TEMPLATES = {
1252
- 'gemma': applyTurnBasedTemplate,
1253
- 'gemma4': applyGemma4Template,
1254
- 'llama3': applyHeaderBasedTemplate,
1255
- 'gpt-oss': applyChannelBasedTemplate,
1256
- 'chatml': applyChatMLTemplate,
1257
- 'qwen': applyQwenTemplate,
1258
- 'translategemma': applyTranslateGemmaTemplate,
1259
- };
1260
-
1261
- export function applyChatTemplate(prompt, templateType, options = undefined) {
1262
- if (templateType == null) {
1263
- return prompt;
1264
- }
1265
- const formatter = PROMPT_TEMPLATES[templateType];
1266
- if (formatter) {
1267
- return formatter(prompt, options);
1268
- }
1269
- throw new Error(`Unrecognized chat template type: ${templateType}`);
1270
- }
1271
-
1272
- // Exports preserved for existing external imports.
1273
- export const applyGemmaChatTemplate = applyTurnBasedTemplate;
1274
- export const applyGemma4ChatTemplate = applyGemma4Template;
1275
- export const applyLlama3ChatTemplate = applyHeaderBasedTemplate;
1276
- export const applyGptOssChatTemplate = applyChannelBasedTemplate;
1277
- export const applyQwenChatTemplate = applyQwenTemplate;
1278
-
1279
-
1280
- export function isStopToken(token, stopTokenIds, eosTokenId) {
1281
- if (stopTokenIds.includes(token)) return true;
1282
- if (typeof eosTokenId === 'number' && token === eosTokenId) return true;
1283
- return false;
1284
- }
1285
-
1286
962
  // ============================================================================
1287
963
  // MoE Router Setup
1288
964
  // ============================================================================
@@ -1340,202 +1016,6 @@ export function initSpeculativeDecoder(manifest, speculativeConfig) {
1340
1016
  randomSeed: speculativeConfig.randomSeed,
1341
1017
  });
1342
1018
  }
1343
-
1344
- // ============================================================================
1345
- // QKV Fusion
1346
- // ============================================================================
1347
-
1348
-
1349
- function kernelPathVariantRequiresQ4KWeights(variant) {
1350
- if (typeof variant !== 'string' || variant.length === 0) {
1351
- return false;
1352
- }
1353
- let config;
1354
- try {
1355
- config = getKernelConfig('matmul', variant);
1356
- } catch {
1357
- return false;
1358
- }
1359
- const shaderFile = String(config?.shaderFile ?? config?.wgsl ?? '');
1360
- return shaderFile.startsWith('fused_matmul_q4');
1361
- }
1362
-
1363
- function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
1364
- if (!kernelPath) {
1365
- return false;
1366
- }
1367
- const prefillVariant = getKernelPathMatmulVariant('qkv_proj', 'prefill', layerIdx, kernelPath);
1368
- if (kernelPathVariantRequiresQ4KWeights(prefillVariant)) {
1369
- return true;
1370
- }
1371
- const decodeVariant = getKernelPathMatmulVariant('qkv_proj', 'decode', layerIdx, kernelPath);
1372
- return kernelPathVariantRequiresQ4KWeights(decodeVariant);
1373
- }
1374
-
1375
- export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
1376
- const device = getDevice();
1377
- if (!device) {
1378
- log.debug('QKV Fusion', 'No GPU device, skipping fusion');
1379
- return;
1380
- }
1381
- if (
1382
- Number.isFinite(modelConfig?.globalHeadDim)
1383
- && modelConfig.globalHeadDim !== modelConfig.headDim
1384
- ) {
1385
- log.debug(
1386
- 'QKV Fusion',
1387
- 'Skipping QKV fusion for mixed-head-dim model; per-layer attention geometry must stay explicit.'
1388
- );
1389
- return;
1390
- }
1391
-
1392
- const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
1393
- const qSize = numHeads * headDim;
1394
- const kSize = numKVHeads * headDim;
1395
- const vSize = numKVHeads * headDim;
1396
- const qkvSize = qSize + kSize + vSize;
1397
-
1398
- const resolveWeight = (value) => {
1399
- if (isWeightBuffer(value)) {
1400
- return {
1401
- buffer: value.buffer,
1402
- dtype: value.dtype ?? null,
1403
- layout: value.layout ?? null,
1404
- shape: Array.isArray(value.shape) ? value.shape : null,
1405
- };
1406
- }
1407
- if (isGpuBufferInstance(value)) {
1408
- return {
1409
- buffer: value,
1410
- dtype: getWeightDtype(value),
1411
- layout: null,
1412
- shape: null,
1413
- };
1414
- }
1415
- return null;
1416
- };
1417
-
1418
- log.debug('QKV Fusion', `Fusing Q/K/V weights for ${numLayers} layers (${qSize}+${kSize}+${vSize}=${qkvSize})`);
1419
-
1420
- const resolveBytesPerElement = (weight, expectedElements) => {
1421
- const dtype = typeof weight?.dtype === 'string'
1422
- ? weight.dtype.toLowerCase()
1423
- : null;
1424
- if (dtype === 'f16' || dtype === 'bf16') return 2;
1425
- if (dtype === 'f32') return 4;
1426
- const minF16Bytes = expectedElements * 2;
1427
- const minF32Bytes = expectedElements * 4;
1428
- if (weight?.buffer?.size >= minF32Bytes) return 4;
1429
- if (weight?.buffer?.size >= minF16Bytes) return 2;
1430
- return 0;
1431
- };
1432
-
1433
- let fusedCount = 0;
1434
- for (let l = 0; l < numLayers; l++) {
1435
- const weights = layerWeights.get(`layer_${l}`);
1436
- if (!weights) continue;
1437
-
1438
- // Skip if already fused or if weights are not GPUBuffers
1439
- if (weights.qkvProj) continue;
1440
- if (shouldSkipQKVFusionForLayer(l, kernelPath)) {
1441
- log.debug(
1442
- 'QKV Fusion',
1443
- `Layer ${l}: skipped because active kernel path requires Q4K weights for qkv_proj`
1444
- );
1445
- continue;
1446
- }
1447
- const qProj = resolveWeight(weights.qProj);
1448
- const kProj = resolveWeight(weights.kProj);
1449
- const vProj = resolveWeight(weights.vProj);
1450
- if (!qProj || !kProj || !vProj) {
1451
- continue;
1452
- }
1453
-
1454
- const qExpectedElements = qSize * hiddenSize;
1455
- const kExpectedElements = kSize * hiddenSize;
1456
- const vExpectedElements = vSize * hiddenSize;
1457
- const bytesPerElement = resolveBytesPerElement(qProj, qExpectedElements);
1458
- const kBytesPerElement = resolveBytesPerElement(kProj, kExpectedElements);
1459
- const vBytesPerElement = resolveBytesPerElement(vProj, vExpectedElements);
1460
-
1461
- // Pool allocation can round GPUBuffer.size up, so infer logical dtype first and
1462
- // only use buffer size as a minimum-size inference.
1463
- if ((bytesPerElement !== 2 && bytesPerElement !== 4)
1464
- || kBytesPerElement !== bytesPerElement
1465
- || vBytesPerElement !== bytesPerElement) {
1466
- log.debug(
1467
- 'QKV Fusion',
1468
- `Layer ${l}: inconsistent projection dtypes (q=${bytesPerElement}, k=${kBytesPerElement}, v=${vBytesPerElement}), skipping`
1469
- );
1470
- continue;
1471
- }
1472
-
1473
- const normalizedDtype = typeof qProj.dtype === 'string'
1474
- ? qProj.dtype.toLowerCase()
1475
- : null;
1476
- const dtype = normalizedDtype === 'bf16'
1477
- ? 'f16'
1478
- : (
1479
- normalizedDtype === 'f16' || normalizedDtype === 'f32'
1480
- ? normalizedDtype
1481
- : selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement })
1482
- );
1483
- const layout = qProj.layout ?? kProj.layout ?? vProj.layout ?? 'row';
1484
- let fusedShape = [qkvSize, hiddenSize];
1485
- if (Array.isArray(qProj.shape) && qProj.shape.length === 2) {
1486
- if (qProj.shape[0] === qSize && qProj.shape[1] === hiddenSize) {
1487
- fusedShape = [qkvSize, hiddenSize];
1488
- } else if (qProj.shape[1] === qSize && qProj.shape[0] === hiddenSize) {
1489
- fusedShape = [hiddenSize, qkvSize];
1490
- }
1491
- }
1492
-
1493
- // Create fused QKV buffer: [qkvSize, hiddenSize] row-major
1494
- // Each row is concatenated: [q_row, k_row, v_row]
1495
- const qkvBuffer = device.createBuffer({
1496
- label: `layer_${l}_qkv_proj`,
1497
- size: qkvSize * hiddenSize * bytesPerElement,
1498
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
1499
- });
1500
-
1501
- // Copy Q, K, V weights into fused buffer
1502
- // Q: [qSize, hiddenSize] -> offset 0
1503
- // K: [kSize, hiddenSize] -> offset qSize * hiddenSize * bytesPerElement
1504
- // V: [vSize, hiddenSize] -> offset (qSize + kSize) * hiddenSize * bytesPerElement
1505
- const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
1506
- encoder.copyBufferToBuffer(
1507
- qProj.buffer, 0,
1508
- qkvBuffer, 0,
1509
- qSize * hiddenSize * bytesPerElement
1510
- );
1511
- encoder.copyBufferToBuffer(
1512
- kProj.buffer, 0,
1513
- qkvBuffer, qSize * hiddenSize * bytesPerElement,
1514
- kSize * hiddenSize * bytesPerElement
1515
- );
1516
- encoder.copyBufferToBuffer(
1517
- vProj.buffer, 0,
1518
- qkvBuffer, (qSize + kSize) * hiddenSize * bytesPerElement,
1519
- vSize * hiddenSize * bytesPerElement
1520
- );
1521
- device.queue.submit([encoder.finish()]);
1522
-
1523
- // Store fused buffer, sizes, and dtype
1524
- weights.qkvProj = createWeightBuffer(
1525
- qkvBuffer,
1526
- dtype,
1527
- layout,
1528
- fusedShape,
1529
- `layer_${l}_qkv_proj`
1530
- );
1531
- weights.qkvSizes = [qSize, kSize, vSize];
1532
- weights.qkvDtype = dtype;
1533
- fusedCount++;
1534
- }
1535
-
1536
- log.debug('QKV Fusion', `Fused ${fusedCount}/${numLayers} layers`);
1537
- }
1538
-
1539
1019
  // ============================================================================
1540
1020
  // Emulation Setup
1541
1021
  // ============================================================================