doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -12,6 +12,17 @@ import { unifiedKernelWrapper } from './utils.js';
12
12
  // Conservative fallback dtype for norm weight inference when metadata is unavailable.
13
13
  const DEFAULT_DTYPE = 'f32';
14
14
 
15
+ // Must equal MAX_CACHE_SIZE in rmsnorm.wgsl. The "residual" variant routes to
16
+ // main_cached, which caches the input in shared_cache[MAX_CACHE_SIZE]. When
17
+ // hiddenSize exceeds this limit, OOB cache writes silently drop indices and the
18
+ // second pass reads zero for those dims, producing wrong output (only residual
19
+ // preserved). Models with hiddenSize > limit must skip the cached variant.
20
+ export const RMSNORM_CACHE_LIMIT = 4608;
21
+
22
+ export function residualVariantBypassesCache(residual, hiddenSize) {
23
+ return !!residual && hiddenSize !== null && hiddenSize !== undefined && hiddenSize > RMSNORM_CACHE_LIMIT;
24
+ }
25
+
15
26
  function inferHiddenSize(input, hiddenSize) {
16
27
  if (hiddenSize != null) return hiddenSize;
17
28
  const shape = input?.shape;
@@ -104,7 +115,12 @@ export function selectRMSNormKernel(options = {}, isF16 = false) {
104
115
  return selectRuleValue(
105
116
  'rmsnorm',
106
117
  'variant',
107
- { isF16, residual: !!residual, hasSubgroups, isSmall }
118
+ {
119
+ isF16,
120
+ residual: !!residual && !residualVariantBypassesCache(residual, hiddenSize),
121
+ hasSubgroups,
122
+ isSmall,
123
+ }
108
124
  );
109
125
  }
110
126
 
@@ -10,7 +10,7 @@
10
10
  //
11
11
  // Uses override constants for compile-time feature selection:
12
12
  // - RMS_NORM_OFFSET: Use (1 + weight) pattern (Gemma models)
13
- // - HAS_RESIDUAL: Enable post-norm residual addition
13
+ // - HAS_RESIDUAL: Compile-time override (reserved for codegen patches, not used by JS dispatch)
14
14
  // - OUTPUT_PRENORM: Output pre-normalized values (before weight multiplication)
15
15
 
16
16
  enable subgroups;
@@ -30,6 +30,7 @@ override OUTPUT_PRENORM: bool = false; // Write pre-norm sum to residual_sum_
30
30
  override WEIGHT_IS_F16: bool = false; // Weight buffer packed as f16 pairs
31
31
 
32
32
  const MAX_SUBGROUPS: u32 = 32u; // Support up to 32 subgroups per workgroup
33
+ const MAX_CACHE_SIZE: u32 = 4608u; // Supports hiddenSize up to 1152 with 4 elements/thread
33
34
 
34
35
  // =============================================================================
35
36
  // Uniforms (per-dispatch)
@@ -81,11 +82,9 @@ fn load_weight(idx: u32) -> f32 {
81
82
  return bitcast<f32>(weight[idx]);
82
83
  }
83
84
 
84
- // Check if residual should be added (compile-time OR runtime flag)
85
+ // Check if residual should be added (runtime uniform only — JS dispatch controls this)
85
86
  fn should_add_residual() -> bool {
86
- // HAS_RESIDUAL is compile-time override (for optimized paths)
87
- // u.has_residual is runtime flag (for flexibility)
88
- return HAS_RESIDUAL || (u.has_residual != 0u);
87
+ return u.has_residual != 0u;
89
88
  }
90
89
 
91
90
  // Load input value, fusing residual add when PRE_RESIDUAL is active
@@ -242,7 +241,7 @@ fn main_small(
242
241
 
243
242
  // OPTIMIZED: Caches input to avoid double loads
244
243
  // Uses shared memory to store input values between passes
245
- var<workgroup> shared_cache: array<f32, 4608>; // Max for hiddenSize=1152 × 4 elements/thread
244
+ var<workgroup> shared_cache: array<f32, MAX_CACHE_SIZE>;
246
245
 
247
246
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
248
247
  fn main_cached(
@@ -1,12 +1,9 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
3
- // AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
4
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
5
- // RMSNorm Kernel with F16 Input/Output
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
6
3
  //
7
4
  // F16 variant for reduced memory bandwidth when using F16 activations.
8
- // Intermediate computations (sum of squares, RMS) remain in F32 for precision.
9
- // Weight buffer may be F16 or F32 (small size, precision matters).
5
+ // The all-f16 lane keeps normalization arithmetic in f16.
6
+ // Weight buffer may be F16 or F32.
10
7
  //
11
8
  // RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
12
9
 
@@ -37,38 +34,37 @@ struct Uniforms {
37
34
  @group(0) @binding(4) var<storage, read> residual: array<f16>; // Optional residual
38
35
  @group(0) @binding(5) var<storage, read_write> residual_sum_output: array<f16>; // Pre-norm sum output (PRE_RESIDUAL + OUTPUT_PRENORM)
39
36
 
40
- // Shared memory for reduction (F32 for precision)
41
- var<workgroup> shared_sum: array<f32, MAX_WORKGROUP_SIZE>;
37
+ var<workgroup> shared_sum: array<f16, MAX_WORKGROUP_SIZE>;
42
38
 
43
- fn apply_weight(w: f32) -> f32 {
39
+ fn apply_weight(w: f16) -> f16 {
44
40
  if (RMS_NORM_OFFSET) {
45
- return 1.0 + w;
41
+ return f16(1.0) + w;
46
42
  }
47
43
  return w;
48
44
  }
49
45
 
50
- fn load_weight(idx: u32) -> f32 {
46
+ fn load_weight(idx: u32) -> f16 {
51
47
  if (WEIGHT_IS_F16) {
52
48
  let packed = weight[idx >> 1u];
53
49
  let pair = unpack2x16float(packed);
54
- return select(pair.x, pair.y, (idx & 1u) == 1u);
50
+ return f16(select(pair.x, pair.y, (idx & 1u) == 1u));
55
51
  }
56
- return bitcast<f32>(weight[idx]);
52
+ return f16(bitcast<f32>(weight[idx]));
57
53
  }
58
54
 
59
55
  // Load input value, fusing residual add when PRE_RESIDUAL is active
60
- fn load_input(base_offset: u32, idx: u32) -> f32 {
61
- let x = f32(input[base_offset + idx]);
56
+ fn load_input(base_offset: u32, idx: u32) -> f16 {
57
+ let x = input[base_offset + idx];
62
58
  if (PRE_RESIDUAL) {
63
- return x + f32(residual[base_offset + idx]);
59
+ return x + residual[base_offset + idx];
64
60
  }
65
61
  return x;
66
62
  }
67
63
 
68
64
  // Write pre-norm sum when OUTPUT_PRENORM is active (for downstream residual reuse)
69
- fn write_prenorm(base_offset: u32, idx: u32, val: f32) {
65
+ fn write_prenorm(base_offset: u32, idx: u32, val: f16) {
70
66
  if (OUTPUT_PRENORM && PRE_RESIDUAL) {
71
- residual_sum_output[base_offset + idx] = f16(val);
67
+ residual_sum_output[base_offset + idx] = val;
72
68
  }
73
69
  }
74
70
 
@@ -93,16 +89,36 @@ fn main(
93
89
 
94
90
  let base_offset = token_idx * size;
95
91
 
96
- // Each thread computes partial sum of squares (promote to F32 for precision)
97
- // When PRE_RESIDUAL is active, sum is computed on (input + residual)
98
- var local_sum_sq: f32 = 0.0;
99
92
  let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
100
93
 
94
+ var local_max: f16 = f16(0.0);
101
95
  for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
102
96
  let idx = thread_idx * elements_per_thread + i;
103
97
  if (idx < size) {
104
- let x = load_input(base_offset, idx);
105
- local_sum_sq = local_sum_sq + x * x;
98
+ local_max = max(local_max, abs(load_input(base_offset, idx)));
99
+ }
100
+ }
101
+
102
+ shared_sum[thread_idx] = local_max;
103
+ workgroupBarrier();
104
+
105
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
106
+ if (thread_idx < stride) {
107
+ shared_sum[thread_idx] = max(shared_sum[thread_idx], shared_sum[thread_idx + stride]);
108
+ }
109
+ workgroupBarrier();
110
+ }
111
+
112
+ let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
113
+ workgroupBarrier();
114
+
115
+ var local_sum_sq: f16 = f16(0.0);
116
+ let inv_size = f16(1.0) / f16(size);
117
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
118
+ let idx = thread_idx * elements_per_thread + i;
119
+ if (idx < size) {
120
+ let x = load_input(base_offset, idx) / row_scale;
121
+ local_sum_sq = local_sum_sq + (x * x) * inv_size;
106
122
  }
107
123
  }
108
124
 
@@ -118,10 +134,9 @@ fn main(
118
134
  workgroupBarrier();
119
135
  }
120
136
 
121
- // Compute RMS (F32 for precision)
122
- let mean_sq = shared_sum[0] / f32(size);
123
- let rms = sqrt(mean_sq + u.eps);
124
- let inv_rms = 1.0 / rms;
137
+ let mean_sq = shared_sum[0];
138
+ let eps_scaled = f16(u.eps) / (row_scale * row_scale);
139
+ let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
125
140
 
126
141
  workgroupBarrier();
127
142
 
@@ -131,19 +146,17 @@ fn main(
131
146
  if (idx < size) {
132
147
  let x = load_input(base_offset, idx);
133
148
 
134
- // Normalize and scale (compute in F32)
135
149
  var result = x * inv_rms * apply_weight(load_weight(idx));
136
150
 
137
151
  // Add residual AFTER normalization
138
152
  if (u.has_residual == 1u) {
139
- result = result + f32(residual[base_offset + idx]);
153
+ result = result + residual[base_offset + idx];
140
154
  }
141
155
 
142
156
  // Write pre-norm sum for downstream residual reuse
143
157
  write_prenorm(base_offset, idx, x);
144
158
 
145
- // Convert back to F16 for output
146
- output[base_offset + idx] = f16(result);
159
+ output[base_offset + idx] = result;
147
160
  }
148
161
  }
149
162
  }
@@ -165,13 +178,25 @@ fn rmsnorm_small_f16(
165
178
  let base_offset = token_idx * size;
166
179
 
167
180
  // Each thread handles one element (for size <= 256)
168
- var x: f32 = 0.0;
181
+ var x: f16 = f16(0.0);
169
182
  if (thread_idx < size) {
170
183
  x = load_input(base_offset, thread_idx);
171
184
  }
172
185
 
173
- // Sum of squares (F32 for precision)
174
- shared_sum[thread_idx] = x * x;
186
+ shared_sum[thread_idx] = abs(x);
187
+ workgroupBarrier();
188
+
189
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
190
+ if (thread_idx < stride) {
191
+ shared_sum[thread_idx] = max(shared_sum[thread_idx], shared_sum[thread_idx + stride]);
192
+ }
193
+ workgroupBarrier();
194
+ }
195
+
196
+ let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
197
+ let x_scaled = x / row_scale;
198
+ let inv_size = f16(1.0) / f16(size);
199
+ shared_sum[thread_idx] = (x_scaled * x_scaled) * inv_size;
175
200
  workgroupBarrier();
176
201
 
177
202
  // Parallel reduction
@@ -183,16 +208,17 @@ fn rmsnorm_small_f16(
183
208
  }
184
209
 
185
210
  // Compute inverse RMS
186
- let mean_sq = shared_sum[0] / f32(size);
187
- let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
211
+ let mean_sq = shared_sum[0];
212
+ let eps_scaled = f16(u.eps) / (row_scale * row_scale);
213
+ let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
188
214
 
189
- // Apply normalization (compute in F32, output F16)
215
+ // Apply normalization
190
216
  if (thread_idx < size) {
191
217
  var result = x * inv_rms * apply_weight(load_weight(thread_idx));
192
218
  if (u.has_residual == 1u) {
193
- result = result + f32(residual[base_offset + thread_idx]);
219
+ result = result + residual[base_offset + thread_idx];
194
220
  }
195
221
  write_prenorm(base_offset, thread_idx, x);
196
- output[base_offset + thread_idx] = f16(result);
222
+ output[base_offset + thread_idx] = result;
197
223
  }
198
224
  }
@@ -17,17 +17,13 @@ export interface RoPEOptions extends OutputBufferOptions {
17
17
  numHeads?: number;
18
18
  headDim?: number;
19
19
  rotaryDim?: number;
20
+ pairSpanDim?: number;
20
21
  interleaved?: boolean;
21
22
  ropeTheta?: number;
22
23
  startPos?: number;
23
24
  executionPolicies?: ExecutionV1PoliciesSchema | null;
24
25
  }
25
26
 
26
- /** Context for RoPE variant selection rules. */
27
- export interface RoPESelectionContext {
28
- useF16: boolean;
29
- }
30
-
31
27
  /**
32
28
  * Run RoPE operation
33
29
  */
@@ -17,19 +17,28 @@ async function _rope(target, input, freqsCos, freqsSin, seqLen, options = {}) {
17
17
  numHeads = 1,
18
18
  headDim = 64,
19
19
  rotaryDim = headDim,
20
+ pairSpanDim = rotaryDim,
20
21
  interleaved = false,
21
22
  ropeTheta = ropeDefaults.defaultTheta,
22
23
  } = options;
23
24
 
24
- if (headDim % 2 !== 0) {
25
+ if (!Number.isFinite(headDim) || headDim % 2 !== 0) {
25
26
  throw new Error(`RoPE headDim must be even, got ${headDim}`);
26
27
  }
27
- if (rotaryDim % 2 !== 0) {
28
+ if (!Number.isFinite(rotaryDim) || rotaryDim % 2 !== 0) {
28
29
  throw new Error(`RoPE rotaryDim must be even, got ${rotaryDim}`);
29
30
  }
30
31
  if (rotaryDim <= 0 || rotaryDim > headDim) {
31
32
  throw new Error(`RoPE rotaryDim must be in (0, headDim]; got ${rotaryDim} for headDim ${headDim}`);
32
33
  }
34
+ if (!Number.isFinite(pairSpanDim) || pairSpanDim % 2 !== 0) {
35
+ throw new Error(`RoPE pairSpanDim must be even, got ${pairSpanDim}`);
36
+ }
37
+ if (pairSpanDim < rotaryDim || pairSpanDim > headDim) {
38
+ throw new Error(
39
+ `RoPE pairSpanDim must be in [rotaryDim, headDim]; got ${pairSpanDim} for rotaryDim ${rotaryDim} and headDim ${headDim}`
40
+ );
41
+ }
33
42
 
34
43
  const caps = getKernelCapabilities();
35
44
  const needsF32Cast = input.dtype === 'f16'
@@ -70,6 +79,7 @@ async function _rope(target, input, freqsCos, freqsSin, seqLen, options = {}) {
70
79
  rope_base: ropeTheta,
71
80
  rope_scale: 1.0,
72
81
  interleaved: interleaved ? 1 : 0,
82
+ pair_span_dim: pairSpanDim,
73
83
  },
74
84
  workgroups
75
85
  );
@@ -16,6 +16,11 @@
16
16
 
17
17
  override WORKGROUP_SIZE: u32 = 256u;
18
18
 
19
+ // YaRN parameters (model-family-specific, config-driven)
20
+ override YARN_BETA_FAST: f32 = 32.0;
21
+ override YARN_BETA_SLOW: f32 = 1.0;
22
+ override YARN_ALPHA: f32 = 1.0;
23
+
19
24
  // Mathematical constant
20
25
  const PI: f32 = 3.14159265359;
21
26
 
@@ -28,6 +33,10 @@ struct Uniforms {
28
33
  rope_scale: f32, // Scaling factor for extended context
29
34
  rotary_dim: u32, // Rotary slice within head_dim
30
35
  interleaved: u32, // 1 = adjacent pairs, 0 = rotate-half
36
+ pair_span_dim: u32, // Rotate-half partner span within head_dim
37
+ _pad0: u32,
38
+ _pad1: u32,
39
+ _pad2: u32,
31
40
  }
32
41
 
33
42
  @group(0) @binding(0) var<uniform> u: Uniforms;
@@ -42,11 +51,11 @@ fn get_first_rotary_idx(pair_idx: u32) -> u32 {
42
51
  return pair_idx;
43
52
  }
44
53
 
45
- fn get_second_rotary_idx(pair_idx: u32, head_dim: u32) -> u32 {
54
+ fn get_second_rotary_idx(pair_idx: u32, pair_span_dim: u32) -> u32 {
46
55
  if (u.interleaved == 1u) {
47
56
  return pair_idx * 2u + 1u;
48
57
  }
49
- return pair_idx + (head_dim / 2u);
58
+ return pair_idx + (pair_span_dim / 2u);
50
59
  }
51
60
 
52
61
  // Apply RoPE using precomputed frequencies
@@ -83,7 +92,7 @@ fn main(
83
92
 
84
93
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
85
94
  let first_idx = get_first_rotary_idx(pair_idx);
86
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
95
+ let second_idx = get_second_rotary_idx(pair_idx, u.pair_span_dim);
87
96
  let x0 = input[base_idx + first_idx];
88
97
  let x1 = input[base_idx + second_idx];
89
98
 
@@ -108,6 +117,7 @@ fn rope_compute_freqs(
108
117
  let rope_base = u.rope_base;
109
118
  let rope_scale = u.rope_scale;
110
119
  let rotary_dim = u.rotary_dim;
120
+ let pair_span_dim = u.pair_span_dim;
111
121
 
112
122
  let idx = global_id.x;
113
123
  let half_dim = rotary_dim / 2u;
@@ -126,7 +136,7 @@ fn rope_compute_freqs(
126
136
  let actual_pos = f32(start_pos + pos) / rope_scale;
127
137
 
128
138
  // Compute frequency: 1 / (base^(2*pair_idx/head_dim))
129
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
139
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
130
140
  let freq = 1.0 / pow(rope_base, exponent);
131
141
  let theta = actual_pos * freq;
132
142
 
@@ -135,7 +145,7 @@ fn rope_compute_freqs(
135
145
 
136
146
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
137
147
  let first_idx = get_first_rotary_idx(pair_idx);
138
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
148
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
139
149
  let x0 = input[base_idx + first_idx];
140
150
  let x1 = input[base_idx + second_idx];
141
151
 
@@ -157,6 +167,7 @@ fn rope_qk(
157
167
  let rope_base = u.rope_base;
158
168
  let rope_scale = u.rope_scale;
159
169
  let rotary_dim = u.rotary_dim;
170
+ let pair_span_dim = u.pair_span_dim;
160
171
 
161
172
  let idx = global_id.x;
162
173
  // Each thread handles one Q-K pair at one dimension pair
@@ -175,7 +186,7 @@ fn rope_qk(
175
186
  let actual_pos = f32(start_pos + pos) / rope_scale;
176
187
 
177
188
  // Compute frequency
178
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
189
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
179
190
  let freq = 1.0 / pow(rope_base, exponent);
180
191
  let theta = actual_pos * freq;
181
192
 
@@ -188,7 +199,7 @@ fn rope_qk(
188
199
 
189
200
  // Process Q
190
201
  let first_idx = get_first_rotary_idx(pair_idx);
191
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
202
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
192
203
  let q0 = input[q_base_idx + first_idx];
193
204
  let q1 = input[q_base_idx + second_idx];
194
205
  input[q_base_idx + first_idx] = q0 * cos_val - q1 * sin_val;
@@ -212,6 +223,7 @@ fn precompute_freqs(
212
223
  let rope_base = u.rope_base;
213
224
  let rope_scale = u.rope_scale;
214
225
  let rotary_dim = u.rotary_dim;
226
+ let pair_span_dim = u.pair_span_dim;
215
227
 
216
228
  let idx = global_id.x;
217
229
  let half_dim = rotary_dim / 2u;
@@ -225,7 +237,7 @@ fn precompute_freqs(
225
237
  let dim_idx = idx % half_dim;
226
238
 
227
239
  let actual_pos = f32(pos) / rope_scale;
228
- let exponent = f32(dim_idx * 2u) / f32(rotary_dim);
240
+ let exponent = f32(dim_idx * 2u) / f32(pair_span_dim);
229
241
  let freq = 1.0 / pow(rope_base, exponent);
230
242
  let theta = actual_pos * freq;
231
243
 
@@ -246,6 +258,7 @@ fn rope_ntk_scaled(
246
258
  let start_pos = u.start_pos;
247
259
  var rope_base = u.rope_base;
248
260
  let rope_scale = u.rope_scale;
261
+ let pair_span_dim = u.pair_span_dim;
249
262
 
250
263
  let idx = global_id.x;
251
264
  let half_dim = rotary_dim / 2u;
@@ -257,7 +270,7 @@ fn rope_ntk_scaled(
257
270
 
258
271
  // NTK scaling: increase base proportionally to scale factor
259
272
  // This preserves high-frequency components better than linear interpolation
260
- rope_base = rope_base * pow(rope_scale, f32(rotary_dim) / (f32(rotary_dim) - 2.0));
273
+ rope_base = rope_base * pow(rope_scale, f32(pair_span_dim) / (f32(pair_span_dim) - 2.0));
261
274
 
262
275
  let pos = idx / (num_heads * half_dim);
263
276
  let remainder = idx % (num_heads * half_dim);
@@ -266,7 +279,7 @@ fn rope_ntk_scaled(
266
279
 
267
280
  let actual_pos = f32(start_pos + pos);
268
281
 
269
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
282
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
270
283
  let freq = 1.0 / pow(rope_base, exponent);
271
284
  let theta = actual_pos * freq;
272
285
 
@@ -275,7 +288,7 @@ fn rope_ntk_scaled(
275
288
 
276
289
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
277
290
  let first_idx = get_first_rotary_idx(pair_idx);
278
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
291
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
279
292
  let x0 = input[base_idx + first_idx];
280
293
  let x1 = input[base_idx + second_idx];
281
294
 
@@ -296,6 +309,7 @@ fn rope_yarn(
296
309
  let start_pos = u.start_pos;
297
310
  let rope_base = u.rope_base;
298
311
  let rope_scale = u.rope_scale;
312
+ let pair_span_dim = u.pair_span_dim;
299
313
 
300
314
  let idx = global_id.x;
301
315
  let half_dim = rotary_dim / 2u;
@@ -312,13 +326,13 @@ fn rope_yarn(
312
326
 
313
327
  let actual_pos = f32(start_pos + pos);
314
328
 
315
- // YaRN parameters
316
- let beta_fast: f32 = 32.0;
317
- let beta_slow: f32 = 1.0;
318
- let alpha: f32 = 1.0;
329
+ // YaRN parameters (from override constants)
330
+ let beta_fast: f32 = YARN_BETA_FAST;
331
+ let beta_slow: f32 = YARN_BETA_SLOW;
332
+ let alpha: f32 = YARN_ALPHA;
319
333
 
320
334
  // Compute original frequency
321
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
335
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
322
336
  let orig_freq = 1.0 / pow(rope_base, exponent);
323
337
 
324
338
  // Compute wavelength
@@ -326,8 +340,8 @@ fn rope_yarn(
326
340
 
327
341
  // Interpolation factor based on wavelength
328
342
  var ramp: f32;
329
- let low_wavelength = f32(rotary_dim) / beta_fast;
330
- let high_wavelength = f32(rotary_dim) / beta_slow;
343
+ let low_wavelength = f32(pair_span_dim) / beta_fast;
344
+ let high_wavelength = f32(pair_span_dim) / beta_slow;
331
345
 
332
346
  if (wavelength < low_wavelength) {
333
347
  ramp = 0.0; // No interpolation for high frequencies
@@ -347,7 +361,7 @@ fn rope_yarn(
347
361
 
348
362
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
349
363
  let first_idx = get_first_rotary_idx(pair_idx);
350
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
364
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
351
365
  let x0 = input[base_idx + first_idx];
352
366
  let x1 = input[base_idx + second_idx];
353
367