doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -0,0 +1,41 @@
1
+ import type { Tensor } from '../tensor.js';
2
+ import type { CommandRecorder } from '../command-recorder.js';
3
+ import type { RoPEOptions } from './rope.js';
4
+
5
+ export interface RoPEQKOptions extends Omit<RoPEOptions, 'numHeads'> {
6
+ numQHeads: number;
7
+ numKVHeads: number;
8
+ headDim: number;
9
+ rotaryDim?: number;
10
+ pairSpanDim?: number;
11
+ }
12
+
13
+ export interface RoPEQKResult {
14
+ q: Tensor;
15
+ k: Tensor;
16
+ }
17
+
18
+ export function canUseRoPEQK(
19
+ qTensor: Tensor | null | undefined,
20
+ kTensor: Tensor | null | undefined,
21
+ options?: { reusesSharedKV?: boolean }
22
+ ): boolean;
23
+
24
+ export declare function runRoPEQK(
25
+ qTensor: Tensor,
26
+ kTensor: Tensor,
27
+ freqsCos: GPUBuffer | Tensor,
28
+ freqsSin: GPUBuffer | Tensor,
29
+ seqLen: number,
30
+ options: RoPEQKOptions
31
+ ): Promise<RoPEQKResult>;
32
+
33
+ export declare function recordRoPEQK(
34
+ recorder: CommandRecorder,
35
+ qTensor: Tensor,
36
+ kTensor: Tensor,
37
+ freqsCos: GPUBuffer | Tensor,
38
+ freqsSin: GPUBuffer | Tensor,
39
+ seqLen: number,
40
+ options: RoPEQKOptions
41
+ ): Promise<RoPEQKResult>;
@@ -0,0 +1,143 @@
1
+ import { createTensor, dtypeBytes } from '../tensor.js';
2
+ import { WORKGROUP_SIZES } from './constants.js';
3
+ import { unifiedKernelWrapper } from './utils.js';
4
+ import { selectRuleValue } from './rule-registry.js';
5
+
6
+ function assertPositiveInteger(value, label) {
7
+ if (!Number.isInteger(value) || value <= 0) {
8
+ throw new Error(`RoPE QK requires ${label} to be a positive integer.`);
9
+ }
10
+ }
11
+
12
+ function assertNonNegativeInteger(value, label) {
13
+ if (!Number.isInteger(value) || value < 0) {
14
+ throw new Error(`RoPE QK requires ${label} to be a non-negative integer.`);
15
+ }
16
+ }
17
+
18
+ function assertRopeGeometry(options) {
19
+ const { headDim, rotaryDim, pairSpanDim } = options;
20
+ assertPositiveInteger(headDim, 'headDim');
21
+ if ((headDim % 2) !== 0) {
22
+ throw new Error(`RoPE QK headDim must be even, got ${headDim}.`);
23
+ }
24
+ assertPositiveInteger(rotaryDim, 'rotaryDim');
25
+ if ((rotaryDim % 2) !== 0) {
26
+ throw new Error(`RoPE QK rotaryDim must be even, got ${rotaryDim}.`);
27
+ }
28
+ if (rotaryDim > headDim) {
29
+ throw new Error(`RoPE QK rotaryDim must be <= headDim; got ${rotaryDim} for headDim ${headDim}.`);
30
+ }
31
+ assertPositiveInteger(pairSpanDim, 'pairSpanDim');
32
+ if ((pairSpanDim % 2) !== 0) {
33
+ throw new Error(`RoPE QK pairSpanDim must be even, got ${pairSpanDim}.`);
34
+ }
35
+ if (pairSpanDim < rotaryDim || pairSpanDim > headDim) {
36
+ throw new Error(
37
+ `RoPE QK pairSpanDim must be in [rotaryDim, headDim]; got ${pairSpanDim} ` +
38
+ `for rotaryDim ${rotaryDim} and headDim ${headDim}.`
39
+ );
40
+ }
41
+ }
42
+
43
+ function bufferOf(bufferOrTensor) {
44
+ return bufferOrTensor?.buffer || bufferOrTensor;
45
+ }
46
+
47
+ function assertBufferCapacity(buffer, requiredBytes, label) {
48
+ if (Number.isFinite(buffer?.size) && requiredBytes > buffer.size) {
49
+ throw new Error(`RoPE QK ${label} buffer is smaller than requested range (${requiredBytes} > ${buffer.size} bytes).`);
50
+ }
51
+ }
52
+
53
+ function selectRoPEQKVariant(dtype) {
54
+ return selectRuleValue('ropeQk', 'variant', { useF16: dtype === 'f16' });
55
+ }
56
+
57
+ export function canUseRoPEQK(qTensor, kTensor, options = {}) {
58
+ if (options.reusesSharedKV === true) {
59
+ return false;
60
+ }
61
+ if (!qTensor || !kTensor || qTensor.buffer === kTensor.buffer) {
62
+ return false;
63
+ }
64
+ return qTensor.dtype === kTensor.dtype && (qTensor.dtype === 'f32' || qTensor.dtype === 'f16');
65
+ }
66
+
67
+ async function _ropeQK(target, qTensor, kTensor, freqsCos, freqsSin, seqLen, options = {}) {
68
+ const {
69
+ numQHeads,
70
+ numKVHeads,
71
+ headDim,
72
+ rotaryDim = headDim,
73
+ pairSpanDim = rotaryDim,
74
+ interleaved = false,
75
+ startPos = 0,
76
+ } = options;
77
+
78
+ assertNonNegativeInteger(seqLen, 'seqLen');
79
+ assertNonNegativeInteger(startPos, 'startPos');
80
+ assertPositiveInteger(numQHeads, 'numQHeads');
81
+ assertPositiveInteger(numKVHeads, 'numKVHeads');
82
+ assertRopeGeometry({ headDim, rotaryDim, pairSpanDim });
83
+ if (qTensor.dtype !== kTensor.dtype) {
84
+ throw new Error(`RoPE QK requires matching Q/K dtypes, got ${qTensor.dtype} and ${kTensor.dtype}.`);
85
+ }
86
+ if (qTensor.dtype !== 'f32' && qTensor.dtype !== 'f16') {
87
+ throw new Error(`RoPE QK requires f32 or f16 tensors, got ${qTensor.dtype}.`);
88
+ }
89
+ if (seqLen === 0) {
90
+ return {
91
+ q: createTensor(qTensor.buffer, qTensor.dtype, [...qTensor.shape], 'rope_q_output'),
92
+ k: createTensor(kTensor.buffer, kTensor.dtype, [...kTensor.shape], 'rope_k_output'),
93
+ };
94
+ }
95
+
96
+ const bytesPerElement = dtypeBytes(qTensor.dtype);
97
+ const qElementCount = seqLen * numQHeads * headDim;
98
+ const kElementCount = seqLen * numKVHeads * headDim;
99
+ const halfDim = rotaryDim / 2;
100
+ const freqElementCount = (startPos + seqLen) * halfDim;
101
+ assertBufferCapacity(qTensor.buffer, qElementCount * bytesPerElement, 'Q input');
102
+ assertBufferCapacity(kTensor.buffer, kElementCount * bytesPerElement, 'K input');
103
+ assertBufferCapacity(bufferOf(freqsCos), freqElementCount * 4, 'cos frequencies');
104
+ assertBufferCapacity(bufferOf(freqsSin), freqElementCount * 4, 'sin frequencies');
105
+
106
+ const variant = selectRoPEQKVariant(qTensor.dtype);
107
+ const totalPairs = seqLen * (numQHeads + numKVHeads) * halfDim;
108
+ const workgroups = Math.ceil(totalPairs / WORKGROUP_SIZES.DEFAULT);
109
+
110
+ await unifiedKernelWrapper(
111
+ 'rope_qk',
112
+ target,
113
+ variant,
114
+ [qTensor, kTensor, freqsCos, freqsSin],
115
+ {
116
+ seq_len: seqLen,
117
+ num_q_heads: numQHeads,
118
+ num_kv_heads: numKVHeads,
119
+ head_dim: headDim,
120
+ start_pos: startPos,
121
+ rotary_dim: rotaryDim,
122
+ interleaved: interleaved ? 1 : 0,
123
+ pair_span_dim: pairSpanDim,
124
+ },
125
+ workgroups,
126
+ null,
127
+ null,
128
+ 'rope_qk'
129
+ );
130
+
131
+ return {
132
+ q: createTensor(qTensor.buffer, qTensor.dtype, [...qTensor.shape], 'rope_q_output'),
133
+ k: createTensor(kTensor.buffer, kTensor.dtype, [...kTensor.shape], 'rope_k_output'),
134
+ };
135
+ }
136
+
137
+ export async function runRoPEQK(qTensor, kTensor, freqsCos, freqsSin, seqLen, options = {}) {
138
+ return _ropeQK(null, qTensor, kTensor, freqsCos, freqsSin, seqLen, options);
139
+ }
140
+
141
+ export async function recordRoPEQK(recorder, qTensor, kTensor, freqsCos, freqsSin, seqLen, options = {}) {
142
+ return _ropeQK(recorder, qTensor, kTensor, freqsCos, freqsSin, seqLen, options);
143
+ }
@@ -17,17 +17,13 @@ export interface RoPEOptions extends OutputBufferOptions {
17
17
  numHeads?: number;
18
18
  headDim?: number;
19
19
  rotaryDim?: number;
20
+ pairSpanDim?: number;
20
21
  interleaved?: boolean;
21
22
  ropeTheta?: number;
22
23
  startPos?: number;
23
24
  executionPolicies?: ExecutionV1PoliciesSchema | null;
24
25
  }
25
26
 
26
- /** Context for RoPE variant selection rules. */
27
- export interface RoPESelectionContext {
28
- useF16: boolean;
29
- }
30
-
31
27
  /**
32
28
  * Run RoPE operation
33
29
  */
@@ -17,19 +17,28 @@ async function _rope(target, input, freqsCos, freqsSin, seqLen, options = {}) {
17
17
  numHeads = 1,
18
18
  headDim = 64,
19
19
  rotaryDim = headDim,
20
+ pairSpanDim = rotaryDim,
20
21
  interleaved = false,
21
22
  ropeTheta = ropeDefaults.defaultTheta,
22
23
  } = options;
23
24
 
24
- if (headDim % 2 !== 0) {
25
+ if (!Number.isFinite(headDim) || headDim % 2 !== 0) {
25
26
  throw new Error(`RoPE headDim must be even, got ${headDim}`);
26
27
  }
27
- if (rotaryDim % 2 !== 0) {
28
+ if (!Number.isFinite(rotaryDim) || rotaryDim % 2 !== 0) {
28
29
  throw new Error(`RoPE rotaryDim must be even, got ${rotaryDim}`);
29
30
  }
30
31
  if (rotaryDim <= 0 || rotaryDim > headDim) {
31
32
  throw new Error(`RoPE rotaryDim must be in (0, headDim]; got ${rotaryDim} for headDim ${headDim}`);
32
33
  }
34
+ if (!Number.isFinite(pairSpanDim) || pairSpanDim % 2 !== 0) {
35
+ throw new Error(`RoPE pairSpanDim must be even, got ${pairSpanDim}`);
36
+ }
37
+ if (pairSpanDim < rotaryDim || pairSpanDim > headDim) {
38
+ throw new Error(
39
+ `RoPE pairSpanDim must be in [rotaryDim, headDim]; got ${pairSpanDim} for rotaryDim ${rotaryDim} and headDim ${headDim}`
40
+ );
41
+ }
33
42
 
34
43
  const caps = getKernelCapabilities();
35
44
  const needsF32Cast = input.dtype === 'f16'
@@ -70,6 +79,7 @@ async function _rope(target, input, freqsCos, freqsSin, seqLen, options = {}) {
70
79
  rope_base: ropeTheta,
71
80
  rope_scale: 1.0,
72
81
  interleaved: interleaved ? 1 : 0,
82
+ pair_span_dim: pairSpanDim,
73
83
  },
74
84
  workgroups
75
85
  );
@@ -16,6 +16,11 @@
16
16
 
17
17
  override WORKGROUP_SIZE: u32 = 256u;
18
18
 
19
+ // YaRN parameters (model-family-specific, config-driven)
20
+ override YARN_BETA_FAST: f32 = 32.0;
21
+ override YARN_BETA_SLOW: f32 = 1.0;
22
+ override YARN_ALPHA: f32 = 1.0;
23
+
19
24
  // Mathematical constant
20
25
  const PI: f32 = 3.14159265359;
21
26
 
@@ -28,6 +33,10 @@ struct Uniforms {
28
33
  rope_scale: f32, // Scaling factor for extended context
29
34
  rotary_dim: u32, // Rotary slice within head_dim
30
35
  interleaved: u32, // 1 = adjacent pairs, 0 = rotate-half
36
+ pair_span_dim: u32, // Rotate-half partner span within head_dim
37
+ _pad0: u32,
38
+ _pad1: u32,
39
+ _pad2: u32,
31
40
  }
32
41
 
33
42
  @group(0) @binding(0) var<uniform> u: Uniforms;
@@ -42,11 +51,11 @@ fn get_first_rotary_idx(pair_idx: u32) -> u32 {
42
51
  return pair_idx;
43
52
  }
44
53
 
45
- fn get_second_rotary_idx(pair_idx: u32, head_dim: u32) -> u32 {
54
+ fn get_second_rotary_idx(pair_idx: u32, pair_span_dim: u32) -> u32 {
46
55
  if (u.interleaved == 1u) {
47
56
  return pair_idx * 2u + 1u;
48
57
  }
49
- return pair_idx + (head_dim / 2u);
58
+ return pair_idx + (pair_span_dim / 2u);
50
59
  }
51
60
 
52
61
  // Apply RoPE using precomputed frequencies
@@ -83,7 +92,7 @@ fn main(
83
92
 
84
93
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
85
94
  let first_idx = get_first_rotary_idx(pair_idx);
86
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
95
+ let second_idx = get_second_rotary_idx(pair_idx, u.pair_span_dim);
87
96
  let x0 = input[base_idx + first_idx];
88
97
  let x1 = input[base_idx + second_idx];
89
98
 
@@ -108,6 +117,7 @@ fn rope_compute_freqs(
108
117
  let rope_base = u.rope_base;
109
118
  let rope_scale = u.rope_scale;
110
119
  let rotary_dim = u.rotary_dim;
120
+ let pair_span_dim = u.pair_span_dim;
111
121
 
112
122
  let idx = global_id.x;
113
123
  let half_dim = rotary_dim / 2u;
@@ -126,7 +136,7 @@ fn rope_compute_freqs(
126
136
  let actual_pos = f32(start_pos + pos) / rope_scale;
127
137
 
128
138
  // Compute frequency: 1 / (base^(2*pair_idx/head_dim))
129
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
139
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
130
140
  let freq = 1.0 / pow(rope_base, exponent);
131
141
  let theta = actual_pos * freq;
132
142
 
@@ -135,7 +145,7 @@ fn rope_compute_freqs(
135
145
 
136
146
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
137
147
  let first_idx = get_first_rotary_idx(pair_idx);
138
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
148
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
139
149
  let x0 = input[base_idx + first_idx];
140
150
  let x1 = input[base_idx + second_idx];
141
151
 
@@ -157,6 +167,7 @@ fn rope_qk(
157
167
  let rope_base = u.rope_base;
158
168
  let rope_scale = u.rope_scale;
159
169
  let rotary_dim = u.rotary_dim;
170
+ let pair_span_dim = u.pair_span_dim;
160
171
 
161
172
  let idx = global_id.x;
162
173
  // Each thread handles one Q-K pair at one dimension pair
@@ -175,7 +186,7 @@ fn rope_qk(
175
186
  let actual_pos = f32(start_pos + pos) / rope_scale;
176
187
 
177
188
  // Compute frequency
178
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
189
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
179
190
  let freq = 1.0 / pow(rope_base, exponent);
180
191
  let theta = actual_pos * freq;
181
192
 
@@ -188,7 +199,7 @@ fn rope_qk(
188
199
 
189
200
  // Process Q
190
201
  let first_idx = get_first_rotary_idx(pair_idx);
191
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
202
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
192
203
  let q0 = input[q_base_idx + first_idx];
193
204
  let q1 = input[q_base_idx + second_idx];
194
205
  input[q_base_idx + first_idx] = q0 * cos_val - q1 * sin_val;
@@ -212,6 +223,7 @@ fn precompute_freqs(
212
223
  let rope_base = u.rope_base;
213
224
  let rope_scale = u.rope_scale;
214
225
  let rotary_dim = u.rotary_dim;
226
+ let pair_span_dim = u.pair_span_dim;
215
227
 
216
228
  let idx = global_id.x;
217
229
  let half_dim = rotary_dim / 2u;
@@ -225,7 +237,7 @@ fn precompute_freqs(
225
237
  let dim_idx = idx % half_dim;
226
238
 
227
239
  let actual_pos = f32(pos) / rope_scale;
228
- let exponent = f32(dim_idx * 2u) / f32(rotary_dim);
240
+ let exponent = f32(dim_idx * 2u) / f32(pair_span_dim);
229
241
  let freq = 1.0 / pow(rope_base, exponent);
230
242
  let theta = actual_pos * freq;
231
243
 
@@ -246,6 +258,7 @@ fn rope_ntk_scaled(
246
258
  let start_pos = u.start_pos;
247
259
  var rope_base = u.rope_base;
248
260
  let rope_scale = u.rope_scale;
261
+ let pair_span_dim = u.pair_span_dim;
249
262
 
250
263
  let idx = global_id.x;
251
264
  let half_dim = rotary_dim / 2u;
@@ -257,7 +270,7 @@ fn rope_ntk_scaled(
257
270
 
258
271
  // NTK scaling: increase base proportionally to scale factor
259
272
  // This preserves high-frequency components better than linear interpolation
260
- rope_base = rope_base * pow(rope_scale, f32(rotary_dim) / (f32(rotary_dim) - 2.0));
273
+ rope_base = rope_base * pow(rope_scale, f32(pair_span_dim) / (f32(pair_span_dim) - 2.0));
261
274
 
262
275
  let pos = idx / (num_heads * half_dim);
263
276
  let remainder = idx % (num_heads * half_dim);
@@ -266,7 +279,7 @@ fn rope_ntk_scaled(
266
279
 
267
280
  let actual_pos = f32(start_pos + pos);
268
281
 
269
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
282
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
270
283
  let freq = 1.0 / pow(rope_base, exponent);
271
284
  let theta = actual_pos * freq;
272
285
 
@@ -275,7 +288,7 @@ fn rope_ntk_scaled(
275
288
 
276
289
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
277
290
  let first_idx = get_first_rotary_idx(pair_idx);
278
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
291
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
279
292
  let x0 = input[base_idx + first_idx];
280
293
  let x1 = input[base_idx + second_idx];
281
294
 
@@ -296,6 +309,7 @@ fn rope_yarn(
296
309
  let start_pos = u.start_pos;
297
310
  let rope_base = u.rope_base;
298
311
  let rope_scale = u.rope_scale;
312
+ let pair_span_dim = u.pair_span_dim;
299
313
 
300
314
  let idx = global_id.x;
301
315
  let half_dim = rotary_dim / 2u;
@@ -312,13 +326,13 @@ fn rope_yarn(
312
326
 
313
327
  let actual_pos = f32(start_pos + pos);
314
328
 
315
- // YaRN parameters
316
- let beta_fast: f32 = 32.0;
317
- let beta_slow: f32 = 1.0;
318
- let alpha: f32 = 1.0;
329
+ // YaRN parameters (from override constants)
330
+ let beta_fast: f32 = YARN_BETA_FAST;
331
+ let beta_slow: f32 = YARN_BETA_SLOW;
332
+ let alpha: f32 = YARN_ALPHA;
319
333
 
320
334
  // Compute original frequency
321
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
335
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
322
336
  let orig_freq = 1.0 / pow(rope_base, exponent);
323
337
 
324
338
  // Compute wavelength
@@ -326,8 +340,8 @@ fn rope_yarn(
326
340
 
327
341
  // Interpolation factor based on wavelength
328
342
  var ramp: f32;
329
- let low_wavelength = f32(rotary_dim) / beta_fast;
330
- let high_wavelength = f32(rotary_dim) / beta_slow;
343
+ let low_wavelength = f32(pair_span_dim) / beta_fast;
344
+ let high_wavelength = f32(pair_span_dim) / beta_slow;
331
345
 
332
346
  if (wavelength < low_wavelength) {
333
347
  ramp = 0.0; // No interpolation for high frequencies
@@ -347,7 +361,7 @@ fn rope_yarn(
347
361
 
348
362
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
349
363
  let first_idx = get_first_rotary_idx(pair_idx);
350
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
364
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
351
365
  let x0 = input[base_idx + first_idx];
352
366
  let x1 = input[base_idx + second_idx];
353
367
 
@@ -1,10 +1,10 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/rope.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Rotary Position Embeddings (RoPE) Kernel (F16)
4
4
  //
5
5
  // Applies rotary position embeddings to Q and K tensors.
6
6
  // Same math as rope.wgsl, but input/output are f16.
7
- // Computation is performed in f32 for stability.
7
+ // The main entry keeps apply-time arithmetic in f16 for the all-f16 lane.
8
8
  //
9
9
  // Supports:
10
10
  // - Original RoPE (base = 10000)
@@ -15,6 +15,11 @@ enable f16;
15
15
 
16
16
  override WORKGROUP_SIZE: u32 = 256u;
17
17
 
18
+ // YaRN parameters (model-family-specific, config-driven)
19
+ override YARN_BETA_FAST: f32 = 32.0;
20
+ override YARN_BETA_SLOW: f32 = 1.0;
21
+ override YARN_ALPHA: f32 = 1.0;
22
+
18
23
  // Mathematical constant
19
24
  const PI: f32 = 3.14159265359;
20
25
 
@@ -27,6 +32,10 @@ struct Uniforms {
27
32
  rope_scale: f32,
28
33
  rotary_dim: u32,
29
34
  interleaved: u32,
35
+ pair_span_dim: u32,
36
+ _pad0: u32,
37
+ _pad1: u32,
38
+ _pad2: u32,
30
39
  }
31
40
 
32
41
  @group(0) @binding(0) var<uniform> u: Uniforms;
@@ -41,11 +50,11 @@ fn get_first_rotary_idx(pair_idx: u32) -> u32 {
41
50
  return pair_idx;
42
51
  }
43
52
 
44
- fn get_second_rotary_idx(pair_idx: u32, head_dim: u32) -> u32 {
53
+ fn get_second_rotary_idx(pair_idx: u32, pair_span_dim: u32) -> u32 {
45
54
  if (u.interleaved == 1u) {
46
55
  return pair_idx * 2u + 1u;
47
56
  }
48
- return pair_idx + (head_dim / 2u);
57
+ return pair_idx + (pair_span_dim / 2u);
49
58
  }
50
59
 
51
60
  // Apply RoPE using precomputed frequencies
@@ -74,20 +83,20 @@ fn main(
74
83
  let actual_pos = start_pos + pos;
75
84
 
76
85
  let freq_idx = actual_pos * half_dim + pair_idx;
77
- let cos_val = freqs_cos[freq_idx];
78
- let sin_val = freqs_sin[freq_idx];
86
+ let cos_val = f16(freqs_cos[freq_idx]);
87
+ let sin_val = f16(freqs_sin[freq_idx]);
79
88
 
80
89
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
81
90
  let first_idx = get_first_rotary_idx(pair_idx);
82
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
83
- let x0 = f32(input[base_idx + first_idx]);
84
- let x1 = f32(input[base_idx + second_idx]);
91
+ let second_idx = get_second_rotary_idx(pair_idx, u.pair_span_dim);
92
+ let x0 = input[base_idx + first_idx];
93
+ let x1 = input[base_idx + second_idx];
85
94
 
86
95
  let y0 = x0 * cos_val - x1 * sin_val;
87
96
  let y1 = x0 * sin_val + x1 * cos_val;
88
97
 
89
- input[base_idx + first_idx] = f16(y0);
90
- input[base_idx + second_idx] = f16(y1);
98
+ input[base_idx + first_idx] = y0;
99
+ input[base_idx + second_idx] = y1;
91
100
  }
92
101
 
93
102
  // Compute frequencies on-the-fly (no precomputation needed)
@@ -102,6 +111,7 @@ fn rope_compute_freqs(
102
111
  let rope_base = u.rope_base;
103
112
  let rope_scale = u.rope_scale;
104
113
  let rotary_dim = u.rotary_dim;
114
+ let pair_span_dim = u.pair_span_dim;
105
115
 
106
116
  let idx = global_id.x;
107
117
  let half_dim = rotary_dim / 2u;
@@ -118,7 +128,7 @@ fn rope_compute_freqs(
118
128
 
119
129
  let actual_pos = f32(start_pos + pos) / rope_scale;
120
130
 
121
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
131
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
122
132
  let freq = 1.0 / pow(rope_base, exponent);
123
133
  let theta = actual_pos * freq;
124
134
 
@@ -127,7 +137,7 @@ fn rope_compute_freqs(
127
137
 
128
138
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
129
139
  let first_idx = get_first_rotary_idx(pair_idx);
130
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
140
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
131
141
  let x0 = f32(input[base_idx + first_idx]);
132
142
  let x1 = f32(input[base_idx + second_idx]);
133
143
 
@@ -150,6 +160,7 @@ fn rope_qk(
150
160
  let rope_base = u.rope_base;
151
161
  let rope_scale = u.rope_scale;
152
162
  let rotary_dim = u.rotary_dim;
163
+ let pair_span_dim = u.pair_span_dim;
153
164
 
154
165
  let idx = global_id.x;
155
166
  let half_dim = rotary_dim / 2u;
@@ -166,7 +177,7 @@ fn rope_qk(
166
177
 
167
178
  let actual_pos = f32(start_pos + pos) / rope_scale;
168
179
 
169
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
180
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
170
181
  let freq = 1.0 / pow(rope_base, exponent);
171
182
  let theta = actual_pos * freq;
172
183
 
@@ -177,7 +188,7 @@ fn rope_qk(
177
188
  let k_base_idx = q_base_idx + head_dim;
178
189
 
179
190
  let first_idx = get_first_rotary_idx(pair_idx);
180
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
191
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
181
192
  let q0 = f32(input[q_base_idx + first_idx]);
182
193
  let q1 = f32(input[q_base_idx + second_idx]);
183
194
  input[q_base_idx + first_idx] = f16(q0 * cos_val - q1 * sin_val);
@@ -198,6 +209,7 @@ fn precompute_freqs(
198
209
  let rope_base = u.rope_base;
199
210
  let rope_scale = u.rope_scale;
200
211
  let rotary_dim = u.rotary_dim;
212
+ let pair_span_dim = u.pair_span_dim;
201
213
 
202
214
  let idx = global_id.x;
203
215
  let half_dim = rotary_dim / 2u;
@@ -211,7 +223,7 @@ fn precompute_freqs(
211
223
  let dim_idx = idx % half_dim;
212
224
 
213
225
  let actual_pos = f32(pos) / rope_scale;
214
- let exponent = f32(dim_idx * 2u) / f32(rotary_dim);
226
+ let exponent = f32(dim_idx * 2u) / f32(pair_span_dim);
215
227
  let freq = 1.0 / pow(rope_base, exponent);
216
228
  let theta = actual_pos * freq;
217
229
 
@@ -230,6 +242,7 @@ fn rope_ntk_scaled(
230
242
  let start_pos = u.start_pos;
231
243
  var rope_base = u.rope_base;
232
244
  let rope_scale = u.rope_scale;
245
+ let pair_span_dim = u.pair_span_dim;
233
246
 
234
247
  let idx = global_id.x;
235
248
  let half_dim = rotary_dim / 2u;
@@ -239,7 +252,7 @@ fn rope_ntk_scaled(
239
252
  return;
240
253
  }
241
254
 
242
- rope_base = rope_base * pow(rope_scale, f32(rotary_dim) / (f32(rotary_dim) - 2.0));
255
+ rope_base = rope_base * pow(rope_scale, f32(pair_span_dim) / (f32(pair_span_dim) - 2.0));
243
256
 
244
257
  let pos = idx / (num_heads * half_dim);
245
258
  let remainder = idx % (num_heads * half_dim);
@@ -248,7 +261,7 @@ fn rope_ntk_scaled(
248
261
 
249
262
  let actual_pos = f32(start_pos + pos);
250
263
 
251
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
264
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
252
265
  let freq = 1.0 / pow(rope_base, exponent);
253
266
  let theta = actual_pos * freq;
254
267
 
@@ -257,7 +270,7 @@ fn rope_ntk_scaled(
257
270
 
258
271
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
259
272
  let first_idx = get_first_rotary_idx(pair_idx);
260
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
273
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
261
274
  let x0 = f32(input[base_idx + first_idx]);
262
275
  let x1 = f32(input[base_idx + second_idx]);
263
276
 
@@ -276,6 +289,7 @@ fn rope_yarn(
276
289
  let start_pos = u.start_pos;
277
290
  let rope_base = u.rope_base;
278
291
  let rope_scale = u.rope_scale;
292
+ let pair_span_dim = u.pair_span_dim;
279
293
 
280
294
  let idx = global_id.x;
281
295
  let half_dim = rotary_dim / 2u;
@@ -292,17 +306,17 @@ fn rope_yarn(
292
306
 
293
307
  let actual_pos = f32(start_pos + pos);
294
308
 
295
- let beta_fast: f32 = 32.0;
296
- let beta_slow: f32 = 1.0;
309
+ let beta_fast: f32 = YARN_BETA_FAST;
310
+ let beta_slow: f32 = YARN_BETA_SLOW;
297
311
 
298
- let exponent = f32(pair_idx * 2u) / f32(rotary_dim);
312
+ let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
299
313
  let orig_freq = 1.0 / pow(rope_base, exponent);
300
314
 
301
315
  let wavelength = 2.0 * PI / orig_freq;
302
316
 
303
317
  var ramp: f32;
304
- let low_wavelength = f32(rotary_dim) / beta_fast;
305
- let high_wavelength = f32(rotary_dim) / beta_slow;
318
+ let low_wavelength = f32(pair_span_dim) / beta_fast;
319
+ let high_wavelength = f32(pair_span_dim) / beta_slow;
306
320
 
307
321
  if (wavelength < low_wavelength) {
308
322
  ramp = 0.0;
@@ -321,7 +335,7 @@ fn rope_yarn(
321
335
 
322
336
  let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
323
337
  let first_idx = get_first_rotary_idx(pair_idx);
324
- let second_idx = get_second_rotary_idx(pair_idx, head_dim);
338
+ let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
325
339
  let x0 = f32(input[base_idx + first_idx]);
326
340
  let x1 = f32(input[base_idx + second_idx]);
327
341