doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -0,0 +1,86 @@
1
+ // ============================================================================
2
+ // Chat Templates
3
+ // ============================================================================
4
+
5
+ // Simple prompt templates for single-turn chat.
6
+ // For multi-turn conversations, use formatChatMessages from chat-format.js.
7
+
8
+ function applyTurnBasedTemplate(prompt) {
9
+ // Turn-based format: <start_of_turn>role\ncontent<end_of_turn>
10
+ const userTurn = `<start_of_turn>user\n${prompt}<end_of_turn>\n`;
11
+ const modelTurn = '<start_of_turn>model\n';
12
+ return userTurn + modelTurn;
13
+ }
14
+
15
+ function applyHeaderBasedTemplate(prompt) {
16
+ // Header-based format: <|start_header_id|>role<|end_header_id|>\n\ncontent<|eot_id|>
17
+ return `<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n${prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n`;
18
+ }
19
+
20
+ function applyChannelBasedTemplate(prompt) {
21
+ // Channel-based format: <|start|>role<|message|>content<|end|>
22
+ return `<|start|>user<|message|>${prompt}<|end|><|start|>assistant<|channel|>final<|message|>`;
23
+ }
24
+
25
+ function applyChatMLTemplate(prompt) {
26
+ // ChatML format: <|im_start|>role\ncontent<|im_end|>
27
+ return `<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n`;
28
+ }
29
+
30
+ function getQwenAssistantPrefix(options) {
31
+ return options?.thinking === true
32
+ ? '<|im_start|>assistant\n<think>\n'
33
+ : '<|im_start|>assistant\n<think>\n\n</think>\n\n';
34
+ }
35
+
36
+ function applyQwenTemplate(prompt, options) {
37
+ return `<|im_start|>user\n${prompt}<|im_end|>\n${getQwenAssistantPrefix(options)}`;
38
+ }
39
+
40
+ function applyGemma4Template(prompt) {
41
+ return `<bos><|turn>user\n${prompt}<turn|>\n<|turn>model\n<|channel>thought\n<channel|>`;
42
+ }
43
+
44
+ function applyTranslateGemmaTemplate() {
45
+ throw new Error(
46
+ 'TranslateGemma template requires structured messages. ' +
47
+ 'Use formatChatMessages(messages, "translategemma") instead of applyChatTemplate(prompt, ...).'
48
+ );
49
+ }
50
+
51
+ // Template type to formatter mapping.
52
+ // Add new template types here rather than adding switch cases.
53
+ const PROMPT_TEMPLATES = {
54
+ 'gemma': applyTurnBasedTemplate,
55
+ 'gemma4': applyGemma4Template,
56
+ 'llama3': applyHeaderBasedTemplate,
57
+ 'gpt-oss': applyChannelBasedTemplate,
58
+ 'chatml': applyChatMLTemplate,
59
+ 'qwen': applyQwenTemplate,
60
+ 'translategemma': applyTranslateGemmaTemplate,
61
+ };
62
+
63
+ export function applyChatTemplate(prompt, templateType, options = undefined) {
64
+ if (templateType == null) {
65
+ return prompt;
66
+ }
67
+ const formatter = PROMPT_TEMPLATES[templateType];
68
+ if (formatter) {
69
+ return formatter(prompt, options);
70
+ }
71
+ throw new Error(`Unrecognized chat template type: ${templateType}`);
72
+ }
73
+
74
+ // Exports preserved for existing external imports.
75
+ export const applyGemmaChatTemplate = applyTurnBasedTemplate;
76
+ export const applyGemma4ChatTemplate = applyGemma4Template;
77
+ export const applyLlama3ChatTemplate = applyHeaderBasedTemplate;
78
+ export const applyGptOssChatTemplate = applyChannelBasedTemplate;
79
+ export const applyQwenChatTemplate = applyQwenTemplate;
80
+
81
+
82
+ export function isStopToken(token, stopTokenIds, eosTokenId) {
83
+ if (stopTokenIds.includes(token)) return true;
84
+ if (typeof eosTokenId === 'number' && token === eosTokenId) return true;
85
+ return false;
86
+ }
@@ -0,0 +1,201 @@
1
+ import { getDevice } from '../../../gpu/device.js';
2
+ import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
3
+ import { getKernelPathMatmulVariant } from '../../../config/kernel-path-loader.js';
4
+ import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
5
+ import { selectRuleValue } from '../../../rules/rule-registry.js';
6
+ import { log } from '../../../debug/index.js';
7
+
8
+ // ============================================================================
9
+ // QKV Fusion
10
+ // ============================================================================
11
+
12
+
13
+ function kernelPathVariantRequiresQ4KWeights(variant) {
14
+ if (typeof variant !== 'string' || variant.length === 0) {
15
+ return false;
16
+ }
17
+ let config;
18
+ try {
19
+ config = getKernelConfig('matmul', variant);
20
+ } catch {
21
+ return false;
22
+ }
23
+ const shaderFile = String(config?.shaderFile ?? config?.wgsl ?? '');
24
+ return shaderFile.startsWith('fused_matmul_q4');
25
+ }
26
+
27
+ function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
28
+ if (!kernelPath) {
29
+ return false;
30
+ }
31
+ const prefillVariant = getKernelPathMatmulVariant('qkv_proj', 'prefill', layerIdx, kernelPath);
32
+ if (kernelPathVariantRequiresQ4KWeights(prefillVariant)) {
33
+ return true;
34
+ }
35
+ const decodeVariant = getKernelPathMatmulVariant('qkv_proj', 'decode', layerIdx, kernelPath);
36
+ return kernelPathVariantRequiresQ4KWeights(decodeVariant);
37
+ }
38
+
39
+ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
40
+ const device = getDevice();
41
+ if (!device) {
42
+ log.debug('QKV Fusion', 'No GPU device, skipping fusion');
43
+ return;
44
+ }
45
+ if (
46
+ Number.isFinite(modelConfig?.globalHeadDim)
47
+ && modelConfig.globalHeadDim !== modelConfig.headDim
48
+ ) {
49
+ log.debug(
50
+ 'QKV Fusion',
51
+ 'Skipping QKV fusion for mixed-head-dim model; per-layer attention geometry must stay explicit.'
52
+ );
53
+ return;
54
+ }
55
+
56
+ const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
57
+ const qSize = numHeads * headDim;
58
+ const kSize = numKVHeads * headDim;
59
+ const vSize = numKVHeads * headDim;
60
+ const qkvSize = qSize + kSize + vSize;
61
+
62
+ const resolveWeight = (value) => {
63
+ if (isWeightBuffer(value)) {
64
+ return {
65
+ buffer: value.buffer,
66
+ dtype: value.dtype ?? null,
67
+ layout: value.layout ?? null,
68
+ shape: Array.isArray(value.shape) ? value.shape : null,
69
+ };
70
+ }
71
+ if (isGpuBufferInstance(value)) {
72
+ return {
73
+ buffer: value,
74
+ dtype: getWeightDtype(value),
75
+ layout: null,
76
+ shape: null,
77
+ };
78
+ }
79
+ return null;
80
+ };
81
+
82
+ log.debug('QKV Fusion', `Fusing Q/K/V weights for ${numLayers} layers (${qSize}+${kSize}+${vSize}=${qkvSize})`);
83
+
84
+ const resolveBytesPerElement = (weight, expectedElements) => {
85
+ const dtype = typeof weight?.dtype === 'string'
86
+ ? weight.dtype.toLowerCase()
87
+ : null;
88
+ if (dtype === 'f16' || dtype === 'bf16') return 2;
89
+ if (dtype === 'f32') return 4;
90
+ const minF16Bytes = expectedElements * 2;
91
+ const minF32Bytes = expectedElements * 4;
92
+ if (weight?.buffer?.size >= minF32Bytes) return 4;
93
+ if (weight?.buffer?.size >= minF16Bytes) return 2;
94
+ return 0;
95
+ };
96
+
97
+ let fusedCount = 0;
98
+ for (let l = 0; l < numLayers; l++) {
99
+ const weights = layerWeights.get(`layer_${l}`);
100
+ if (!weights) continue;
101
+
102
+ // Skip if already fused or if weights are not GPUBuffers
103
+ if (weights.qkvProj) continue;
104
+ if (shouldSkipQKVFusionForLayer(l, kernelPath)) {
105
+ log.debug(
106
+ 'QKV Fusion',
107
+ `Layer ${l}: skipped because active kernel path requires Q4K weights for qkv_proj`
108
+ );
109
+ continue;
110
+ }
111
+ const qProj = resolveWeight(weights.qProj);
112
+ const kProj = resolveWeight(weights.kProj);
113
+ const vProj = resolveWeight(weights.vProj);
114
+ if (!qProj || !kProj || !vProj) {
115
+ continue;
116
+ }
117
+
118
+ const qExpectedElements = qSize * hiddenSize;
119
+ const kExpectedElements = kSize * hiddenSize;
120
+ const vExpectedElements = vSize * hiddenSize;
121
+ const bytesPerElement = resolveBytesPerElement(qProj, qExpectedElements);
122
+ const kBytesPerElement = resolveBytesPerElement(kProj, kExpectedElements);
123
+ const vBytesPerElement = resolveBytesPerElement(vProj, vExpectedElements);
124
+
125
+ // Pool allocation can round GPUBuffer.size up, so infer logical dtype first and
126
+ // only use buffer size as a minimum-size inference.
127
+ if ((bytesPerElement !== 2 && bytesPerElement !== 4)
128
+ || kBytesPerElement !== bytesPerElement
129
+ || vBytesPerElement !== bytesPerElement) {
130
+ log.debug(
131
+ 'QKV Fusion',
132
+ `Layer ${l}: inconsistent projection dtypes (q=${bytesPerElement}, k=${kBytesPerElement}, v=${vBytesPerElement}), skipping`
133
+ );
134
+ continue;
135
+ }
136
+
137
+ const normalizedDtype = typeof qProj.dtype === 'string'
138
+ ? qProj.dtype.toLowerCase()
139
+ : null;
140
+ const dtype = normalizedDtype === 'bf16'
141
+ ? 'f16'
142
+ : (
143
+ normalizedDtype === 'f16' || normalizedDtype === 'f32'
144
+ ? normalizedDtype
145
+ : selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement })
146
+ );
147
+ const layout = qProj.layout ?? kProj.layout ?? vProj.layout ?? 'row';
148
+ let fusedShape = [qkvSize, hiddenSize];
149
+ if (Array.isArray(qProj.shape) && qProj.shape.length === 2) {
150
+ if (qProj.shape[0] === qSize && qProj.shape[1] === hiddenSize) {
151
+ fusedShape = [qkvSize, hiddenSize];
152
+ } else if (qProj.shape[1] === qSize && qProj.shape[0] === hiddenSize) {
153
+ fusedShape = [hiddenSize, qkvSize];
154
+ }
155
+ }
156
+
157
+ // Create fused QKV buffer: [qkvSize, hiddenSize] row-major
158
+ // Each row is concatenated: [q_row, k_row, v_row]
159
+ const qkvBuffer = device.createBuffer({
160
+ label: `layer_${l}_qkv_proj`,
161
+ size: qkvSize * hiddenSize * bytesPerElement,
162
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
163
+ });
164
+
165
+ // Copy Q, K, V weights into fused buffer
166
+ // Q: [qSize, hiddenSize] -> offset 0
167
+ // K: [kSize, hiddenSize] -> offset qSize * hiddenSize * bytesPerElement
168
+ // V: [vSize, hiddenSize] -> offset (qSize + kSize) * hiddenSize * bytesPerElement
169
+ const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
170
+ encoder.copyBufferToBuffer(
171
+ qProj.buffer, 0,
172
+ qkvBuffer, 0,
173
+ qSize * hiddenSize * bytesPerElement
174
+ );
175
+ encoder.copyBufferToBuffer(
176
+ kProj.buffer, 0,
177
+ qkvBuffer, qSize * hiddenSize * bytesPerElement,
178
+ kSize * hiddenSize * bytesPerElement
179
+ );
180
+ encoder.copyBufferToBuffer(
181
+ vProj.buffer, 0,
182
+ qkvBuffer, (qSize + kSize) * hiddenSize * bytesPerElement,
183
+ vSize * hiddenSize * bytesPerElement
184
+ );
185
+ device.queue.submit([encoder.finish()]);
186
+
187
+ // Store fused buffer, sizes, and dtype
188
+ weights.qkvProj = createWeightBuffer(
189
+ qkvBuffer,
190
+ dtype,
191
+ layout,
192
+ fusedShape,
193
+ `layer_${l}_qkv_proj`
194
+ );
195
+ weights.qkvSizes = [qSize, kSize, vSize];
196
+ weights.qkvDtype = dtype;
197
+ fusedCount++;
198
+ }
199
+
200
+ log.debug('QKV Fusion', `Fused ${fusedCount}/${numLayers} layers`);
201
+ }
@@ -0,0 +1,408 @@
1
+ import { getDevice } from '../../../gpu/device.js';
2
+ import { acquireBuffer, isBufferActive, releaseBuffer } from '../../../memory/buffer-pool.js';
3
+ import { isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
4
+ import { log } from '../../../debug/index.js';
5
+
6
+ // ============================================================================
7
+ // RoPE Initialization
8
+ // ============================================================================
9
+
10
+
11
+ function computeRoPEFreqsForTheta(
12
+ theta,
13
+ rotaryDim,
14
+ frequencyBaseDim,
15
+ maxSeqLen,
16
+ ropeScale,
17
+ ropeScalingType,
18
+ ropeScaling
19
+ ) {
20
+ const halfDim = rotaryDim / 2;
21
+
22
+ // Compute base frequencies: theta_i = 1 / (base^(2i/d))
23
+ const freqs = new Float32Array(halfDim);
24
+ for (let i = 0; i < halfDim; i++) {
25
+ freqs[i] = 1.0 / Math.pow(theta, (2 * i) / frequencyBaseDim);
26
+ }
27
+
28
+ // Compute per-dimension scaling factors
29
+ const scales = new Float32Array(halfDim);
30
+ const isYarn = ropeScalingType === 'yarn';
31
+ if (isYarn) {
32
+ // YARN scaling - validate ALL required params (fail fast on incomplete manifest)
33
+ if (ropeScaling?.factor == null ||
34
+ ropeScaling?.beta_fast == null || ropeScaling?.beta_slow == null ||
35
+ ropeScaling?.original_max_position_embeddings == null) {
36
+ throw new Error(
37
+ `RoPE scaling type is 'yarn' but YARN params missing. ` +
38
+ `Manifest must provide factor, beta_fast, beta_slow, and original_max_position_embeddings. ` +
39
+ `Got: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}, ` +
40
+ `original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
41
+ );
42
+ }
43
+ // Extract validated YARN params (no hidden defaults - all guaranteed non-null)
44
+ const yarnFactor = ropeScaling.factor;
45
+ const yarnBetaFast = ropeScaling.beta_fast;
46
+ const yarnBetaSlow = ropeScaling.beta_slow;
47
+ const originalMaxPos = ropeScaling.original_max_position_embeddings;
48
+
49
+ // YARN: wavelength-based interpolation
50
+ for (let i = 0; i < halfDim; i++) {
51
+ const wavelength = (2 * Math.PI) / freqs[i];
52
+ const lowThresh = originalMaxPos / yarnBetaSlow;
53
+ const highThresh = originalMaxPos / yarnBetaFast;
54
+
55
+ if (wavelength < highThresh) {
56
+ scales[i] = 1.0;
57
+ } else if (wavelength > lowThresh) {
58
+ scales[i] = yarnFactor;
59
+ } else {
60
+ const t = (wavelength - highThresh) / (lowThresh - highThresh);
61
+ scales[i] = 1.0 + (yarnFactor - 1.0) * t;
62
+ }
63
+ }
64
+ } else {
65
+ // Linear scaling: uniform across all dimensions
66
+ if (ropeScalingType != null && ropeScalingType !== 'linear') {
67
+ log.warn(
68
+ 'Pipeline',
69
+ `Unrecognized RoPE scaling type "${ropeScalingType}"; falling back to linear scaling. ` +
70
+ 'Known types: "linear", "yarn".'
71
+ );
72
+ }
73
+ for (let i = 0; i < halfDim; i++) {
74
+ scales[i] = ropeScale;
75
+ }
76
+ }
77
+
78
+ // Compute cos/sin for each position
79
+ const cosValues = new Float32Array(maxSeqLen * halfDim);
80
+ const sinValues = new Float32Array(maxSeqLen * halfDim);
81
+
82
+ for (let pos = 0; pos < maxSeqLen; pos++) {
83
+ for (let i = 0; i < halfDim; i++) {
84
+ const scaledPos = pos / scales[i];
85
+ const angle = scaledPos * freqs[i];
86
+ cosValues[pos * halfDim + i] = Math.cos(angle);
87
+ sinValues[pos * halfDim + i] = Math.sin(angle);
88
+ }
89
+ }
90
+
91
+ return { cos: cosValues, sin: sinValues };
92
+ }
93
+
94
+ function isSameRoPEScalingConfig(
95
+ leftType,
96
+ leftScale,
97
+ leftScaling,
98
+ rightType,
99
+ rightScale,
100
+ rightScaling
101
+ ) {
102
+ if (leftType !== rightType) return false;
103
+ if (leftScale !== rightScale) return false;
104
+ if (leftType !== 'yarn') return true;
105
+ return (leftScaling?.beta_fast ?? null) === (rightScaling?.beta_fast ?? null)
106
+ && (leftScaling?.beta_slow ?? null) === (rightScaling?.beta_slow ?? null)
107
+ && (leftScaling?.original_max_position_embeddings ?? null)
108
+ === (rightScaling?.original_max_position_embeddings ?? null);
109
+ }
110
+
111
+ const GPU_ROPE_BUFFER_CACHE = new WeakMap();
112
+ const CPU_ROPE_BUFFER_CACHE = new Map();
113
+
114
+ function isLiveCachedRopeBuffer(buffer) {
115
+ return buffer == null || isBufferActive(buffer);
116
+ }
117
+
118
+ function hasLiveCachedGpuRopeBuffers(buffers) {
119
+ return !!buffers
120
+ && isLiveCachedRopeBuffer(buffers.cos)
121
+ && isLiveCachedRopeBuffer(buffers.sin)
122
+ && isLiveCachedRopeBuffer(buffers.localCos)
123
+ && isLiveCachedRopeBuffer(buffers.localSin);
124
+ }
125
+
126
+ function buildRoPECacheKey(config) {
127
+ return JSON.stringify({
128
+ headDim: config.headDim,
129
+ localHeadDim: config.localHeadDim ?? null,
130
+ rotaryDim: config.rotaryDim ?? null,
131
+ ropeLocalRotaryDim: config.ropeLocalRotaryDim ?? null,
132
+ ropeFrequencyBaseDim: config.ropeFrequencyBaseDim ?? null,
133
+ ropeLocalFrequencyBaseDim: config.ropeLocalFrequencyBaseDim ?? null,
134
+ maxSeqLen: config.maxSeqLen,
135
+ ropeTheta: config.ropeTheta,
136
+ ropeLocalTheta: config.ropeLocalTheta ?? null,
137
+ mropeInterleaved: config.mropeInterleaved === true,
138
+ mropeSection: Array.isArray(config.mropeSection) ? [...config.mropeSection] : null,
139
+ partialRotaryFactor: config.partialRotaryFactor ?? null,
140
+ ropeLocalPartialRotaryFactor: config.ropeLocalPartialRotaryFactor ?? null,
141
+ ropeScale: config.ropeScale,
142
+ ropeLocalScale: config.ropeLocalScale ?? null,
143
+ ropeScalingType: config.ropeScalingType ?? null,
144
+ ropeLocalScalingType: config.ropeLocalScalingType ?? null,
145
+ ropeScaling: config.ropeScaling ?? null,
146
+ ropeLocalScaling: config.ropeLocalScaling ?? null,
147
+ });
148
+ }
149
+
150
+ function resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor) {
151
+ if (rotaryDim != null) {
152
+ if (!Number.isFinite(rotaryDim) || rotaryDim <= 0 || (rotaryDim % 2) !== 0) {
153
+ throw new Error(`RoPE rotary dim must be a positive even integer; got "${rotaryDim}".`);
154
+ }
155
+ if (rotaryDim > headDim) {
156
+ throw new Error(`RoPE rotary dim ${rotaryDim} cannot exceed headDim ${headDim}.`);
157
+ }
158
+ return rotaryDim;
159
+ }
160
+ if (partialRotaryFactor == null) {
161
+ return headDim;
162
+ }
163
+ if (!Number.isFinite(partialRotaryFactor) || partialRotaryFactor <= 0 || partialRotaryFactor > 1) {
164
+ throw new Error(
165
+ `RoPE partialRotaryFactor must be a number in (0, 1]; got "${partialRotaryFactor}".`
166
+ );
167
+ }
168
+ const resolved = Math.trunc(headDim * partialRotaryFactor);
169
+ if (resolved <= 0 || (resolved % 2) !== 0) {
170
+ throw new Error(
171
+ `RoPE partialRotaryFactor=${partialRotaryFactor} with headDim=${headDim} resolves ` +
172
+ `to rotaryDim=${resolved}, but rotaryDim must be a positive even integer.`
173
+ );
174
+ }
175
+ return resolved;
176
+ }
177
+
178
+ function resolveFrequencyBaseDim(rotaryDim, frequencyBaseDim, label) {
179
+ if (frequencyBaseDim == null) {
180
+ return rotaryDim;
181
+ }
182
+ if (!Number.isFinite(frequencyBaseDim) || frequencyBaseDim <= 0 || (Math.trunc(frequencyBaseDim) % 2) !== 0) {
183
+ throw new Error(`${label} must be a positive even integer; got "${frequencyBaseDim}".`);
184
+ }
185
+ const resolved = Math.trunc(frequencyBaseDim);
186
+ if (resolved < rotaryDim) {
187
+ throw new Error(`${label} ${resolved} cannot be smaller than rotaryDim ${rotaryDim}.`);
188
+ }
189
+ return resolved;
190
+ }
191
+
192
+
193
+ export async function initRoPEFrequencies(config, useGPU) {
194
+ const cacheKey = buildRoPECacheKey(config);
195
+ const {
196
+ headDim,
197
+ localHeadDim,
198
+ rotaryDim,
199
+ ropeLocalRotaryDim,
200
+ ropeFrequencyBaseDim,
201
+ ropeLocalFrequencyBaseDim,
202
+ maxSeqLen,
203
+ ropeTheta,
204
+ ropeLocalTheta,
205
+ mropeInterleaved,
206
+ mropeSection,
207
+ partialRotaryFactor,
208
+ ropeLocalPartialRotaryFactor,
209
+ ropeScale,
210
+ ropeLocalScale,
211
+ ropeScalingType,
212
+ ropeLocalScalingType,
213
+ ropeScaling,
214
+ ropeLocalScaling,
215
+ } = config;
216
+ if (!Number.isFinite(ropeScale) || ropeScale <= 0) {
217
+ throw new Error(`RoPE scale must be a positive number; got "${ropeScale}".`);
218
+ }
219
+ const resolvedLocalScale = ropeLocalScale;
220
+ if (resolvedLocalScale != null && (!Number.isFinite(resolvedLocalScale) || resolvedLocalScale <= 0)) {
221
+ throw new Error(`Local RoPE scale must be a positive number; got "${resolvedLocalScale}".`);
222
+ }
223
+ const resolvedLocalTheta = ropeLocalTheta ?? ropeTheta;
224
+ const resolvedLocalScalingType = (
225
+ ropeLocalScalingType === undefined
226
+ ? ropeScalingType
227
+ : ropeLocalScalingType
228
+ );
229
+ const resolvedLocalScaling = (
230
+ ropeLocalScalingType === undefined
231
+ ? ropeScaling
232
+ : ropeLocalScaling
233
+ );
234
+ const resolvedLocalHeadDim = localHeadDim ?? headDim;
235
+ const resolvedRotaryDim = resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor);
236
+ const resolvedLocalRotaryDim = resolveRotaryDim(
237
+ resolvedLocalHeadDim,
238
+ ropeLocalRotaryDim,
239
+ ropeLocalPartialRotaryFactor
240
+ );
241
+ const resolvedFrequencyBaseDim = resolveFrequencyBaseDim(
242
+ resolvedRotaryDim,
243
+ ropeFrequencyBaseDim,
244
+ 'RoPE frequency base dim'
245
+ );
246
+ const resolvedLocalFrequencyBaseDim = resolveFrequencyBaseDim(
247
+ resolvedLocalRotaryDim,
248
+ ropeLocalFrequencyBaseDim,
249
+ 'Local RoPE frequency base dim'
250
+ );
251
+ const halfDim = resolvedRotaryDim / 2;
252
+ if (mropeInterleaved === true && Array.isArray(mropeSection)) {
253
+ const expandedDim = mropeSection.reduce((sum, entry) => sum + entry, 0) * 2;
254
+ if (expandedDim !== resolvedRotaryDim) {
255
+ throw new Error(
256
+ `RoPE mropeSection expands to ${expandedDim} dims, but rotaryDim is ${resolvedRotaryDim}.`
257
+ );
258
+ }
259
+ }
260
+
261
+ const isYarn = ropeScalingType === 'yarn';
262
+ const isLocalYarn = resolvedLocalScalingType === 'yarn';
263
+
264
+ // Compute global (full_attention) frequencies
265
+ const globalFreqs = computeRoPEFreqsForTheta(
266
+ ropeTheta,
267
+ resolvedRotaryDim,
268
+ resolvedFrequencyBaseDim,
269
+ maxSeqLen,
270
+ ropeScale,
271
+ ropeScalingType,
272
+ ropeScaling
273
+ );
274
+
275
+ // Compute local (sliding_attention) frequencies if different from global.
276
+ // Models with dual RoPE use different theta for local vs global attention layers.
277
+
278
+ let localFreqs = null;
279
+ const hasDistinctLocalTheta = resolvedLocalTheta !== ropeTheta;
280
+ const hasDistinctLocalDim = resolvedLocalRotaryDim !== resolvedRotaryDim;
281
+ const hasDistinctLocalScaling = !isSameRoPEScalingConfig(
282
+ ropeScalingType,
283
+ ropeScale,
284
+ ropeScaling,
285
+ resolvedLocalScalingType,
286
+ resolvedLocalScale,
287
+ resolvedLocalScaling
288
+ );
289
+ if (hasDistinctLocalTheta || hasDistinctLocalScaling || hasDistinctLocalDim) {
290
+ localFreqs = computeRoPEFreqsForTheta(
291
+ resolvedLocalTheta,
292
+ resolvedLocalRotaryDim,
293
+ resolvedLocalFrequencyBaseDim,
294
+ maxSeqLen,
295
+ resolvedLocalScale,
296
+ resolvedLocalScalingType,
297
+ resolvedLocalScaling
298
+ );
299
+ log.debug(
300
+ 'Pipeline',
301
+ `Dual RoPE: local theta=${resolvedLocalTheta}, global theta=${ropeTheta}, ` +
302
+ `localRotaryDim=${resolvedLocalRotaryDim}, globalRotaryDim=${resolvedRotaryDim}, ` +
303
+ `localFrequencyBaseDim=${resolvedLocalFrequencyBaseDim}, globalFrequencyBaseDim=${resolvedFrequencyBaseDim}, ` +
304
+ `localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}, ` +
305
+ `globalScaling=${ropeScalingType ?? 'none'}:${ropeScale}`
306
+ );
307
+ }
308
+
309
+ if (isYarn) {
310
+ // Log YARN params (already validated in computeRoPEFreqs)
311
+ log.debug('Pipeline', `YARN RoPE: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}`);
312
+ }
313
+ if (isLocalYarn && hasDistinctLocalScaling) {
314
+ log.debug(
315
+ 'Pipeline',
316
+ `Local YARN RoPE: factor=${resolvedLocalScaling?.factor}, ` +
317
+ `beta_fast=${resolvedLocalScaling?.beta_fast}, beta_slow=${resolvedLocalScaling?.beta_slow}`
318
+ );
319
+ }
320
+
321
+ // Upload to GPU if available
322
+ const device = getDevice();
323
+ if (device && useGPU) {
324
+ let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
325
+ if (!perDeviceCache) {
326
+ perDeviceCache = new Map();
327
+ GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
328
+ }
329
+ const cachedBuffers = perDeviceCache.get(cacheKey);
330
+ if (cachedBuffers) {
331
+ if (hasLiveCachedGpuRopeBuffers(cachedBuffers)) {
332
+ return cachedBuffers;
333
+ }
334
+ perDeviceCache.delete(cacheKey);
335
+ }
336
+ let cosBuffer = null;
337
+ let sinBuffer = null;
338
+ let localCosBuffer = null;
339
+ let localSinBuffer = null;
340
+ try {
341
+ cosBuffer = acquireBuffer(globalFreqs.cos.byteLength, undefined, 'rope_cos');
342
+ sinBuffer = acquireBuffer(globalFreqs.sin.byteLength, undefined, 'rope_sin');
343
+ device.queue.writeBuffer(cosBuffer, 0, globalFreqs.cos.buffer, globalFreqs.cos.byteOffset, globalFreqs.cos.byteLength);
344
+ device.queue.writeBuffer(sinBuffer, 0, globalFreqs.sin.buffer, globalFreqs.sin.byteOffset, globalFreqs.sin.byteLength);
345
+
346
+ if (localFreqs) {
347
+ localCosBuffer = acquireBuffer(localFreqs.cos.byteLength, undefined, 'rope_local_cos');
348
+ localSinBuffer = acquireBuffer(localFreqs.sin.byteLength, undefined, 'rope_local_sin');
349
+ device.queue.writeBuffer(localCosBuffer, 0, localFreqs.cos.buffer, localFreqs.cos.byteOffset, localFreqs.cos.byteLength);
350
+ device.queue.writeBuffer(localSinBuffer, 0, localFreqs.sin.buffer, localFreqs.sin.byteOffset, localFreqs.sin.byteLength);
351
+ }
352
+ } catch (error) {
353
+ for (const buffer of [cosBuffer, sinBuffer, localCosBuffer, localSinBuffer]) {
354
+ if (buffer) {
355
+ releaseBuffer(buffer);
356
+ }
357
+ }
358
+ throw error;
359
+ }
360
+
361
+ log.debug(
362
+ 'Pipeline',
363
+ `RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
364
+ `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
365
+ `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
366
+ `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
367
+ `interleaved=${mropeInterleaved === true}`
368
+ );
369
+
370
+ const buffers = {
371
+ cos: cosBuffer,
372
+ sin: sinBuffer,
373
+ localCos: localCosBuffer,
374
+ localSin: localSinBuffer,
375
+ };
376
+ perDeviceCache.set(cacheKey, buffers);
377
+ return buffers;
378
+ }
379
+
380
+ const cachedCpuBuffers = CPU_ROPE_BUFFER_CACHE.get(cacheKey);
381
+ if (cachedCpuBuffers) {
382
+ return cachedCpuBuffers;
383
+ }
384
+
385
+ log.debug(
386
+ 'Pipeline',
387
+ `RoPE frequencies initialized (CPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
388
+ `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
389
+ `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
390
+ `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
391
+ `interleaved=${mropeInterleaved === true}`
392
+ );
393
+
394
+ const buffers = {
395
+ cos: globalFreqs.cos,
396
+ sin: globalFreqs.sin,
397
+ localCos: localFreqs?.cos,
398
+ localSin: localFreqs?.sin,
399
+ };
400
+ CPU_ROPE_BUFFER_CACHE.set(cacheKey, buffers);
401
+ return buffers;
402
+ }
403
+
404
+
405
+ export function isGPURoPEBuffers(buffers) {
406
+ if (typeof GPUBuffer === 'undefined') return false;
407
+ return !!buffers?.cos && isGpuBufferInstance(buffers.cos);
408
+ }