doppler-gpu 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1589 -0
  4. package/package.json +44 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  16. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  17. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  18. package/src/config/kernel-path-loader.js +1 -0
  19. package/src/config/kernels/backward-registry.json +13 -2
  20. package/src/config/kernels/kernel-ref-digests.js +34 -21
  21. package/src/config/kernels/registry.json +1402 -2623
  22. package/src/config/merge.js +11 -0
  23. package/src/config/param-validator.js +25 -0
  24. package/src/config/runtime/default.json +3 -1
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  29. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  30. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  31. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  32. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  33. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  34. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  35. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  36. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  37. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  38. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  39. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  40. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  41. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  42. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  43. package/src/config/schema/doppler.schema.js +40 -5
  44. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  45. package/src/config/schema/execution-v1.schema.js +10 -0
  46. package/src/config/schema/index.d.ts +3 -1
  47. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  48. package/src/config/schema/training-metrics.schema.js +4 -0
  49. package/src/config/transforms/capability-transform-resolver.js +7 -1
  50. package/src/config/transforms/execution-graph-transforms.js +2 -2
  51. package/src/debug/config.d.ts +4 -0
  52. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  53. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  54. package/src/experimental/orchestration/index.d.ts +10 -1
  55. package/src/experimental/training/attention-backward.js +123 -55
  56. package/src/experimental/training/autograd.js +101 -5
  57. package/src/experimental/training/clip.js +6 -1
  58. package/src/experimental/training/datasets/text-pairs.js +80 -8
  59. package/src/experimental/training/distillation/student-fixture.js +470 -114
  60. package/src/experimental/training/lora-pipeline.js +202 -27
  61. package/src/experimental/training/lora.js +10 -4
  62. package/src/experimental/training/optimizer.js +9 -1
  63. package/src/experimental/training/runner.js +18 -4
  64. package/src/experimental/training/suite.js +1 -1
  65. package/src/experimental/training/workloads.js +11 -0
  66. package/src/formats/rdrr/classification.js +3 -0
  67. package/src/formats/rdrr/integrity.d.ts +0 -2
  68. package/src/formats/rdrr/integrity.js +0 -2
  69. package/src/formats/rdrr/types.d.ts +1 -1
  70. package/src/gpu/kernels/attention.d.ts +2 -0
  71. package/src/gpu/kernels/attention.js +75 -16
  72. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  73. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  74. package/src/gpu/kernels/backward/adam.js +6 -4
  75. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  76. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  77. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  78. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  79. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  80. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  81. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  82. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  83. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  84. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  85. package/src/gpu/kernels/backward/index.d.ts +3 -0
  86. package/src/gpu/kernels/backward/index.js +3 -0
  87. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  88. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  89. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  90. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  91. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  92. package/src/gpu/kernels/backward/utils.js +9 -2
  93. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  94. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  95. package/src/gpu/kernels/fused_ffn.js +540 -19
  96. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  97. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  98. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  99. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  100. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  101. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  102. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  103. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  104. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  105. package/src/gpu/kernels/index.d.ts +26 -0
  106. package/src/gpu/kernels/index.js +18 -0
  107. package/src/gpu/kernels/linear-attention-core.js +413 -15
  108. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  109. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  110. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  111. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  112. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  113. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  114. package/src/gpu/kernels/matmul-selection.js +10 -1
  115. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  116. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  117. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  118. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  119. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  120. package/src/gpu/kernels/silu.d.ts +1 -0
  121. package/src/gpu/kernels/silu.js +19 -4
  122. package/src/gpu/kernels/silu.wgsl +7 -1
  123. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  124. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  125. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  126. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  127. package/src/gpu/tensor.d.ts +1 -0
  128. package/src/index.d.ts +1 -1
  129. package/src/index.js +1 -1
  130. package/src/inference/browser-harness-suite-helpers.js +1 -1
  131. package/src/inference/browser-harness-text-helpers.js +251 -20
  132. package/src/inference/browser-harness.js +221 -3
  133. package/src/inference/kv-cache/base.js +42 -1
  134. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  135. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  136. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  137. package/src/inference/pipelines/text/attention/projections.js +174 -4
  138. package/src/inference/pipelines/text/attention/record.js +44 -21
  139. package/src/inference/pipelines/text/attention/run.js +18 -3
  140. package/src/inference/pipelines/text/config.d.ts +18 -0
  141. package/src/inference/pipelines/text/config.js +100 -0
  142. package/src/inference/pipelines/text/embed.js +1 -1
  143. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  144. package/src/inference/pipelines/text/execution-plan.js +37 -0
  145. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  146. package/src/inference/pipelines/text/execution-v1.js +130 -10
  147. package/src/inference/pipelines/text/ffn/dense.js +241 -21
  148. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  149. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  152. package/src/inference/pipelines/text/generator-steps.js +2 -2
  153. package/src/inference/pipelines/text/generator.d.ts +18 -1
  154. package/src/inference/pipelines/text/generator.js +326 -19
  155. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  156. package/src/inference/pipelines/text/init.d.ts +22 -3
  157. package/src/inference/pipelines/text/init.js +2 -2
  158. package/src/inference/pipelines/text/layer.js +87 -8
  159. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  160. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  161. package/src/inference/pipelines/text/linear-attention.js +173 -16
  162. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  163. package/src/inference/pipelines/text/logits/index.js +62 -1
  164. package/src/inference/pipelines/text/lora-apply.js +4 -4
  165. package/src/inference/pipelines/text/model-load.js +6 -0
  166. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  167. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  168. package/src/inference/pipelines/text/ops.js +22 -0
  169. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  170. package/src/inference/pipelines/text/probes.js +9 -4
  171. package/src/inference/pipelines/text/state.d.ts +4 -4
  172. package/src/inference/pipelines/text/state.js +4 -0
  173. package/src/inference/pipelines/text/types.d.ts +27 -2
  174. package/src/inference/pipelines/text.d.ts +21 -1
  175. package/src/inference/pipelines/text.js +40 -14
  176. package/src/inference/tokenizers/bundled.js +150 -55
  177. package/src/inference/tokenizers/types.d.ts +7 -4
  178. package/src/loader/doppler-loader.js +25 -2
  179. package/src/loader/index.d.ts +3 -1
  180. package/src/loader/index.js +2 -0
  181. package/src/loader/loader-types.d.ts +3 -1
  182. package/src/loader/manifest-config.d.ts +18 -0
  183. package/src/loader/manifest-config.js +11 -3
  184. package/src/loader/shard-cache.d.ts +2 -0
  185. package/src/loader/shard-cache.js +29 -1
  186. package/src/models/gemma4.js +8 -0
  187. package/src/rules/inference/capability-transforms.rules.json +145 -20
  188. package/src/rules/inference/ffn.rules.json +33 -0
  189. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  190. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  191. package/src/rules/kernels/matmul.rules.json +2 -0
  192. package/src/rules/kernels/silu.rules.json +1 -0
  193. package/src/rules/rule-registry.js +2 -0
  194. package/src/storage/artifact-storage-context.js +36 -0
  195. package/src/tooling/node-browser-command-runner.js +42 -7
  196. package/src/tooling/node-converter.js +20 -5
  197. package/src/tooling/opfs-cache.d.ts +21 -1
  198. package/src/tooling/opfs-cache.js +168 -32
  199. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  200. package/src/tooling-experimental-exports.shared.js +18 -0
  201. package/src/tooling-exports/storage.d.ts +1 -1
  202. package/src/tooling-exports/storage.js +1 -1
  203. package/src/types/gpu.d.ts +25 -0
  204. package/src/version.js +1 -1
  205. package/tests/kernels/browser/kernel-suite.js +1593 -0
  206. package/tests/kernels/browser/test-page.js +2629 -0
  207. package/tests/kernels/harness/benchmark.js +225 -0
  208. package/tests/kernels/harness/buffer-utils.js +88 -0
  209. package/tests/kernels/harness/tolerance.js +174 -0
  210. package/tests/kernels/reference/adam.js +15 -0
  211. package/tests/kernels/reference/attention.js +158 -0
  212. package/tests/kernels/reference/backward.js +96 -0
  213. package/tests/kernels/reference/bias-add.js +20 -0
  214. package/tests/kernels/reference/conv2d.js +58 -0
  215. package/tests/kernels/reference/cross-entropy.js +26 -0
  216. package/tests/kernels/reference/dequant.js +312 -0
  217. package/tests/kernels/reference/embed.js +15 -0
  218. package/tests/kernels/reference/gather.js +61 -0
  219. package/tests/kernels/reference/gelu.js +37 -0
  220. package/tests/kernels/reference/groupnorm.js +36 -0
  221. package/tests/kernels/reference/index.js +89 -0
  222. package/tests/kernels/reference/layernorm.js +22 -0
  223. package/tests/kernels/reference/matmul.js +57 -0
  224. package/tests/kernels/reference/modulate.js +24 -0
  225. package/tests/kernels/reference/moe-gather.js +83 -0
  226. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  227. package/tests/kernels/reference/residual.js +55 -0
  228. package/tests/kernels/reference/rmsnorm.js +52 -0
  229. package/tests/kernels/reference/rope.js +79 -0
  230. package/tests/kernels/reference/sample.js +97 -0
  231. package/tests/kernels/reference/scatter-add.js +58 -0
  232. package/tests/kernels/reference/silu.js +52 -0
  233. package/tests/kernels/reference/softmax.js +86 -0
  234. package/tests/kernels/reference/split-qkv.js +58 -0
  235. package/tests/kernels/reference/topk.js +88 -0
  236. package/tests/kernels/reference/upsample2d.js +37 -0
  237. package/BRANDING.md +0 -14
  238. package/SECURITY.md +0 -19
  239. package/assets/doppler-runtime-map.svg +0 -82
  240. package/src/bootstrap.js +0 -30
  241. package/src/cli/doppler-cli.d.ts +0 -46
  242. package/src/cli/doppler-quickstart.d.ts +0 -45
  243. package/src/cli/doppler-serve.d.ts +0 -210
  244. package/src/cli/runtime-profiles.d.ts +0 -40
  245. package/src/client/doppler-api.browser.d.ts +0 -1
  246. package/src/client/doppler-api.browser.js +0 -25
  247. package/src/client/doppler-registry.d.ts +0 -54
  248. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  249. package/src/client/runtime/source-runtime.d.ts +0 -26
  250. package/src/client/runtime/storage.d.ts +0 -5
  251. package/src/config/README.md +0 -61
  252. package/src/config/conversion/README.md +0 -62
  253. package/src/config/goal-completion-matrix.json +0 -552
  254. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  255. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  256. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  257. package/src/config/kernels/registry.d.ts +0 -51
  258. package/src/config/merge-contract-check.d.ts +0 -16
  259. package/src/config/merge-contract-check.js +0 -319
  260. package/src/config/merge-helpers.d.ts +0 -60
  261. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  262. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  263. package/src/config/param-categories.d.ts +0 -8
  264. package/src/config/param-validator.d.ts +0 -33
  265. package/src/config/platforms/loader.d.ts +0 -38
  266. package/src/config/quantization-contract-check.d.ts +0 -12
  267. package/src/config/quantization-contract-check.js +0 -91
  268. package/src/config/runtime/README.md +0 -28
  269. package/src/config/runtime-merge.d.ts +0 -5
  270. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  271. package/src/config/schema/command-surface-contract.schema.json +0 -81
  272. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  273. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  274. package/src/config/schema/platform.schema.js +0 -1
  275. package/src/config/schema/policy-schema-registry.json +0 -31
  276. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  277. package/src/config/support-tiers/subsystems.json +0 -502
  278. package/src/config/supported-operations.d.ts +0 -14
  279. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  280. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  281. package/src/converter/conversion-plan.d.ts +0 -56
  282. package/src/converter/index.d.ts +0 -98
  283. package/src/converter/index.js +0 -59
  284. package/src/converter/parsers/diffusion.d.ts +0 -50
  285. package/src/converter/parsers/gguf.d.ts +0 -31
  286. package/src/converter/parsers/index.d.ts +0 -21
  287. package/src/converter/parsers/index.js +0 -12
  288. package/src/converter/parsers/transformer.d.ts +0 -31
  289. package/src/converter/quantization-info.d.ts +0 -38
  290. package/src/converter/rope-config.d.ts +0 -16
  291. package/src/converter/tokenizer-utils.d.ts +0 -12
  292. package/src/debug/reference/README.md +0 -114
  293. package/src/debug/reference/hf_attn_debug.py +0 -114
  294. package/src/debug/reference/hf_embed_check.py +0 -89
  295. package/src/debug/reference/hf_layer_out.py +0 -100
  296. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  297. package/src/debug/reference/hf_rope_check.py +0 -116
  298. package/src/debug/reference/hf_weights.py +0 -75
  299. package/src/errors/doppler-error.d.ts +0 -20
  300. package/src/experimental/bridge/extension/background.d.ts +0 -14
  301. package/src/experimental/bridge/extension/background.js +0 -168
  302. package/src/experimental/bridge/extension/manifest.json +0 -34
  303. package/src/experimental/bridge/extension-client.d.ts +0 -114
  304. package/src/experimental/bridge/index.d.ts +0 -69
  305. package/src/experimental/bridge/node-bridge.js +0 -24
  306. package/src/experimental/bridge/protocol.d.ts +0 -96
  307. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  308. package/src/experimental/browser/gguf-importer.js +0 -532
  309. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  310. package/src/experimental/browser/quantization.d.ts +0 -74
  311. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  312. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  313. package/src/experimental/browser/shard-io-browser.js +0 -89
  314. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  315. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  316. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  317. package/src/experimental/distribution/p2p-transport.js +0 -75
  318. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  319. package/src/experimental/hotswap/manifest.d.ts +0 -42
  320. package/src/experimental/hotswap/model-swap.js +0 -42
  321. package/src/experimental/hotswap/runtime.d.ts +0 -31
  322. package/src/experimental/training/README.md +0 -164
  323. package/src/experimental/training/artifacts.d.ts +0 -160
  324. package/src/experimental/training/attention-backward.d.ts +0 -30
  325. package/src/experimental/training/attention-forward.d.ts +0 -22
  326. package/src/experimental/training/attention-forward.js +0 -82
  327. package/src/experimental/training/autograd.d.ts +0 -51
  328. package/src/experimental/training/backward-kernels.js +0 -39
  329. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  330. package/src/experimental/training/checkpoint.d.ts +0 -36
  331. package/src/experimental/training/clip.d.ts +0 -9
  332. package/src/experimental/training/dataloader.d.ts +0 -8
  333. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  334. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  335. package/src/experimental/training/datasets/index.d.ts +0 -24
  336. package/src/experimental/training/datasets/index.js +0 -13
  337. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  338. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  339. package/src/experimental/training/datasets/reploid.js +0 -36
  340. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  341. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  342. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  343. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  344. package/src/experimental/training/datasets/token-batch.js +0 -52
  345. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  346. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  347. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  348. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  349. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  350. package/src/experimental/training/distillation/eval.d.ts +0 -34
  351. package/src/experimental/training/distillation/index.d.ts +0 -29
  352. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  353. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  354. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  355. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  356. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  357. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  358. package/src/experimental/training/export.d.ts +0 -41
  359. package/src/experimental/training/index.d.ts +0 -74
  360. package/src/experimental/training/index.js +0 -63
  361. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  362. package/src/experimental/training/lora.d.ts +0 -19
  363. package/src/experimental/training/loss-scaling.d.ts +0 -21
  364. package/src/experimental/training/loss.d.ts +0 -10
  365. package/src/experimental/training/objectives/base.d.ts +0 -58
  366. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  367. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  368. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  369. package/src/experimental/training/objectives/index.d.ts +0 -12
  370. package/src/experimental/training/objectives/index.js +0 -6
  371. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  372. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  373. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  374. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  375. package/src/experimental/training/operator-command.d.ts +0 -5
  376. package/src/experimental/training/operator-eval.d.ts +0 -48
  377. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  378. package/src/experimental/training/optimizer.d.ts +0 -22
  379. package/src/experimental/training/runner.d.ts +0 -249
  380. package/src/experimental/training/suite.d.ts +0 -299
  381. package/src/experimental/training/tensor-factory.d.ts +0 -9
  382. package/src/experimental/training/trainer.d.ts +0 -89
  383. package/src/experimental/training/ul_dataset.d.ts +0 -47
  384. package/src/experimental/training/ul_schedule.d.ts +0 -6
  385. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  386. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  387. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  388. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  389. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  390. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  391. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  392. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  393. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  394. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  395. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  396. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  397. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  398. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  399. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  400. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  401. package/src/experimental/training/workload-packs/registry.json +0 -278
  402. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  403. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  404. package/src/experimental/training/workloads.d.ts +0 -201
  405. package/src/formats/gguf/types.d.ts +0 -74
  406. package/src/formats/index.d.ts +0 -75
  407. package/src/formats/index.js +0 -19
  408. package/src/formats/litert/types.d.ts +0 -74
  409. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  410. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  411. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  412. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  413. package/src/formats/safetensors/types.d.ts +0 -67
  414. package/src/formats/tflite/types.d.ts +0 -61
  415. package/src/formats/tokenizer/index.d.ts +0 -5
  416. package/src/formats/tokenizer/index.js +0 -3
  417. package/src/formats/tokenizer/types.d.ts +0 -9
  418. package/src/gpu/kernel-runtime.d.ts +0 -13
  419. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  420. package/src/gpu/kernel-selection-log.d.ts +0 -4
  421. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  422. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  423. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  424. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  425. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  426. package/src/gpu/kernel-tuner/types.js +0 -4
  427. package/src/gpu/kernel-tuner.d.ts +0 -33
  428. package/src/gpu/kernels/README.md +0 -166
  429. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  430. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  431. package/src/gpu/kernels/check-stop.d.ts +0 -31
  432. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  433. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  434. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  435. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  436. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  437. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  438. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  439. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  440. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  441. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  442. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  443. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  444. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  445. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  446. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  447. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  448. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  449. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  450. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  451. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  452. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  453. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  454. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  455. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  456. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  457. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  458. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  459. package/src/gpu/kernels/constants.d.ts +0 -129
  460. package/src/gpu/kernels/dispatch.d.ts +0 -72
  461. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  462. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  463. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  464. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  465. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  466. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  467. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  468. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  469. package/src/gpu/kernels/types.js +0 -4
  470. package/src/gpu/perf-guards.d.ts +0 -13
  471. package/src/gpu/profiler.d.ts +0 -93
  472. package/src/gpu/readback-utils.d.ts +0 -16
  473. package/src/gpu/readback-utils.js +0 -49
  474. package/src/gpu/submit-probe.d.ts +0 -12
  475. package/src/index-browser.d.ts +0 -2
  476. package/src/index-browser.js +0 -2
  477. package/src/inference/README.md +0 -593
  478. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  479. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  480. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  481. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  482. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  483. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  484. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  485. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  486. package/src/inference/network-evolution.js +0 -79
  487. package/src/inference/pipelines/context.d.ts +0 -21
  488. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  489. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  490. package/src/inference/pipelines/diffusion/index.js +0 -3
  491. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  492. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  493. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  494. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  495. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  496. package/src/inference/pipelines/diffusion/types.js +0 -1
  497. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  498. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  499. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  500. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  501. package/src/inference/pipelines/dream/pipeline.js +0 -8
  502. package/src/inference/pipelines/energy/index.d.ts +0 -1
  503. package/src/inference/pipelines/energy/index.js +0 -1
  504. package/src/inference/pipelines/energy/types.js +0 -1
  505. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  506. package/src/inference/pipelines/energy-head/index.js +0 -6
  507. package/src/inference/pipelines/factory.d.ts +0 -10
  508. package/src/inference/pipelines/index.d.ts +0 -22
  509. package/src/inference/pipelines/index.js +0 -19
  510. package/src/inference/pipelines/registry.d.ts +0 -15
  511. package/src/inference/pipelines/rng.d.ts +0 -2
  512. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  513. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  514. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  515. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  516. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  517. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  518. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  519. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  520. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  521. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  522. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  523. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  524. package/src/inference/pipelines/text/attention.d.ts +0 -23
  525. package/src/inference/pipelines/text/attention.js +0 -12
  526. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  527. package/src/inference/pipelines/text/buffer-types.js +0 -4
  528. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  529. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  530. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  531. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  532. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  533. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  534. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  535. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  536. package/src/inference/pipelines/text/embed.d.ts +0 -69
  537. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  538. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  539. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  540. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  541. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  542. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  543. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  544. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  545. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  546. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  547. package/src/inference/pipelines/text/ffn.js +0 -18
  548. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  549. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  550. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  551. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  552. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  553. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  554. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  555. package/src/inference/pipelines/text/index.d.ts +0 -5
  556. package/src/inference/pipelines/text/index.js +0 -6
  557. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  558. package/src/inference/pipelines/text/layer.d.ts +0 -118
  559. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  560. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  561. package/src/inference/pipelines/text/logits/types.js +0 -4
  562. package/src/inference/pipelines/text/logits.d.ts +0 -27
  563. package/src/inference/pipelines/text/logits.js +0 -16
  564. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  565. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  566. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  567. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  568. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  569. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  570. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  571. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  572. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  573. package/src/inference/pipelines/text/ops.d.ts +0 -222
  574. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  575. package/src/inference/pipelines/text/probes.d.ts +0 -28
  576. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  577. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  578. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  579. package/src/inference/pipelines/text/types.js +0 -4
  580. package/src/inference/tokenizers/base.d.ts +0 -44
  581. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  582. package/src/inference/tokenizers/bpe.d.ts +0 -27
  583. package/src/inference/tokenizers/bundled.d.ts +0 -67
  584. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  585. package/src/inference/tokenizers/types.js +0 -7
  586. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  587. package/src/loader/per-layer-input-loader.d.ts +0 -30
  588. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  589. package/src/loader/tensors/source-transform.d.ts +0 -65
  590. package/src/memory/address-table.d.ts +0 -96
  591. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  592. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  593. package/src/rules/rule-registry.d.ts +0 -77
  594. package/src/storage/artifact-storage-context.d.ts +0 -63
  595. package/src/storage/backends/idb-store.d.ts +0 -52
  596. package/src/storage/backends/memory-store.d.ts +0 -36
  597. package/src/storage/backends/opfs-store.d.ts +0 -41
  598. package/src/storage/blake3.d.ts +0 -17
  599. package/src/storage/download-types.d.ts +0 -157
  600. package/src/storage/downloader.d.ts +0 -103
  601. package/src/storage/index.d.ts +0 -274
  602. package/src/storage/index.js +0 -201
  603. package/src/storage/preflight.d.ts +0 -144
  604. package/src/storage/preflight.js +0 -360
  605. package/src/storage/quickstart-downloader.d.ts +0 -157
  606. package/src/storage/quickstart-downloader.js +0 -289
  607. package/src/storage/source-artifact-store.d.ts +0 -59
  608. package/src/sw.js +0 -184
  609. package/src/tooling/command-api-constants.d.ts +0 -9
  610. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  611. package/src/tooling/command-api-helpers.d.ts +0 -23
  612. package/src/tooling/command-runner-shared.d.ts +0 -89
  613. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  614. package/src/tooling/conversion-config-materializer.js +0 -96
  615. package/src/tooling/diagnose-runner.d.ts +0 -28
  616. package/src/tooling/hf-registry-utils.d.ts +0 -55
  617. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  618. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  619. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  620. package/src/tooling/kernel-path-builder/index.js +0 -1361
  621. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  622. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  623. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  624. package/src/tooling/lean-execution-contract-runner.js +0 -158
  625. package/src/tooling/lean-execution-contract.d.ts +0 -16
  626. package/src/tooling/lean-execution-contract.js +0 -81
  627. package/src/tooling/litert-package-runtime.d.ts +0 -46
  628. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  629. package/src/tooling/node-convert-worker.d.ts +0 -1
  630. package/src/tooling/node-converter.d.ts +0 -1
  631. package/src/tooling/node-file-fetch.d.ts +0 -1
  632. package/src/tooling/node-source-runtime.d.ts +0 -24
  633. package/src/tooling/node-webgpu.d.ts +0 -17
  634. package/src/tooling/precision-replay-math.js +0 -174
  635. package/src/tooling/program-bundle-host.d.ts +0 -16
  636. package/src/tooling/program-bundle-host.js +0 -20
  637. package/src/tooling/runtime-input-composition.d.ts +0 -37
  638. package/src/tooling/source-package-profiles.d.ts +0 -50
  639. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  640. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  641. package/src/tooling/source-runtime-materializer.js +0 -89
  642. package/src/tooling-exports.browser.d.ts +0 -7
  643. package/src/types/chrome.d.ts +0 -36
  644. package/src/types/chrome.js +0 -1
  645. package/src/types/index.d.ts +0 -1
  646. package/src/types/index.js +0 -1
  647. package/src/utils/array-buffer.d.ts +0 -4
  648. package/src/utils/canonical-hash.d.ts +0 -7
  649. package/src/utils/clone-json.d.ts +0 -1
  650. package/src/utils/hf-resolve-url.d.ts +0 -16
  651. package/src/utils/index.d.ts +0 -10
  652. package/src/utils/index.js +0 -10
  653. package/src/utils/load-json.d.ts +0 -5
  654. package/src/utils/plain-object.d.ts +0 -1
  655. package/src/utils/sha256.d.ts +0 -5
  656. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -28,8 +28,12 @@ export {
28
28
  isFusedQ4KDisabled,
29
29
  } from './matmul.js';
30
30
  export {
31
+ recordLmHeadArgmax,
31
32
  recordLmHeadArgmaxF16,
32
33
  } from './lm-head-argmax.js';
34
+ export {
35
+ runLmHeadSelectLogitsF16,
36
+ } from './lm-head-select-logits.js';
33
37
 
34
38
  // Dequantization
35
39
  export {
@@ -80,6 +84,8 @@ export {
80
84
  RMSNORM_PAIR_CACHE_LIMIT,
81
85
  runSandwichRMSNormPair,
82
86
  recordSandwichRMSNormPair,
87
+ runResidualNextRMSNormPair,
88
+ recordResidualNextRMSNormPair,
83
89
  } from './rmsnorm-pair.js';
84
90
 
85
91
  // LayerNorm
@@ -297,9 +303,16 @@ export {
297
303
  export {
298
304
  runFusedFFN,
299
305
  recordFusedFFN,
306
+ runFusedFFNFromRMSNormStats,
307
+ recordFusedFFNFromRMSNormStats,
300
308
  calculateFusedFFNSavings,
301
309
  } from './fused_ffn.js';
302
310
 
311
+ export {
312
+ runRMSNormStats,
313
+ recordRMSNormStats,
314
+ } from './rmsnorm-stats.js';
315
+
303
316
  // Fused Matmul + RMSNorm (P0 - 1.2-1.5x decode speedup)
304
317
  export {
305
318
  selectMatmulRMSNormFusedVariant,
@@ -357,6 +370,11 @@ export {
357
370
  recordSoftmaxBackward,
358
371
  runRmsNormBackward,
359
372
  recordRmsNormBackward,
373
+ runGatedRmsNormBackward,
374
+ recordGatedRmsNormBackward,
375
+ runCausalConv1dSiluBackward,
376
+ recordCausalConv1dSiluBackward,
377
+ runGatedDeltaRecurrentBackward,
360
378
  runLayerNormBackward,
361
379
  recordLayerNormBackward,
362
380
  runAttentionBackward,
@@ -2,6 +2,7 @@ import { getDevice, getDeviceEpoch } from '../device.js';
2
2
  import { WORKGROUP_SIZES } from './constants.js';
3
3
  import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
4
4
  import { createTensor } from '../tensor.js';
5
+ import { getRuntimeConfig } from '../../config/runtime.js';
5
6
  import { castF32ToF16, recordCastF32ToF16 } from './cast.js';
6
7
  import {
7
8
  createUniformBufferFromData,
@@ -35,8 +36,8 @@ struct LinearAttentionParams {
35
36
  _pad_u32_0: u32,
36
37
  rms_norm_eps: f32,
37
38
  qk_l2norm_eps: f32,
38
- _pad_f32_0: f32,
39
- _pad_f32_1: f32,
39
+ packed_flags: u32,
40
+ b_proj_offset_elements: u32,
40
41
  }
41
42
 
42
43
  @group(0) @binding(0) var<uniform> params: LinearAttentionParams;
@@ -60,12 +61,14 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
60
61
  if (channel >= params.conv_dim) {
61
62
  return;
62
63
  }
64
+ let qkvz_packed = (params.packed_flags & 2u) != 0u;
65
+ let qkv_stride = select(params.conv_dim, params.conv_dim + params.value_dim, qkvz_packed);
63
66
 
64
67
  let kernel_size = params.conv_kernel_size;
65
68
  let state_base = channel * kernel_size;
66
69
 
67
70
  for (var token_idx: u32 = 0u; token_idx < params.num_tokens; token_idx = token_idx + 1u) {
68
- let qkv_idx = token_idx * params.conv_dim + channel;
71
+ let qkv_idx = token_idx * qkv_stride + channel;
69
72
  let newest = f32(qkv[qkv_idx]);
70
73
 
71
74
  for (var k: u32 = 0u; k + 1u < kernel_size; k = k + 1u) {
@@ -104,8 +107,8 @@ struct LinearAttentionParams {
104
107
  norm_mode: u32,
105
108
  rms_norm_eps: f32,
106
109
  qk_l2norm_eps: f32,
107
- _pad_f32_0: f32,
108
- _pad_f32_1: f32,
110
+ packed_flags: u32,
111
+ b_proj_offset_elements: u32,
109
112
  }
110
113
 
111
114
  @group(0) @binding(0) var<uniform> params: LinearAttentionParams;
@@ -163,6 +166,7 @@ fn main(@builtin(workgroup_id) wid: vec3<u32>,
163
166
  for (var token_idx: u32 = 0u; token_idx < params.num_tokens; token_idx = token_idx + 1u) {
164
167
  let conv_row_base = token_idx * params.conv_dim;
165
168
  let z_row_base = token_idx * params.value_dim + head * head_v_dim;
169
+ let z_packed_base = token_idx * (params.conv_dim + params.value_dim) + params.conv_dim + head * head_v_dim;
166
170
  let ab_row_base = token_idx * params.num_v_heads + head;
167
171
  let out_row_base = token_idx * params.value_dim + head * head_v_dim;
168
172
 
@@ -195,7 +199,8 @@ fn main(@builtin(workgroup_id) wid: vec3<u32>,
195
199
  workgroupBarrier();
196
200
  }
197
201
  let k_norm_scale = inverseSqrt(shared_sq[0] + params.qk_l2norm_eps);
198
- let beta = 1.0 / (1.0 + exp(-f32(b_proj[ab_row_base])));
202
+ let b_index = select(ab_row_base, params.b_proj_offset_elements + ab_row_base, (params.packed_flags & 1u) != 0u);
203
+ let beta = 1.0 / (1.0 + exp(-f32(b_proj[b_index])));
199
204
  let g = a_neg_exp[head] * softplus(f32(a_proj[ab_row_base]) + dt_bias[head]);
200
205
  let g_exp = exp(g);
201
206
 
@@ -243,7 +248,8 @@ fn main(@builtin(workgroup_id) wid: vec3<u32>,
243
248
  let inv_rms = inverseSqrt(shared_sq[0] / f32(head_v_dim) + params.rms_norm_eps);
244
249
 
245
250
  if (is_active) {
246
- let gate = silu(f32(z_proj[z_row_base + vd]));
251
+ let z_index = select(z_row_base + vd, z_packed_base + vd, (params.packed_flags & 2u) != 0u);
252
+ let gate = silu(f32(z_proj[z_index]));
247
253
  let norm_index = select(vd, head * head_v_dim + vd, params.norm_mode == 1u);
248
254
  output[out_row_base + vd] = (output[out_row_base + vd] * inv_rms) * norm_weight[norm_index] * gate;
249
255
  }
@@ -252,10 +258,190 @@ fn main(@builtin(workgroup_id) wid: vec3<u32>,
252
258
  `;
253
259
  }
254
260
 
261
+ function buildLinearAttentionFusedDecodeShader(inputDtype) {
262
+ const inputScalar = selectRuleValue('shared', 'dtype', 'f16OrF32', { useF16: inputDtype === 'f16' });
263
+ const enableF16 = inputScalar === 'f16' ? 'enable f16;\n\n' : '';
264
+ return /* wgsl */ `${enableF16}override WORKGROUP_SIZE: u32 = 128u;
265
+
266
+ struct LinearAttentionParams {
267
+ num_tokens: u32,
268
+ conv_dim: u32,
269
+ conv_kernel_size: u32,
270
+ num_v_heads: u32,
271
+ num_k_heads: u32,
272
+ head_k_dim: u32,
273
+ head_v_dim: u32,
274
+ q_size: u32,
275
+ k_size: u32,
276
+ value_dim: u32,
277
+ q_rep: u32,
278
+ norm_mode: u32,
279
+ rms_norm_eps: f32,
280
+ qk_l2norm_eps: f32,
281
+ packed_flags: u32,
282
+ b_proj_offset_elements: u32,
283
+ }
284
+
285
+ @group(0) @binding(0) var<uniform> params: LinearAttentionParams;
286
+ @group(0) @binding(1) var<storage, read> qkv: array<${inputScalar}>;
287
+ @group(0) @binding(2) var<storage, read> z_proj: array<${inputScalar}>;
288
+ @group(0) @binding(3) var<storage, read> ab_proj: array<${inputScalar}>;
289
+ @group(0) @binding(4) var<storage, read> conv_weight: array<f32>;
290
+ @group(0) @binding(5) var<storage, read_write> conv_state: array<f32>;
291
+ @group(0) @binding(6) var<storage, read> dt_bias: array<f32>;
292
+ @group(0) @binding(7) var<storage, read> a_neg_exp: array<f32>;
293
+ @group(0) @binding(8) var<storage, read> norm_weight: array<f32>;
294
+ @group(0) @binding(9) var<storage, read_write> recurrent_state: array<f32>;
295
+ @group(0) @binding(10) var<storage, read_write> output: array<f32>;
296
+
297
+ var<workgroup> shared_q: array<f32, WORKGROUP_SIZE>;
298
+ var<workgroup> shared_k: array<f32, WORKGROUP_SIZE>;
299
+ var<workgroup> shared_reduce: array<f32, WORKGROUP_SIZE>;
300
+
301
+ fn softplus(x: f32) -> f32 {
302
+ if (x > 20.0) {
303
+ return x;
304
+ }
305
+ if (x < -20.0) {
306
+ return exp(x);
307
+ }
308
+ return log(1.0 + exp(x));
309
+ }
310
+
311
+ fn silu(x: f32) -> f32 {
312
+ if (x >= 0.0) {
313
+ let z = exp(-x);
314
+ return x / (1.0 + z);
315
+ }
316
+ let z = exp(x);
317
+ return x * z / (1.0 + z);
318
+ }
319
+
320
+ fn update_conv_channel(channel: u32) -> f32 {
321
+ let kernel_size = params.conv_kernel_size;
322
+ let state_base = channel * kernel_size;
323
+ let newest = f32(qkv[channel]);
324
+
325
+ for (var k: u32 = 0u; k + 1u < kernel_size; k = k + 1u) {
326
+ conv_state[state_base + k] = conv_state[state_base + k + 1u];
327
+ }
328
+ conv_state[state_base + kernel_size - 1u] = newest;
329
+
330
+ var mixed: f32 = 0.0;
331
+ for (var k: u32 = 0u; k < kernel_size; k = k + 1u) {
332
+ mixed = mixed + conv_state[state_base + k] * conv_weight[state_base + k];
333
+ }
334
+ return silu(mixed);
335
+ }
336
+
337
+ fn reduce_sum(value: f32, lid: u32) -> f32 {
338
+ shared_reduce[lid] = value;
339
+ workgroupBarrier();
340
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
341
+ if (lid < stride) {
342
+ shared_reduce[lid] = shared_reduce[lid] + shared_reduce[lid + stride];
343
+ }
344
+ workgroupBarrier();
345
+ }
346
+ return shared_reduce[0];
347
+ }
348
+
349
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
350
+ fn main(@builtin(workgroup_id) wid: vec3<u32>,
351
+ @builtin(local_invocation_id) lid: vec3<u32>) {
352
+ let head = wid.x;
353
+ let lane = lid.x;
354
+ if (head >= params.num_v_heads || params.num_tokens != 1u || params.q_rep != 1u) {
355
+ return;
356
+ }
357
+
358
+ let head_k_dim = params.head_k_dim;
359
+ let head_v_dim = params.head_v_dim;
360
+ let q_base = head * head_k_dim;
361
+ let k_base = params.q_size + head * head_k_dim;
362
+ let v_base = params.q_size + params.k_size + head * head_v_dim;
363
+ let out_row_base = head * head_v_dim;
364
+ let recurrent_head_base = head * head_k_dim * head_v_dim;
365
+ let is_k_active = lane < head_k_dim;
366
+ let is_v_active = lane < head_v_dim;
367
+
368
+ var q_val = 0.0;
369
+ var k_val = 0.0;
370
+ var v_val = 0.0;
371
+ if (is_k_active) {
372
+ q_val = update_conv_channel(q_base + lane);
373
+ k_val = update_conv_channel(k_base + lane);
374
+ }
375
+ if (is_v_active) {
376
+ v_val = update_conv_channel(v_base + lane);
377
+ }
378
+ shared_q[lane] = select(0.0, q_val, is_k_active);
379
+ shared_k[lane] = select(0.0, k_val, is_k_active);
380
+ workgroupBarrier();
381
+
382
+ let q_norm_sq = reduce_sum(select(0.0, q_val * q_val, is_k_active), lane);
383
+ let head_scale = inverseSqrt(f32(head_k_dim));
384
+ let q_norm_scale = head_scale / sqrt(q_norm_sq + params.qk_l2norm_eps);
385
+ let k_norm_sq = reduce_sum(select(0.0, k_val * k_val, is_k_active), lane);
386
+ let k_norm_scale = inverseSqrt(k_norm_sq + params.qk_l2norm_eps);
387
+
388
+ let ab_row_base = head;
389
+ let b_index = params.b_proj_offset_elements + ab_row_base;
390
+ let beta = 1.0 / (1.0 + exp(-f32(ab_proj[b_index])));
391
+ let g = a_neg_exp[head] * softplus(f32(ab_proj[ab_row_base]) + dt_bias[head]);
392
+ let g_exp = exp(g);
393
+
394
+ if (is_v_active) {
395
+ for (var kd: u32 = 0u; kd < head_k_dim; kd = kd + 1u) {
396
+ let state_idx = recurrent_head_base + kd * head_v_dim + lane;
397
+ recurrent_state[state_idx] = recurrent_state[state_idx] * g_exp;
398
+ }
399
+ }
400
+
401
+ var kv_mem = 0.0;
402
+ if (is_v_active) {
403
+ for (var kd: u32 = 0u; kd < head_k_dim; kd = kd + 1u) {
404
+ let k_normed = shared_k[kd] * k_norm_scale;
405
+ let state_idx = recurrent_head_base + kd * head_v_dim + lane;
406
+ kv_mem = kv_mem + recurrent_state[state_idx] * k_normed;
407
+ }
408
+ let delta = (v_val - kv_mem) * beta;
409
+ for (var kd: u32 = 0u; kd < head_k_dim; kd = kd + 1u) {
410
+ let k_normed = shared_k[kd] * k_norm_scale;
411
+ let state_idx = recurrent_head_base + kd * head_v_dim + lane;
412
+ recurrent_state[state_idx] = recurrent_state[state_idx] + k_normed * delta;
413
+ }
414
+ }
415
+
416
+ var out_value = 0.0;
417
+ if (is_v_active) {
418
+ for (var kd: u32 = 0u; kd < head_k_dim; kd = kd + 1u) {
419
+ let q_normed = shared_q[kd] * q_norm_scale;
420
+ let state_idx = recurrent_head_base + kd * head_v_dim + lane;
421
+ out_value = out_value + recurrent_state[state_idx] * q_normed;
422
+ }
423
+ }
424
+
425
+ let out_sum_sq = reduce_sum(select(0.0, out_value * out_value, is_v_active), lane);
426
+ let inv_rms = inverseSqrt(out_sum_sq / f32(head_v_dim) + params.rms_norm_eps);
427
+
428
+ if (is_v_active) {
429
+ let z_row_base = head * head_v_dim;
430
+ let z_packed_base = params.conv_dim + z_row_base;
431
+ let z_index = select(z_row_base + lane, z_packed_base + lane, (params.packed_flags & 2u) != 0u);
432
+ let gate = silu(f32(z_proj[z_index]));
433
+ let norm_index = select(lane, head * head_v_dim + lane, params.norm_mode == 1u);
434
+ output[out_row_base + lane] = (out_value * inv_rms) * norm_weight[norm_index] * gate;
435
+ }
436
+ }
437
+ `;
438
+ }
439
+
255
440
  let cachedEpoch = -1;
256
441
  const pipelineCache = new Map();
257
442
  let convBindGroupLayout = null;
258
443
  let recurrentBindGroupLayout = null;
444
+ let fusedDecodeBindGroupLayout = null;
259
445
 
260
446
  function createBindGroupLayouts(device) {
261
447
  convBindGroupLayout = getOrCreateBindGroupLayout(
@@ -285,6 +471,23 @@ function createBindGroupLayouts(device) {
285
471
  ],
286
472
  device
287
473
  );
474
+ fusedDecodeBindGroupLayout = getOrCreateBindGroupLayout(
475
+ 'linear_attention_fused_decode_layout',
476
+ [
477
+ { binding: 0, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'uniform' } },
478
+ { binding: 1, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
479
+ { binding: 2, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
480
+ { binding: 3, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
481
+ { binding: 4, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
482
+ { binding: 5, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'storage' } },
483
+ { binding: 6, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
484
+ { binding: 7, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
485
+ { binding: 8, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
486
+ { binding: 9, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'storage' } },
487
+ { binding: 10, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'storage' } },
488
+ ],
489
+ device
490
+ );
288
491
  }
289
492
 
290
493
  function createPipelines(device, inputDtype) {
@@ -298,6 +501,10 @@ function createPipelines(device, inputDtype) {
298
501
  label: `linear_attention_recurrent_${variant}`,
299
502
  code: buildLinearAttentionRecurrentShader(variant),
300
503
  });
504
+ const fusedDecodeModule = device.createShaderModule({
505
+ label: `linear_attention_fused_decode_${variant}`,
506
+ code: buildLinearAttentionFusedDecodeShader(variant),
507
+ });
301
508
 
302
509
  const convPipeline = device.createComputePipeline({
303
510
  label: `linear_attention_conv_pipeline_${variant}`,
@@ -321,8 +528,23 @@ function createPipelines(device, inputDtype) {
321
528
  },
322
529
  },
323
530
  });
531
+ const fusedDecodePipeline = device.createComputePipeline({
532
+ label: `linear_attention_fused_decode_pipeline_${variant}`,
533
+ layout: getOrCreatePipelineLayout(
534
+ 'linear_attention_fused_decode_pipeline_layout',
535
+ [fusedDecodeBindGroupLayout],
536
+ device
537
+ ),
538
+ compute: {
539
+ module: fusedDecodeModule,
540
+ entryPoint: 'main',
541
+ constants: {
542
+ WORKGROUP_SIZE: HEAD_WORKGROUP_SIZE,
543
+ },
544
+ },
545
+ });
324
546
 
325
- pipelineCache.set(variant, { convPipeline, recurrentPipeline });
547
+ pipelineCache.set(variant, { convPipeline, recurrentPipeline, fusedDecodePipeline });
326
548
  }
327
549
 
328
550
  function normalizeInputDtype(dtype) {
@@ -332,10 +554,11 @@ function normalizeInputDtype(dtype) {
332
554
  function ensurePipelines(device, inputDtype) {
333
555
  const epoch = getDeviceEpoch();
334
556
  if (epoch !== cachedEpoch) {
335
- pipelineCache.clear();
336
- convBindGroupLayout = null;
337
- recurrentBindGroupLayout = null;
338
- cachedEpoch = epoch;
557
+ pipelineCache.clear();
558
+ convBindGroupLayout = null;
559
+ recurrentBindGroupLayout = null;
560
+ fusedDecodeBindGroupLayout = null;
561
+ cachedEpoch = epoch;
339
562
  }
340
563
  const variant = normalizeInputDtype(inputDtype);
341
564
  if (!pipelineCache.has(variant)) {
@@ -362,8 +585,9 @@ function buildParamsData(params) {
362
585
  view.setUint32(44, params.normMode, true);
363
586
  view.setFloat32(48, params.rmsNormEps, true);
364
587
  view.setFloat32(52, params.qkL2NormEps, true);
365
- view.setFloat32(56, 0, true);
366
- view.setFloat32(60, 0, true);
588
+ const packedFlags = (params.abPacked ? 1 : 0) | (params.qkvzPacked ? 2 : 0);
589
+ view.setUint32(56, packedFlags, true);
590
+ view.setUint32(60, params.bProjOffsetElements ?? 0, true);
367
591
  return data;
368
592
  }
369
593
 
@@ -381,6 +605,24 @@ function resolveOutputDtype(outputDtype) {
381
605
  throw new Error(`linear_attention core output dtype "${outputDtype}" is invalid.`);
382
606
  }
383
607
 
608
+ function canUseFusedDecodeCore(layerState, numTokens, options, aTensor, bTensor) {
609
+ const session = getRuntimeConfig()?.inference?.session;
610
+ return session?.useLinearAttentionFusedDecodeCore === true
611
+ && numTokens === 1
612
+ && options?.abPacked === true
613
+ && Number.isFinite(options?.bProjOffsetElements)
614
+ && options.bProjOffsetElements > 0
615
+ && aTensor?.buffer === bTensor?.buffer
616
+ && layerState.qRep === 1
617
+ && layerState.numKHeads === layerState.numVHeads
618
+ && layerState.headKDim <= HEAD_WORKGROUP_SIZE
619
+ && layerState.headVDim <= HEAD_WORKGROUP_SIZE
620
+ && layerState.qSize === layerState.numKHeads * layerState.headKDim
621
+ && layerState.kSize === layerState.numKHeads * layerState.headKDim
622
+ && layerState.vSize === layerState.numVHeads * layerState.headVDim
623
+ && layerState.convDim === layerState.qSize + layerState.kSize + layerState.vSize;
624
+ }
625
+
384
626
  export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTensor, layerState, options = {}) {
385
627
  const device = getDevice();
386
628
  if (!device) {
@@ -431,10 +673,160 @@ export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTe
431
673
 
432
674
  const convOutSize = numTokens * layerState.convDim * Float32Array.BYTES_PER_ELEMENT;
433
675
  const outputSize = numTokens * layerState.valueDim * Float32Array.BYTES_PER_ELEMENT;
434
- const convOutBuffer = acquireBuffer(convOutSize, undefined, `L${options.layerIdx ?? 0}.linear_conv_out`);
676
+ let convOutBuffer = null;
435
677
  const outputBuffer = acquireBuffer(outputSize, undefined, `L${options.layerIdx ?? 0}.linear_attention_core_out`);
436
678
  const outputDtype = resolveOutputDtype(options.outputDtype);
437
679
  const outputShape = [numTokens, layerState.valueDim];
680
+ const paramsPayload = {
681
+ numTokens,
682
+ convDim: layerState.convDim,
683
+ convKernelSize: layerState.convKernelSize,
684
+ numVHeads: layerState.numVHeads,
685
+ numKHeads: layerState.numKHeads,
686
+ headKDim: layerState.headKDim,
687
+ headVDim: layerState.headVDim,
688
+ qSize: layerState.qSize,
689
+ kSize: layerState.kSize,
690
+ valueDim: layerState.valueDim,
691
+ qRep: layerState.qRep,
692
+ normMode: layerState.normMode === 'per_head' ? 1 : 0,
693
+ rmsNormEps: Number(layerState.rmsNormEps) || 1e-6,
694
+ qkL2NormEps: Number(options.qkL2NormEps) || 1e-6,
695
+ abPacked: options.abPacked === true,
696
+ qkvzPacked: options.qkvzPacked === true,
697
+ bProjOffsetElements: options.bProjOffsetElements,
698
+ };
699
+ const useFusedDecodeCore = canUseFusedDecodeCore(layerState, numTokens, options, aTensor, bTensor);
700
+ if (useFusedDecodeCore) {
701
+ if (useRecorder) {
702
+ const paramsBuffer = createUniformBufferFromData(
703
+ 'linear_attention_params',
704
+ buildParamsData(paramsPayload),
705
+ recorder
706
+ );
707
+ try {
708
+ const fusedBindGroup = device.createBindGroup({
709
+ label: 'linear_attention_fused_decode_bind_group',
710
+ layout: fusedDecodeBindGroupLayout,
711
+ entries: [
712
+ { binding: 0, resource: { buffer: paramsBuffer } },
713
+ { binding: 1, resource: { buffer: qkvTensor.buffer } },
714
+ { binding: 2, resource: { buffer: zTensor.buffer } },
715
+ { binding: 3, resource: { buffer: aTensor.buffer } },
716
+ { binding: 4, resource: { buffer: layerState.convWeightGPU } },
717
+ { binding: 5, resource: { buffer: layerState.convStateGPU } },
718
+ { binding: 6, resource: { buffer: layerState.dtBiasGPU } },
719
+ { binding: 7, resource: { buffer: layerState.aNegExpGPU } },
720
+ { binding: 8, resource: { buffer: layerState.normWeightGPU } },
721
+ { binding: 9, resource: { buffer: layerState.recurrentStateGPU } },
722
+ { binding: 10, resource: { buffer: outputBuffer } },
723
+ ],
724
+ });
725
+ recordDispatch(
726
+ recorder,
727
+ pipelines.fusedDecodePipeline,
728
+ fusedBindGroup,
729
+ [layerState.numVHeads, 1, 1],
730
+ 'linear_attention_fused_decode_core'
731
+ );
732
+ const output = createTensor(
733
+ outputBuffer,
734
+ 'f32',
735
+ outputShape,
736
+ `L${options.layerIdx ?? 0}.linear_attention_core`
737
+ );
738
+ if (outputDtype === 'f16') {
739
+ assertImplicitDtypeTransitionAllowed({
740
+ executionPolicies: options.executionPolicies ?? null,
741
+ fromDtype: output.dtype,
742
+ toDtype: 'f16',
743
+ op: 'linear_attention_core',
744
+ detail: 'Linear attention core would narrow activations implicitly.',
745
+ });
746
+ const casted = await recordCastF32ToF16(recorder, output);
747
+ recorder.trackTemporaryBuffer(outputBuffer);
748
+ return casted;
749
+ }
750
+ return output;
751
+ } catch (error) {
752
+ releaseBuffer(outputBuffer);
753
+ throw error;
754
+ }
755
+ }
756
+
757
+ const paramsBuffer = createUniformBufferFromData(
758
+ 'linear_attention_params',
759
+ buildParamsData(paramsPayload),
760
+ null,
761
+ device,
762
+ { useCache: false }
763
+ );
764
+ let submitted = false;
765
+ try {
766
+ const fusedBindGroup = device.createBindGroup({
767
+ label: 'linear_attention_fused_decode_bind_group',
768
+ layout: fusedDecodeBindGroupLayout,
769
+ entries: [
770
+ { binding: 0, resource: { buffer: paramsBuffer } },
771
+ { binding: 1, resource: { buffer: qkvTensor.buffer } },
772
+ { binding: 2, resource: { buffer: zTensor.buffer } },
773
+ { binding: 3, resource: { buffer: aTensor.buffer } },
774
+ { binding: 4, resource: { buffer: layerState.convWeightGPU } },
775
+ { binding: 5, resource: { buffer: layerState.convStateGPU } },
776
+ { binding: 6, resource: { buffer: layerState.dtBiasGPU } },
777
+ { binding: 7, resource: { buffer: layerState.aNegExpGPU } },
778
+ { binding: 8, resource: { buffer: layerState.normWeightGPU } },
779
+ { binding: 9, resource: { buffer: layerState.recurrentStateGPU } },
780
+ { binding: 10, resource: { buffer: outputBuffer } },
781
+ ],
782
+ });
783
+ const encoder = device.createCommandEncoder({ label: 'linear_attention_fused_decode_core' });
784
+ const pass = encoder.beginComputePass({ label: 'linear_attention_fused_decode_core_pass' });
785
+ pass.setPipeline(pipelines.fusedDecodePipeline);
786
+ pass.setBindGroup(0, fusedBindGroup);
787
+ pass.dispatchWorkgroups(layerState.numVHeads, 1, 1);
788
+ pass.end();
789
+ device.queue.submit([encoder.finish()]);
790
+ submitted = true;
791
+
792
+ const output = createTensor(
793
+ outputBuffer,
794
+ 'f32',
795
+ outputShape,
796
+ `L${options.layerIdx ?? 0}.linear_attention_core`
797
+ );
798
+ if (outputDtype === 'f16') {
799
+ assertImplicitDtypeTransitionAllowed({
800
+ executionPolicies: options.executionPolicies ?? null,
801
+ fromDtype: output.dtype,
802
+ toDtype: 'f16',
803
+ op: 'linear_attention_core',
804
+ detail: 'Linear attention core would narrow activations implicitly.',
805
+ });
806
+ const casted = await castF32ToF16(output);
807
+ releaseBuffer(outputBuffer);
808
+ return casted;
809
+ }
810
+ return output;
811
+ } catch (error) {
812
+ releaseBuffer(outputBuffer);
813
+ throw error;
814
+ } finally {
815
+ if (submitted) {
816
+ device.queue.onSubmittedWorkDone()
817
+ .then(() => {
818
+ paramsBuffer.destroy();
819
+ })
820
+ .catch(() => {
821
+ paramsBuffer.destroy();
822
+ });
823
+ } else {
824
+ paramsBuffer.destroy();
825
+ }
826
+ }
827
+ }
828
+
829
+ convOutBuffer = acquireBuffer(convOutSize, undefined, `L${options.layerIdx ?? 0}.linear_conv_out`);
438
830
  if (useRecorder) {
439
831
  const paramsBuffer = createUniformBufferFromData(
440
832
  'linear_attention_params',
@@ -453,6 +845,9 @@ export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTe
453
845
  normMode: layerState.normMode === 'per_head' ? 1 : 0,
454
846
  rmsNormEps: Number(layerState.rmsNormEps) || 1e-6,
455
847
  qkL2NormEps: Number(options.qkL2NormEps) || 1e-6,
848
+ abPacked: options.abPacked === true,
849
+ qkvzPacked: options.qkvzPacked === true,
850
+ bProjOffsetElements: options.bProjOffsetElements,
456
851
  }),
457
852
  recorder
458
853
  );
@@ -546,6 +941,9 @@ export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTe
546
941
  normMode: layerState.normMode === 'per_head' ? 1 : 0,
547
942
  rmsNormEps: Number(layerState.rmsNormEps) || 1e-6,
548
943
  qkL2NormEps: Number(options.qkL2NormEps) || 1e-6,
944
+ abPacked: options.abPacked === true,
945
+ qkvzPacked: options.qkvzPacked === true,
946
+ bProjOffsetElements: options.bProjOffsetElements,
549
947
  }),
550
948
  null,
551
949
  device,
@@ -2,7 +2,7 @@ import type { Tensor } from '../tensor.js';
2
2
  import type { WeightBuffer } from '../weight-buffer.js';
3
3
  import type { CommandRecorder } from '../command-recorder.js';
4
4
 
5
- export interface LmHeadArgmaxF16Options {
5
+ export interface LmHeadArgmaxOptions {
6
6
  vocabSize: number;
7
7
  hiddenSize: number;
8
8
  padTokenId: number | null;
@@ -11,6 +11,15 @@ export interface LmHeadArgmaxF16Options {
11
11
  outputIndex: number;
12
12
  }
13
13
 
14
+ export type LmHeadArgmaxF16Options = LmHeadArgmaxOptions;
15
+
16
+ export declare function recordLmHeadArgmax(
17
+ recorder: CommandRecorder,
18
+ inputTensor: Tensor,
19
+ lmHead: WeightBuffer,
20
+ options: LmHeadArgmaxOptions
21
+ ): Promise<GPUBuffer>;
22
+
14
23
  export declare function recordLmHeadArgmaxF16(
15
24
  recorder: CommandRecorder,
16
25
  inputTensor: Tensor,