doppler-gpu 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1589 -0
  4. package/package.json +44 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  16. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  17. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  18. package/src/config/kernel-path-loader.js +1 -0
  19. package/src/config/kernels/backward-registry.json +13 -2
  20. package/src/config/kernels/kernel-ref-digests.js +34 -21
  21. package/src/config/kernels/registry.json +1402 -2623
  22. package/src/config/merge.js +11 -0
  23. package/src/config/param-validator.js +25 -0
  24. package/src/config/runtime/default.json +3 -1
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  29. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  30. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  31. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  32. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  33. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  34. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  35. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  36. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  37. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  38. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  39. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  40. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  41. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  42. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  43. package/src/config/schema/doppler.schema.js +40 -5
  44. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  45. package/src/config/schema/execution-v1.schema.js +10 -0
  46. package/src/config/schema/index.d.ts +3 -1
  47. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  48. package/src/config/schema/training-metrics.schema.js +4 -0
  49. package/src/config/transforms/capability-transform-resolver.js +7 -1
  50. package/src/config/transforms/execution-graph-transforms.js +2 -2
  51. package/src/debug/config.d.ts +4 -0
  52. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  53. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  54. package/src/experimental/orchestration/index.d.ts +10 -1
  55. package/src/experimental/training/attention-backward.js +123 -55
  56. package/src/experimental/training/autograd.js +101 -5
  57. package/src/experimental/training/clip.js +6 -1
  58. package/src/experimental/training/datasets/text-pairs.js +80 -8
  59. package/src/experimental/training/distillation/student-fixture.js +470 -114
  60. package/src/experimental/training/lora-pipeline.js +202 -27
  61. package/src/experimental/training/lora.js +10 -4
  62. package/src/experimental/training/optimizer.js +9 -1
  63. package/src/experimental/training/runner.js +18 -4
  64. package/src/experimental/training/suite.js +1 -1
  65. package/src/experimental/training/workloads.js +11 -0
  66. package/src/formats/rdrr/classification.js +3 -0
  67. package/src/formats/rdrr/integrity.d.ts +0 -2
  68. package/src/formats/rdrr/integrity.js +0 -2
  69. package/src/formats/rdrr/types.d.ts +1 -1
  70. package/src/gpu/kernels/attention.d.ts +2 -0
  71. package/src/gpu/kernels/attention.js +75 -16
  72. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  73. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  74. package/src/gpu/kernels/backward/adam.js +6 -4
  75. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  76. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  77. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  78. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  79. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  80. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  81. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  82. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  83. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  84. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  85. package/src/gpu/kernels/backward/index.d.ts +3 -0
  86. package/src/gpu/kernels/backward/index.js +3 -0
  87. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  88. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  89. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  90. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  91. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  92. package/src/gpu/kernels/backward/utils.js +9 -2
  93. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  94. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  95. package/src/gpu/kernels/fused_ffn.js +540 -19
  96. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  97. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  98. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  99. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  100. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  101. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  102. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  103. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  104. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  105. package/src/gpu/kernels/index.d.ts +26 -0
  106. package/src/gpu/kernels/index.js +18 -0
  107. package/src/gpu/kernels/linear-attention-core.js +413 -15
  108. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  109. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  110. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  111. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  112. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  113. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  114. package/src/gpu/kernels/matmul-selection.js +10 -1
  115. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  116. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  117. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  118. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  119. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  120. package/src/gpu/kernels/silu.d.ts +1 -0
  121. package/src/gpu/kernels/silu.js +19 -4
  122. package/src/gpu/kernels/silu.wgsl +7 -1
  123. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  124. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  125. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  126. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  127. package/src/gpu/tensor.d.ts +1 -0
  128. package/src/index.d.ts +1 -1
  129. package/src/index.js +1 -1
  130. package/src/inference/browser-harness-suite-helpers.js +1 -1
  131. package/src/inference/browser-harness-text-helpers.js +251 -20
  132. package/src/inference/browser-harness.js +221 -3
  133. package/src/inference/kv-cache/base.js +42 -1
  134. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  135. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  136. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  137. package/src/inference/pipelines/text/attention/projections.js +174 -4
  138. package/src/inference/pipelines/text/attention/record.js +44 -21
  139. package/src/inference/pipelines/text/attention/run.js +18 -3
  140. package/src/inference/pipelines/text/config.d.ts +18 -0
  141. package/src/inference/pipelines/text/config.js +100 -0
  142. package/src/inference/pipelines/text/embed.js +1 -1
  143. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  144. package/src/inference/pipelines/text/execution-plan.js +37 -0
  145. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  146. package/src/inference/pipelines/text/execution-v1.js +130 -10
  147. package/src/inference/pipelines/text/ffn/dense.js +241 -21
  148. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  149. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  152. package/src/inference/pipelines/text/generator-steps.js +2 -2
  153. package/src/inference/pipelines/text/generator.d.ts +18 -1
  154. package/src/inference/pipelines/text/generator.js +326 -19
  155. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  156. package/src/inference/pipelines/text/init.d.ts +22 -3
  157. package/src/inference/pipelines/text/init.js +2 -2
  158. package/src/inference/pipelines/text/layer.js +87 -8
  159. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  160. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  161. package/src/inference/pipelines/text/linear-attention.js +173 -16
  162. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  163. package/src/inference/pipelines/text/logits/index.js +62 -1
  164. package/src/inference/pipelines/text/lora-apply.js +4 -4
  165. package/src/inference/pipelines/text/model-load.js +6 -0
  166. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  167. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  168. package/src/inference/pipelines/text/ops.js +22 -0
  169. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  170. package/src/inference/pipelines/text/probes.js +9 -4
  171. package/src/inference/pipelines/text/state.d.ts +4 -4
  172. package/src/inference/pipelines/text/state.js +4 -0
  173. package/src/inference/pipelines/text/types.d.ts +27 -2
  174. package/src/inference/pipelines/text.d.ts +21 -1
  175. package/src/inference/pipelines/text.js +40 -14
  176. package/src/inference/tokenizers/bundled.js +150 -55
  177. package/src/inference/tokenizers/types.d.ts +7 -4
  178. package/src/loader/doppler-loader.js +25 -2
  179. package/src/loader/index.d.ts +3 -1
  180. package/src/loader/index.js +2 -0
  181. package/src/loader/loader-types.d.ts +3 -1
  182. package/src/loader/manifest-config.d.ts +18 -0
  183. package/src/loader/manifest-config.js +11 -3
  184. package/src/loader/shard-cache.d.ts +2 -0
  185. package/src/loader/shard-cache.js +29 -1
  186. package/src/models/gemma4.js +8 -0
  187. package/src/rules/inference/capability-transforms.rules.json +145 -20
  188. package/src/rules/inference/ffn.rules.json +33 -0
  189. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  190. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  191. package/src/rules/kernels/matmul.rules.json +2 -0
  192. package/src/rules/kernels/silu.rules.json +1 -0
  193. package/src/rules/rule-registry.js +2 -0
  194. package/src/storage/artifact-storage-context.js +36 -0
  195. package/src/tooling/node-browser-command-runner.js +42 -7
  196. package/src/tooling/node-converter.js +20 -5
  197. package/src/tooling/opfs-cache.d.ts +21 -1
  198. package/src/tooling/opfs-cache.js +168 -32
  199. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  200. package/src/tooling-experimental-exports.shared.js +18 -0
  201. package/src/tooling-exports/storage.d.ts +1 -1
  202. package/src/tooling-exports/storage.js +1 -1
  203. package/src/types/gpu.d.ts +25 -0
  204. package/src/version.js +1 -1
  205. package/tests/kernels/browser/kernel-suite.js +1593 -0
  206. package/tests/kernels/browser/test-page.js +2629 -0
  207. package/tests/kernels/harness/benchmark.js +225 -0
  208. package/tests/kernels/harness/buffer-utils.js +88 -0
  209. package/tests/kernels/harness/tolerance.js +174 -0
  210. package/tests/kernels/reference/adam.js +15 -0
  211. package/tests/kernels/reference/attention.js +158 -0
  212. package/tests/kernels/reference/backward.js +96 -0
  213. package/tests/kernels/reference/bias-add.js +20 -0
  214. package/tests/kernels/reference/conv2d.js +58 -0
  215. package/tests/kernels/reference/cross-entropy.js +26 -0
  216. package/tests/kernels/reference/dequant.js +312 -0
  217. package/tests/kernels/reference/embed.js +15 -0
  218. package/tests/kernels/reference/gather.js +61 -0
  219. package/tests/kernels/reference/gelu.js +37 -0
  220. package/tests/kernels/reference/groupnorm.js +36 -0
  221. package/tests/kernels/reference/index.js +89 -0
  222. package/tests/kernels/reference/layernorm.js +22 -0
  223. package/tests/kernels/reference/matmul.js +57 -0
  224. package/tests/kernels/reference/modulate.js +24 -0
  225. package/tests/kernels/reference/moe-gather.js +83 -0
  226. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  227. package/tests/kernels/reference/residual.js +55 -0
  228. package/tests/kernels/reference/rmsnorm.js +52 -0
  229. package/tests/kernels/reference/rope.js +79 -0
  230. package/tests/kernels/reference/sample.js +97 -0
  231. package/tests/kernels/reference/scatter-add.js +58 -0
  232. package/tests/kernels/reference/silu.js +52 -0
  233. package/tests/kernels/reference/softmax.js +86 -0
  234. package/tests/kernels/reference/split-qkv.js +58 -0
  235. package/tests/kernels/reference/topk.js +88 -0
  236. package/tests/kernels/reference/upsample2d.js +37 -0
  237. package/BRANDING.md +0 -14
  238. package/SECURITY.md +0 -19
  239. package/assets/doppler-runtime-map.svg +0 -82
  240. package/src/bootstrap.js +0 -30
  241. package/src/cli/doppler-cli.d.ts +0 -46
  242. package/src/cli/doppler-quickstart.d.ts +0 -45
  243. package/src/cli/doppler-serve.d.ts +0 -210
  244. package/src/cli/runtime-profiles.d.ts +0 -40
  245. package/src/client/doppler-api.browser.d.ts +0 -1
  246. package/src/client/doppler-api.browser.js +0 -25
  247. package/src/client/doppler-registry.d.ts +0 -54
  248. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  249. package/src/client/runtime/source-runtime.d.ts +0 -26
  250. package/src/client/runtime/storage.d.ts +0 -5
  251. package/src/config/README.md +0 -61
  252. package/src/config/conversion/README.md +0 -62
  253. package/src/config/goal-completion-matrix.json +0 -552
  254. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  255. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  256. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  257. package/src/config/kernels/registry.d.ts +0 -51
  258. package/src/config/merge-contract-check.d.ts +0 -16
  259. package/src/config/merge-contract-check.js +0 -319
  260. package/src/config/merge-helpers.d.ts +0 -60
  261. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  262. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  263. package/src/config/param-categories.d.ts +0 -8
  264. package/src/config/param-validator.d.ts +0 -33
  265. package/src/config/platforms/loader.d.ts +0 -38
  266. package/src/config/quantization-contract-check.d.ts +0 -12
  267. package/src/config/quantization-contract-check.js +0 -91
  268. package/src/config/runtime/README.md +0 -28
  269. package/src/config/runtime-merge.d.ts +0 -5
  270. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  271. package/src/config/schema/command-surface-contract.schema.json +0 -81
  272. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  273. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  274. package/src/config/schema/platform.schema.js +0 -1
  275. package/src/config/schema/policy-schema-registry.json +0 -31
  276. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  277. package/src/config/support-tiers/subsystems.json +0 -502
  278. package/src/config/supported-operations.d.ts +0 -14
  279. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  280. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  281. package/src/converter/conversion-plan.d.ts +0 -56
  282. package/src/converter/index.d.ts +0 -98
  283. package/src/converter/index.js +0 -59
  284. package/src/converter/parsers/diffusion.d.ts +0 -50
  285. package/src/converter/parsers/gguf.d.ts +0 -31
  286. package/src/converter/parsers/index.d.ts +0 -21
  287. package/src/converter/parsers/index.js +0 -12
  288. package/src/converter/parsers/transformer.d.ts +0 -31
  289. package/src/converter/quantization-info.d.ts +0 -38
  290. package/src/converter/rope-config.d.ts +0 -16
  291. package/src/converter/tokenizer-utils.d.ts +0 -12
  292. package/src/debug/reference/README.md +0 -114
  293. package/src/debug/reference/hf_attn_debug.py +0 -114
  294. package/src/debug/reference/hf_embed_check.py +0 -89
  295. package/src/debug/reference/hf_layer_out.py +0 -100
  296. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  297. package/src/debug/reference/hf_rope_check.py +0 -116
  298. package/src/debug/reference/hf_weights.py +0 -75
  299. package/src/errors/doppler-error.d.ts +0 -20
  300. package/src/experimental/bridge/extension/background.d.ts +0 -14
  301. package/src/experimental/bridge/extension/background.js +0 -168
  302. package/src/experimental/bridge/extension/manifest.json +0 -34
  303. package/src/experimental/bridge/extension-client.d.ts +0 -114
  304. package/src/experimental/bridge/index.d.ts +0 -69
  305. package/src/experimental/bridge/node-bridge.js +0 -24
  306. package/src/experimental/bridge/protocol.d.ts +0 -96
  307. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  308. package/src/experimental/browser/gguf-importer.js +0 -532
  309. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  310. package/src/experimental/browser/quantization.d.ts +0 -74
  311. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  312. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  313. package/src/experimental/browser/shard-io-browser.js +0 -89
  314. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  315. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  316. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  317. package/src/experimental/distribution/p2p-transport.js +0 -75
  318. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  319. package/src/experimental/hotswap/manifest.d.ts +0 -42
  320. package/src/experimental/hotswap/model-swap.js +0 -42
  321. package/src/experimental/hotswap/runtime.d.ts +0 -31
  322. package/src/experimental/training/README.md +0 -164
  323. package/src/experimental/training/artifacts.d.ts +0 -160
  324. package/src/experimental/training/attention-backward.d.ts +0 -30
  325. package/src/experimental/training/attention-forward.d.ts +0 -22
  326. package/src/experimental/training/attention-forward.js +0 -82
  327. package/src/experimental/training/autograd.d.ts +0 -51
  328. package/src/experimental/training/backward-kernels.js +0 -39
  329. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  330. package/src/experimental/training/checkpoint.d.ts +0 -36
  331. package/src/experimental/training/clip.d.ts +0 -9
  332. package/src/experimental/training/dataloader.d.ts +0 -8
  333. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  334. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  335. package/src/experimental/training/datasets/index.d.ts +0 -24
  336. package/src/experimental/training/datasets/index.js +0 -13
  337. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  338. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  339. package/src/experimental/training/datasets/reploid.js +0 -36
  340. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  341. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  342. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  343. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  344. package/src/experimental/training/datasets/token-batch.js +0 -52
  345. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  346. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  347. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  348. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  349. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  350. package/src/experimental/training/distillation/eval.d.ts +0 -34
  351. package/src/experimental/training/distillation/index.d.ts +0 -29
  352. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  353. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  354. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  355. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  356. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  357. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  358. package/src/experimental/training/export.d.ts +0 -41
  359. package/src/experimental/training/index.d.ts +0 -74
  360. package/src/experimental/training/index.js +0 -63
  361. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  362. package/src/experimental/training/lora.d.ts +0 -19
  363. package/src/experimental/training/loss-scaling.d.ts +0 -21
  364. package/src/experimental/training/loss.d.ts +0 -10
  365. package/src/experimental/training/objectives/base.d.ts +0 -58
  366. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  367. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  368. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  369. package/src/experimental/training/objectives/index.d.ts +0 -12
  370. package/src/experimental/training/objectives/index.js +0 -6
  371. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  372. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  373. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  374. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  375. package/src/experimental/training/operator-command.d.ts +0 -5
  376. package/src/experimental/training/operator-eval.d.ts +0 -48
  377. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  378. package/src/experimental/training/optimizer.d.ts +0 -22
  379. package/src/experimental/training/runner.d.ts +0 -249
  380. package/src/experimental/training/suite.d.ts +0 -299
  381. package/src/experimental/training/tensor-factory.d.ts +0 -9
  382. package/src/experimental/training/trainer.d.ts +0 -89
  383. package/src/experimental/training/ul_dataset.d.ts +0 -47
  384. package/src/experimental/training/ul_schedule.d.ts +0 -6
  385. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  386. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  387. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  388. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  389. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  390. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  391. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  392. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  393. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  394. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  395. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  396. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  397. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  398. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  399. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  400. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  401. package/src/experimental/training/workload-packs/registry.json +0 -278
  402. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  403. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  404. package/src/experimental/training/workloads.d.ts +0 -201
  405. package/src/formats/gguf/types.d.ts +0 -74
  406. package/src/formats/index.d.ts +0 -75
  407. package/src/formats/index.js +0 -19
  408. package/src/formats/litert/types.d.ts +0 -74
  409. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  410. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  411. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  412. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  413. package/src/formats/safetensors/types.d.ts +0 -67
  414. package/src/formats/tflite/types.d.ts +0 -61
  415. package/src/formats/tokenizer/index.d.ts +0 -5
  416. package/src/formats/tokenizer/index.js +0 -3
  417. package/src/formats/tokenizer/types.d.ts +0 -9
  418. package/src/gpu/kernel-runtime.d.ts +0 -13
  419. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  420. package/src/gpu/kernel-selection-log.d.ts +0 -4
  421. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  422. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  423. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  424. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  425. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  426. package/src/gpu/kernel-tuner/types.js +0 -4
  427. package/src/gpu/kernel-tuner.d.ts +0 -33
  428. package/src/gpu/kernels/README.md +0 -166
  429. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  430. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  431. package/src/gpu/kernels/check-stop.d.ts +0 -31
  432. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  433. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  434. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  435. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  436. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  437. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  438. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  439. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  440. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  441. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  442. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  443. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  444. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  445. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  446. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  447. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  448. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  449. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  450. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  451. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  452. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  453. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  454. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  455. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  456. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  457. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  458. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  459. package/src/gpu/kernels/constants.d.ts +0 -129
  460. package/src/gpu/kernels/dispatch.d.ts +0 -72
  461. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  462. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  463. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  464. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  465. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  466. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  467. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  468. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  469. package/src/gpu/kernels/types.js +0 -4
  470. package/src/gpu/perf-guards.d.ts +0 -13
  471. package/src/gpu/profiler.d.ts +0 -93
  472. package/src/gpu/readback-utils.d.ts +0 -16
  473. package/src/gpu/readback-utils.js +0 -49
  474. package/src/gpu/submit-probe.d.ts +0 -12
  475. package/src/index-browser.d.ts +0 -2
  476. package/src/index-browser.js +0 -2
  477. package/src/inference/README.md +0 -593
  478. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  479. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  480. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  481. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  482. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  483. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  484. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  485. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  486. package/src/inference/network-evolution.js +0 -79
  487. package/src/inference/pipelines/context.d.ts +0 -21
  488. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  489. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  490. package/src/inference/pipelines/diffusion/index.js +0 -3
  491. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  492. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  493. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  494. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  495. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  496. package/src/inference/pipelines/diffusion/types.js +0 -1
  497. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  498. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  499. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  500. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  501. package/src/inference/pipelines/dream/pipeline.js +0 -8
  502. package/src/inference/pipelines/energy/index.d.ts +0 -1
  503. package/src/inference/pipelines/energy/index.js +0 -1
  504. package/src/inference/pipelines/energy/types.js +0 -1
  505. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  506. package/src/inference/pipelines/energy-head/index.js +0 -6
  507. package/src/inference/pipelines/factory.d.ts +0 -10
  508. package/src/inference/pipelines/index.d.ts +0 -22
  509. package/src/inference/pipelines/index.js +0 -19
  510. package/src/inference/pipelines/registry.d.ts +0 -15
  511. package/src/inference/pipelines/rng.d.ts +0 -2
  512. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  513. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  514. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  515. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  516. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  517. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  518. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  519. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  520. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  521. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  522. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  523. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  524. package/src/inference/pipelines/text/attention.d.ts +0 -23
  525. package/src/inference/pipelines/text/attention.js +0 -12
  526. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  527. package/src/inference/pipelines/text/buffer-types.js +0 -4
  528. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  529. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  530. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  531. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  532. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  533. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  534. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  535. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  536. package/src/inference/pipelines/text/embed.d.ts +0 -69
  537. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  538. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  539. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  540. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  541. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  542. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  543. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  544. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  545. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  546. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  547. package/src/inference/pipelines/text/ffn.js +0 -18
  548. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  549. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  550. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  551. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  552. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  553. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  554. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  555. package/src/inference/pipelines/text/index.d.ts +0 -5
  556. package/src/inference/pipelines/text/index.js +0 -6
  557. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  558. package/src/inference/pipelines/text/layer.d.ts +0 -118
  559. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  560. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  561. package/src/inference/pipelines/text/logits/types.js +0 -4
  562. package/src/inference/pipelines/text/logits.d.ts +0 -27
  563. package/src/inference/pipelines/text/logits.js +0 -16
  564. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  565. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  566. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  567. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  568. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  569. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  570. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  571. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  572. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  573. package/src/inference/pipelines/text/ops.d.ts +0 -222
  574. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  575. package/src/inference/pipelines/text/probes.d.ts +0 -28
  576. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  577. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  578. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  579. package/src/inference/pipelines/text/types.js +0 -4
  580. package/src/inference/tokenizers/base.d.ts +0 -44
  581. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  582. package/src/inference/tokenizers/bpe.d.ts +0 -27
  583. package/src/inference/tokenizers/bundled.d.ts +0 -67
  584. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  585. package/src/inference/tokenizers/types.js +0 -7
  586. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  587. package/src/loader/per-layer-input-loader.d.ts +0 -30
  588. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  589. package/src/loader/tensors/source-transform.d.ts +0 -65
  590. package/src/memory/address-table.d.ts +0 -96
  591. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  592. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  593. package/src/rules/rule-registry.d.ts +0 -77
  594. package/src/storage/artifact-storage-context.d.ts +0 -63
  595. package/src/storage/backends/idb-store.d.ts +0 -52
  596. package/src/storage/backends/memory-store.d.ts +0 -36
  597. package/src/storage/backends/opfs-store.d.ts +0 -41
  598. package/src/storage/blake3.d.ts +0 -17
  599. package/src/storage/download-types.d.ts +0 -157
  600. package/src/storage/downloader.d.ts +0 -103
  601. package/src/storage/index.d.ts +0 -274
  602. package/src/storage/index.js +0 -201
  603. package/src/storage/preflight.d.ts +0 -144
  604. package/src/storage/preflight.js +0 -360
  605. package/src/storage/quickstart-downloader.d.ts +0 -157
  606. package/src/storage/quickstart-downloader.js +0 -289
  607. package/src/storage/source-artifact-store.d.ts +0 -59
  608. package/src/sw.js +0 -184
  609. package/src/tooling/command-api-constants.d.ts +0 -9
  610. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  611. package/src/tooling/command-api-helpers.d.ts +0 -23
  612. package/src/tooling/command-runner-shared.d.ts +0 -89
  613. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  614. package/src/tooling/conversion-config-materializer.js +0 -96
  615. package/src/tooling/diagnose-runner.d.ts +0 -28
  616. package/src/tooling/hf-registry-utils.d.ts +0 -55
  617. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  618. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  619. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  620. package/src/tooling/kernel-path-builder/index.js +0 -1361
  621. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  622. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  623. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  624. package/src/tooling/lean-execution-contract-runner.js +0 -158
  625. package/src/tooling/lean-execution-contract.d.ts +0 -16
  626. package/src/tooling/lean-execution-contract.js +0 -81
  627. package/src/tooling/litert-package-runtime.d.ts +0 -46
  628. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  629. package/src/tooling/node-convert-worker.d.ts +0 -1
  630. package/src/tooling/node-converter.d.ts +0 -1
  631. package/src/tooling/node-file-fetch.d.ts +0 -1
  632. package/src/tooling/node-source-runtime.d.ts +0 -24
  633. package/src/tooling/node-webgpu.d.ts +0 -17
  634. package/src/tooling/precision-replay-math.js +0 -174
  635. package/src/tooling/program-bundle-host.d.ts +0 -16
  636. package/src/tooling/program-bundle-host.js +0 -20
  637. package/src/tooling/runtime-input-composition.d.ts +0 -37
  638. package/src/tooling/source-package-profiles.d.ts +0 -50
  639. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  640. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  641. package/src/tooling/source-runtime-materializer.js +0 -89
  642. package/src/tooling-exports.browser.d.ts +0 -7
  643. package/src/types/chrome.d.ts +0 -36
  644. package/src/types/chrome.js +0 -1
  645. package/src/types/index.d.ts +0 -1
  646. package/src/types/index.js +0 -1
  647. package/src/utils/array-buffer.d.ts +0 -4
  648. package/src/utils/canonical-hash.d.ts +0 -7
  649. package/src/utils/clone-json.d.ts +0 -1
  650. package/src/utils/hf-resolve-url.d.ts +0 -16
  651. package/src/utils/index.d.ts +0 -10
  652. package/src/utils/index.js +0 -10
  653. package/src/utils/load-json.d.ts +0 -5
  654. package/src/utils/plain-object.d.ts +0 -1
  655. package/src/utils/sha256.d.ts +0 -5
  656. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -1,515 +0,0 @@
1
- --- src/gpu/kernels/rmsnorm.wgsl 2026-06-08 16:21:47.561276560 -0400
2
- +++ src/gpu/kernels/rmsnorm_f16.wgsl 2026-06-08 16:32:38.675963250 -0400
3
- @@ -1,46 +1,28 @@
4
- -// RMSNorm Kernel with Fused Residual Add
5
- +// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
6
- +// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
7
- +// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
8
- +// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
9
- //
10
- -// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
11
- -//
12
- -// Optionally fuses residual addition (POST-norm, for Gemma 3 sandwich norm):
13
- -// output = residual + RMSNorm(x) * weight
14
- -//
15
- -// Uses workgroup reduction for efficient mean calculation.
16
- -// Subgroup variants use subgroupAdd() for 3-5x faster reductions.
17
- +// F16 variant for reduced memory bandwidth when using F16 activations.
18
- +// The all-f16 lane keeps normalization arithmetic in f16.
19
- +// Weight buffer may be F16 or F32.
20
- //
21
- -// Uses override constants for compile-time feature selection:
22
- -// - RMS_NORM_OFFSET: Use (1 + weight) pattern (Gemma models)
23
- -// - HAS_RESIDUAL: Compile-time override (reserved for codegen patches, not used by JS dispatch)
24
- -// - OUTPUT_PRENORM: Output pre-normalized values (before weight multiplication)
25
- -
26
- -enable subgroups;
27
- -
28
- -// =============================================================================
29
- -// Override Constants (compile-time configuration)
30
- -// =============================================================================
31
- +// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
32
- +
33
- +enable f16;
34
-
35
- override WORKGROUP_SIZE: u32 = 256u;
36
- const MAX_WORKGROUP_SIZE: u32 = 256u;
37
- -
38
- -// Feature flags - compiler eliminates dead branches
39
- override RMS_NORM_OFFSET: bool = false; // Use (1 + weight) for Gemma models
40
- -override HAS_RESIDUAL: bool = false; // Add residual after normalization
41
- override PRE_RESIDUAL: bool = false; // Add residual BEFORE normalization: rmsnorm(input + residual)
42
- override OUTPUT_PRENORM: bool = false; // Write pre-norm sum to residual_sum_output binding
43
- override WEIGHT_IS_F16: bool = false; // Weight buffer packed as f16 pairs
44
-
45
- -const MAX_SUBGROUPS: u32 = 32u; // Support up to 32 subgroups per workgroup
46
- -const MAX_CACHE_SIZE: u32 = 4608u; // Supports hiddenSize up to 1152 with 4 elements/thread
47
- -
48
- -// =============================================================================
49
- -// Uniforms (per-dispatch)
50
- -// =============================================================================
51
- -
52
- struct Uniforms {
53
- size: u32, // Hidden dimension
54
- num_tokens: u32, // Number of tokens to process
55
- - eps: f32, // Epsilon for numerical stability (typically 1e-5 or 1e-6)
56
- - has_residual: u32, // Runtime flag: 1 = add residual after norm
57
- + eps: f32, // Epsilon for numerical stability
58
- + has_residual: u32, // 1 if residual input provided, 0 otherwise
59
- token_stride: u32, // Workgroup rows per dispatch row
60
- output_scale: f32, // Output epilogue scale
61
- _pad1: u32,
62
- @@ -48,47 +30,32 @@
63
- }
64
-
65
- @group(0) @binding(0) var<uniform> u: Uniforms;
66
- -@group(0) @binding(1) var<storage, read> input: array<f32>;
67
- -@group(0) @binding(2) var<storage, read> weight: array<u32>; // [size] as f32 or packed f16
68
- -@group(0) @binding(3) var<storage, read_write> output: array<f32>;
69
- -@group(0) @binding(4) var<storage, read> residual: array<f32>; // Optional residual input
70
- -@group(0) @binding(5) var<storage, read_write> residual_sum_output: array<f32>; // Pre-norm sum output (PRE_RESIDUAL + OUTPUT_PRENORM)
71
- -
72
- -// =============================================================================
73
- -// Shared Memory
74
- -// =============================================================================
75
- -
76
- -var<workgroup> shared_sum: array<f32, MAX_WORKGROUP_SIZE>;
77
- -
78
- -// =============================================================================
79
- -// Helper Functions
80
- -// =============================================================================
81
- +@group(0) @binding(1) var<storage, read> input: array<f16>;
82
- +@group(0) @binding(2) var<storage, read> weight: array<u32>; // F32 or packed F16
83
- +@group(0) @binding(3) var<storage, read_write> output: array<f16>;
84
- +@group(0) @binding(4) var<storage, read> residual: array<f16>; // Optional residual
85
- +@group(0) @binding(5) var<storage, read_write> residual_sum_output: array<f16>; // Pre-norm sum output (PRE_RESIDUAL + OUTPUT_PRENORM)
86
-
87
- -// Apply weight with optional offset (Gemma uses 1 + weight pattern)
88
- -fn apply_weight(w: f32) -> f32 {
89
- +var<workgroup> shared_sum: array<f16, MAX_WORKGROUP_SIZE>;
90
- +
91
- +fn apply_weight(w: f16) -> f16 {
92
- if (RMS_NORM_OFFSET) {
93
- - return 1.0 + w;
94
- - } else {
95
- - return w;
96
- + return f16(1.0) + w;
97
- }
98
- + return w;
99
- }
100
-
101
- -fn load_weight(idx: u32) -> f32 {
102
- +fn load_weight(idx: u32) -> f16 {
103
- if (WEIGHT_IS_F16) {
104
- let packed = weight[idx >> 1u];
105
- let pair = unpack2x16float(packed);
106
- - return select(pair.x, pair.y, (idx & 1u) == 1u);
107
- + return f16(select(pair.x, pair.y, (idx & 1u) == 1u));
108
- }
109
- - return bitcast<f32>(weight[idx]);
110
- -}
111
- -
112
- -// Check if residual should be added (runtime uniform only — JS dispatch controls this)
113
- -fn should_add_residual() -> bool {
114
- - return u.has_residual != 0u;
115
- + return f16(bitcast<f32>(weight[idx]));
116
- }
117
-
118
- // Load input value, fusing residual add when PRE_RESIDUAL is active
119
- -fn load_input(base_offset: u32, idx: u32) -> f32 {
120
- +fn load_input(base_offset: u32, idx: u32) -> f16 {
121
- let x = input[base_offset + idx];
122
- if (PRE_RESIDUAL) {
123
- return x + residual[base_offset + idx];
124
- @@ -97,24 +64,16 @@
125
- }
126
-
127
- // Write pre-norm sum when OUTPUT_PRENORM is active (for downstream residual reuse)
128
- -fn write_prenorm(base_offset: u32, idx: u32, val: f32) {
129
- +fn write_prenorm(base_offset: u32, idx: u32, val: f16) {
130
- if (OUTPUT_PRENORM && PRE_RESIDUAL) {
131
- residual_sum_output[base_offset + idx] = val;
132
- }
133
- }
134
-
135
- -fn apply_output_scale(value: f32) -> f32 {
136
- - return value * u.output_scale;
137
- -}
138
- -
139
- fn token_index(wg_id: vec3<u32>) -> u32 {
140
- return wg_id.y * max(u.token_stride, 1u) + wg_id.x;
141
- }
142
-
143
- -// =============================================================================
144
- -// Main Entry Point
145
- -// =============================================================================
146
- -
147
- // Main RMSNorm kernel - one workgroup per token
148
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
149
- fn main(
150
- @@ -132,151 +91,44 @@
151
-
152
- let base_offset = token_idx * size;
153
-
154
- - // Each thread computes partial sum of squares
155
- - // When PRE_RESIDUAL is active, sum is computed on (input + residual)
156
- - var local_sum_sq: f32 = 0.0;
157
- let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
158
-
159
- + var local_max: f16 = f16(0.0);
160
- for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
161
- let idx = thread_idx * elements_per_thread + i;
162
- if (idx < size) {
163
- - let x = load_input(base_offset, idx);
164
- - local_sum_sq = local_sum_sq + x * x;
165
- + local_max = max(local_max, abs(load_input(base_offset, idx)));
166
- }
167
- }
168
-
169
- - // Store local sum for reduction
170
- - shared_sum[thread_idx] = local_sum_sq;
171
- + shared_sum[thread_idx] = local_max;
172
- workgroupBarrier();
173
-
174
- - // Parallel reduction to compute total sum of squares
175
- for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
176
- if (thread_idx < stride) {
177
- - shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
178
- + shared_sum[thread_idx] = max(shared_sum[thread_idx], shared_sum[thread_idx + stride]);
179
- }
180
- workgroupBarrier();
181
- }
182
-
183
- - // Compute RMS
184
- - let mean_sq = shared_sum[0] / f32(size);
185
- - let rms = sqrt(mean_sq + u.eps);
186
- - let inv_rms = 1.0 / rms;
187
- -
188
- + let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
189
- workgroupBarrier();
190
-
191
- - // Apply normalization and weight
192
- - for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
193
- - let idx = thread_idx * elements_per_thread + i;
194
- - if (idx < size) {
195
- - let x = load_input(base_offset, idx);
196
- -
197
- - // Normalize and scale (with optional weight offset for Gemma)
198
- - var result = x * inv_rms * apply_weight(load_weight(idx));
199
- -
200
- - // Add residual AFTER normalization (Gemma 3 sandwich norm pattern)
201
- - if (should_add_residual()) {
202
- - result = result + residual[base_offset + idx];
203
- - }
204
- -
205
- - // Write pre-norm sum for downstream residual reuse
206
- - write_prenorm(base_offset, idx, x);
207
- -
208
- - output[base_offset + idx] = apply_output_scale(result);
209
- - }
210
- - }
211
- -}
212
- -
213
- -// =============================================================================
214
- -// Small Hidden Size Entry Point
215
- -// =============================================================================
216
- -
217
- -// Optimized version for hidden size <= WORKGROUP_SIZE (single pass)
218
- -@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
219
- -fn main_small(
220
- - @builtin(local_invocation_id) local_id: vec3<u32>,
221
- - @builtin(workgroup_id) wg_id: vec3<u32>
222
- -) {
223
- - let token_idx = token_index(wg_id);
224
- - let thread_idx = local_id.x;
225
- - let size = u.size;
226
- -
227
- - if (token_idx >= u.num_tokens) {
228
- - return;
229
- - }
230
- -
231
- - let base_offset = token_idx * size;
232
- -
233
- - // Each thread handles one element (for size <= 256)
234
- - var x: f32 = 0.0;
235
- - if (thread_idx < size) {
236
- - x = load_input(base_offset, thread_idx);
237
- - }
238
- -
239
- - // Sum of squares
240
- - shared_sum[thread_idx] = x * x;
241
- - workgroupBarrier();
242
- -
243
- - // Parallel reduction
244
- - for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
245
- - if (thread_idx < stride && thread_idx + stride < size) {
246
- - shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
247
- - }
248
- - workgroupBarrier();
249
- - }
250
- -
251
- - // Compute inverse RMS
252
- - let mean_sq = shared_sum[0] / f32(size);
253
- - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
254
- -
255
- - // Apply normalization
256
- - if (thread_idx < size) {
257
- - var result = x * inv_rms * apply_weight(load_weight(thread_idx));
258
- - if (should_add_residual()) {
259
- - result = result + residual[base_offset + thread_idx];
260
- - }
261
- - write_prenorm(base_offset, thread_idx, x);
262
- - output[base_offset + thread_idx] = apply_output_scale(result);
263
- - }
264
- -}
265
- -
266
- -// =============================================================================
267
- -// Cached Input Entry Point (avoids double loads)
268
- -// =============================================================================
269
- -
270
- -// OPTIMIZED: Caches input to avoid double loads
271
- -// Uses shared memory to store input values between passes
272
- -var<workgroup> shared_cache: array<f32, MAX_CACHE_SIZE>;
273
- -
274
- -@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
275
- -fn main_cached(
276
- - @builtin(local_invocation_id) local_id: vec3<u32>,
277
- - @builtin(workgroup_id) wg_id: vec3<u32>
278
- -) {
279
- - let token_idx = token_index(wg_id);
280
- - let thread_idx = local_id.x;
281
- - let size = u.size;
282
- -
283
- - if (token_idx >= u.num_tokens) {
284
- - return;
285
- - }
286
- -
287
- - let base_offset = token_idx * size;
288
- - let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
289
- -
290
- - // First pass: cache input and compute sum of squares
291
- - var local_sum_sq: f32 = 0.0;
292
- + var local_sum_sq: f16 = f16(0.0);
293
- + let inv_size = f16(1.0) / f16(size);
294
- for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
295
- let idx = thread_idx * elements_per_thread + i;
296
- if (idx < size) {
297
- - let x = load_input(base_offset, idx);
298
- - shared_cache[idx] = x; // Cache for second pass (avoids reloading residual)
299
- - local_sum_sq = local_sum_sq + x * x;
300
- + let x = load_input(base_offset, idx) / row_scale;
301
- + local_sum_sq = local_sum_sq + (x * x) * inv_size;
302
- }
303
- }
304
-
305
- + // Store local sum for reduction
306
- shared_sum[thread_idx] = local_sum_sq;
307
- workgroupBarrier();
308
-
309
- + // Parallel reduction to compute total sum of squares
310
- for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
311
- if (thread_idx < stride) {
312
- shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
313
- @@ -284,117 +136,38 @@
314
- workgroupBarrier();
315
- }
316
-
317
- - let mean_sq = shared_sum[0] / f32(size);
318
- - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
319
- + let mean_sq = shared_sum[0];
320
- + let eps_scaled = f16(u.eps) / (row_scale * row_scale);
321
- + let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
322
-
323
- workgroupBarrier();
324
-
325
- - // Second pass: normalize cached input
326
- - for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
327
- - let idx = thread_idx * elements_per_thread + i;
328
- - if (idx < size) {
329
- - let x = shared_cache[idx];
330
- - var result = x * inv_rms * apply_weight(load_weight(idx));
331
- - if (should_add_residual()) {
332
- - result = result + residual[base_offset + idx];
333
- - }
334
- - write_prenorm(base_offset, idx, x);
335
- - output[base_offset + idx] = apply_output_scale(result);
336
- - }
337
- - }
338
- -}
339
- -
340
- -// =============================================================================
341
- -// Subgroup-Accelerated Entry Points
342
- -// =============================================================================
343
- -// Use subgroupAdd() for 3-5x faster reductions vs tree reduction with barriers.
344
- -// Requires WebGPU subgroups feature.
345
- -
346
- -var<workgroup> sg_partial_sums: array<f32, MAX_SUBGROUPS>;
347
- -
348
- -// Subgroup-accelerated RMSNorm - one workgroup per token
349
- -// Uses subgroupAdd() to reduce 8 barriers → 1 barrier
350
- -@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
351
- -fn main_subgroup(
352
- - @builtin(local_invocation_id) local_id: vec3<u32>,
353
- - @builtin(workgroup_id) wg_id: vec3<u32>,
354
- - @builtin(subgroup_invocation_id) sg_lane: u32,
355
- - @builtin(subgroup_size) sg_size: u32,
356
- -) {
357
- - let token_idx = token_index(wg_id);
358
- - let thread_idx = local_id.x;
359
- - let size = u.size;
360
- -
361
- - if (token_idx >= u.num_tokens) {
362
- - return;
363
- - }
364
- -
365
- - let base_offset = token_idx * size;
366
- - let subgroup_id = thread_idx / sg_size;
367
- - let num_subgroups = (WORKGROUP_SIZE + sg_size - 1u) / sg_size;
368
- -
369
- - // Each thread computes partial sum of squares
370
- - var local_sum_sq: f32 = 0.0;
371
- - let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
372
- -
373
- - for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
374
- - let idx = thread_idx * elements_per_thread + i;
375
- - if (idx < size) {
376
- - let x = load_input(base_offset, idx);
377
- - local_sum_sq = local_sum_sq + x * x;
378
- - }
379
- - }
380
- -
381
- - // Phase 1: Subgroup reduction (no barrier needed!)
382
- - let sg_sum = subgroupAdd(local_sum_sq);
383
- -
384
- - // Phase 2: Store subgroup results to shared memory
385
- - if (sg_lane == 0u && subgroup_id < num_subgroups) {
386
- - sg_partial_sums[subgroup_id] = sg_sum;
387
- - }
388
- - workgroupBarrier(); // Single barrier instead of 8!
389
- -
390
- - // Phase 3: Final reduction - thread 0 sums all subgroup partials
391
- - // Then broadcast to all threads via shared memory
392
- - if (thread_idx == 0u) {
393
- - var sum: f32 = 0.0;
394
- - for (var s = 0u; s < num_subgroups; s++) {
395
- - sum += sg_partial_sums[s];
396
- - }
397
- - sg_partial_sums[0] = sum;
398
- - }
399
- - workgroupBarrier();
400
- - let total_sum = sg_partial_sums[0];
401
- -
402
- - // Compute RMS (all threads now have the same value)
403
- - let mean_sq = total_sum / f32(size);
404
- - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
405
- -
406
- // Apply normalization and weight
407
- for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
408
- let idx = thread_idx * elements_per_thread + i;
409
- if (idx < size) {
410
- let x = load_input(base_offset, idx);
411
- +
412
- var result = x * inv_rms * apply_weight(load_weight(idx));
413
-
414
- - if (should_add_residual()) {
415
- + // Add residual AFTER normalization
416
- + if (u.has_residual == 1u) {
417
- result = result + residual[base_offset + idx];
418
- }
419
-
420
- + // Write pre-norm sum for downstream residual reuse
421
- write_prenorm(base_offset, idx, x);
422
- - output[base_offset + idx] = apply_output_scale(result);
423
- +
424
- + output[base_offset + idx] = f16(f32(result) * u.output_scale);
425
- }
426
- }
427
- }
428
-
429
- -// Subgroup-accelerated RMSNorm for small hidden sizes (size <= 256)
430
- -// Each thread handles one element, uses subgroup reduction
431
- +// Optimized version for hidden size <= WORKGROUP_SIZE (single pass)
432
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
433
- -fn main_small_subgroup(
434
- +fn rmsnorm_small_f16(
435
- @builtin(local_invocation_id) local_id: vec3<u32>,
436
- - @builtin(workgroup_id) wg_id: vec3<u32>,
437
- - @builtin(subgroup_invocation_id) sg_lane: u32,
438
- - @builtin(subgroup_size) sg_size: u32,
439
- + @builtin(workgroup_id) wg_id: vec3<u32>
440
- ) {
441
- let token_idx = token_index(wg_id);
442
- let thread_idx = local_id.x;
443
- @@ -405,48 +178,49 @@
444
- }
445
-
446
- let base_offset = token_idx * size;
447
- - let subgroup_id = thread_idx / sg_size;
448
- - let num_subgroups = (WORKGROUP_SIZE + sg_size - 1u) / sg_size;
449
-
450
- - // Each thread handles one element
451
- - var x: f32 = 0.0;
452
- - var x_sq: f32 = 0.0;
453
- + // Each thread handles one element (for size <= 256)
454
- + var x: f16 = f16(0.0);
455
- if (thread_idx < size) {
456
- x = load_input(base_offset, thread_idx);
457
- - x_sq = x * x;
458
- }
459
-
460
- - // Phase 1: Subgroup reduction
461
- - let sg_sum = subgroupAdd(x_sq);
462
- + shared_sum[thread_idx] = abs(x);
463
- + workgroupBarrier();
464
-
465
- - // Phase 2: Store to shared memory
466
- - if (sg_lane == 0u && subgroup_id < num_subgroups) {
467
- - sg_partial_sums[subgroup_id] = sg_sum;
468
- + for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
469
- + if (thread_idx < stride) {
470
- + shared_sum[thread_idx] = max(shared_sum[thread_idx], shared_sum[thread_idx + stride]);
471
- + }
472
- + workgroupBarrier();
473
- }
474
- +
475
- + let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
476
- + let x_scaled = x / row_scale;
477
- + let inv_size = f16(1.0) / f16(size);
478
- + shared_sum[thread_idx] = (x_scaled * x_scaled) * inv_size;
479
- workgroupBarrier();
480
-
481
- - // Phase 3: Final reduction - thread 0 sums, then broadcast via shared memory
482
- - if (thread_idx == 0u) {
483
- - var sum: f32 = 0.0;
484
- - for (var s = 0u; s < num_subgroups; s++) {
485
- - sum += sg_partial_sums[s];
486
- + // Parallel reduction
487
- + for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
488
- + if (thread_idx < stride && thread_idx + stride < size) {
489
- + shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
490
- }
491
- - sg_partial_sums[0] = sum;
492
- + workgroupBarrier();
493
- }
494
- - workgroupBarrier();
495
- - let total_sum = sg_partial_sums[0];
496
-
497
- // Compute inverse RMS
498
- - let mean_sq = total_sum / f32(size);
499
- - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
500
- + let mean_sq = shared_sum[0];
501
- + let eps_scaled = f16(u.eps) / (row_scale * row_scale);
502
- + let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
503
-
504
- // Apply normalization
505
- if (thread_idx < size) {
506
- var result = x * inv_rms * apply_weight(load_weight(thread_idx));
507
- - if (should_add_residual()) {
508
- + if (u.has_residual == 1u) {
509
- result = result + residual[base_offset + thread_idx];
510
- }
511
- write_prenorm(base_offset, thread_idx, x);
512
- - output[base_offset + thread_idx] = apply_output_scale(result);
513
- + output[base_offset + thread_idx] = f16(f32(result) * u.output_scale);
514
- }
515
- }