doppler-gpu 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1589 -0
  4. package/package.json +44 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  16. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  17. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  18. package/src/config/kernel-path-loader.js +1 -0
  19. package/src/config/kernels/backward-registry.json +13 -2
  20. package/src/config/kernels/kernel-ref-digests.js +34 -21
  21. package/src/config/kernels/registry.json +1402 -2623
  22. package/src/config/merge.js +11 -0
  23. package/src/config/param-validator.js +25 -0
  24. package/src/config/runtime/default.json +3 -1
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  29. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  30. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  31. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  32. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  33. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  34. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  35. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  36. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  37. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  38. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  39. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  40. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  41. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  42. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  43. package/src/config/schema/doppler.schema.js +40 -5
  44. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  45. package/src/config/schema/execution-v1.schema.js +10 -0
  46. package/src/config/schema/index.d.ts +3 -1
  47. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  48. package/src/config/schema/training-metrics.schema.js +4 -0
  49. package/src/config/transforms/capability-transform-resolver.js +7 -1
  50. package/src/config/transforms/execution-graph-transforms.js +2 -2
  51. package/src/debug/config.d.ts +4 -0
  52. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  53. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  54. package/src/experimental/orchestration/index.d.ts +10 -1
  55. package/src/experimental/training/attention-backward.js +123 -55
  56. package/src/experimental/training/autograd.js +101 -5
  57. package/src/experimental/training/clip.js +6 -1
  58. package/src/experimental/training/datasets/text-pairs.js +80 -8
  59. package/src/experimental/training/distillation/student-fixture.js +470 -114
  60. package/src/experimental/training/lora-pipeline.js +202 -27
  61. package/src/experimental/training/lora.js +10 -4
  62. package/src/experimental/training/optimizer.js +9 -1
  63. package/src/experimental/training/runner.js +18 -4
  64. package/src/experimental/training/suite.js +1 -1
  65. package/src/experimental/training/workloads.js +11 -0
  66. package/src/formats/rdrr/classification.js +3 -0
  67. package/src/formats/rdrr/integrity.d.ts +0 -2
  68. package/src/formats/rdrr/integrity.js +0 -2
  69. package/src/formats/rdrr/types.d.ts +1 -1
  70. package/src/gpu/kernels/attention.d.ts +2 -0
  71. package/src/gpu/kernels/attention.js +75 -16
  72. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  73. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  74. package/src/gpu/kernels/backward/adam.js +6 -4
  75. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  76. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  77. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  78. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  79. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  80. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  81. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  82. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  83. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  84. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  85. package/src/gpu/kernels/backward/index.d.ts +3 -0
  86. package/src/gpu/kernels/backward/index.js +3 -0
  87. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  88. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  89. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  90. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  91. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  92. package/src/gpu/kernels/backward/utils.js +9 -2
  93. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  94. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  95. package/src/gpu/kernels/fused_ffn.js +540 -19
  96. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  97. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  98. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  99. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  100. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  101. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  102. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  103. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  104. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  105. package/src/gpu/kernels/index.d.ts +26 -0
  106. package/src/gpu/kernels/index.js +18 -0
  107. package/src/gpu/kernels/linear-attention-core.js +413 -15
  108. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  109. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  110. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  111. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  112. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  113. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  114. package/src/gpu/kernels/matmul-selection.js +10 -1
  115. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  116. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  117. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  118. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  119. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  120. package/src/gpu/kernels/silu.d.ts +1 -0
  121. package/src/gpu/kernels/silu.js +19 -4
  122. package/src/gpu/kernels/silu.wgsl +7 -1
  123. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  124. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  125. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  126. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  127. package/src/gpu/tensor.d.ts +1 -0
  128. package/src/index.d.ts +1 -1
  129. package/src/index.js +1 -1
  130. package/src/inference/browser-harness-suite-helpers.js +1 -1
  131. package/src/inference/browser-harness-text-helpers.js +251 -20
  132. package/src/inference/browser-harness.js +221 -3
  133. package/src/inference/kv-cache/base.js +42 -1
  134. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  135. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  136. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  137. package/src/inference/pipelines/text/attention/projections.js +174 -4
  138. package/src/inference/pipelines/text/attention/record.js +44 -21
  139. package/src/inference/pipelines/text/attention/run.js +18 -3
  140. package/src/inference/pipelines/text/config.d.ts +18 -0
  141. package/src/inference/pipelines/text/config.js +100 -0
  142. package/src/inference/pipelines/text/embed.js +1 -1
  143. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  144. package/src/inference/pipelines/text/execution-plan.js +37 -0
  145. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  146. package/src/inference/pipelines/text/execution-v1.js +130 -10
  147. package/src/inference/pipelines/text/ffn/dense.js +241 -21
  148. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  149. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  152. package/src/inference/pipelines/text/generator-steps.js +2 -2
  153. package/src/inference/pipelines/text/generator.d.ts +18 -1
  154. package/src/inference/pipelines/text/generator.js +326 -19
  155. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  156. package/src/inference/pipelines/text/init.d.ts +22 -3
  157. package/src/inference/pipelines/text/init.js +2 -2
  158. package/src/inference/pipelines/text/layer.js +87 -8
  159. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  160. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  161. package/src/inference/pipelines/text/linear-attention.js +173 -16
  162. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  163. package/src/inference/pipelines/text/logits/index.js +62 -1
  164. package/src/inference/pipelines/text/lora-apply.js +4 -4
  165. package/src/inference/pipelines/text/model-load.js +6 -0
  166. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  167. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  168. package/src/inference/pipelines/text/ops.js +22 -0
  169. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  170. package/src/inference/pipelines/text/probes.js +9 -4
  171. package/src/inference/pipelines/text/state.d.ts +4 -4
  172. package/src/inference/pipelines/text/state.js +4 -0
  173. package/src/inference/pipelines/text/types.d.ts +27 -2
  174. package/src/inference/pipelines/text.d.ts +21 -1
  175. package/src/inference/pipelines/text.js +40 -14
  176. package/src/inference/tokenizers/bundled.js +150 -55
  177. package/src/inference/tokenizers/types.d.ts +7 -4
  178. package/src/loader/doppler-loader.js +25 -2
  179. package/src/loader/index.d.ts +3 -1
  180. package/src/loader/index.js +2 -0
  181. package/src/loader/loader-types.d.ts +3 -1
  182. package/src/loader/manifest-config.d.ts +18 -0
  183. package/src/loader/manifest-config.js +11 -3
  184. package/src/loader/shard-cache.d.ts +2 -0
  185. package/src/loader/shard-cache.js +29 -1
  186. package/src/models/gemma4.js +8 -0
  187. package/src/rules/inference/capability-transforms.rules.json +145 -20
  188. package/src/rules/inference/ffn.rules.json +33 -0
  189. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  190. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  191. package/src/rules/kernels/matmul.rules.json +2 -0
  192. package/src/rules/kernels/silu.rules.json +1 -0
  193. package/src/rules/rule-registry.js +2 -0
  194. package/src/storage/artifact-storage-context.js +36 -0
  195. package/src/tooling/node-browser-command-runner.js +42 -7
  196. package/src/tooling/node-converter.js +20 -5
  197. package/src/tooling/opfs-cache.d.ts +21 -1
  198. package/src/tooling/opfs-cache.js +168 -32
  199. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  200. package/src/tooling-experimental-exports.shared.js +18 -0
  201. package/src/tooling-exports/storage.d.ts +1 -1
  202. package/src/tooling-exports/storage.js +1 -1
  203. package/src/types/gpu.d.ts +25 -0
  204. package/src/version.js +1 -1
  205. package/tests/kernels/browser/kernel-suite.js +1593 -0
  206. package/tests/kernels/browser/test-page.js +2629 -0
  207. package/tests/kernels/harness/benchmark.js +225 -0
  208. package/tests/kernels/harness/buffer-utils.js +88 -0
  209. package/tests/kernels/harness/tolerance.js +174 -0
  210. package/tests/kernels/reference/adam.js +15 -0
  211. package/tests/kernels/reference/attention.js +158 -0
  212. package/tests/kernels/reference/backward.js +96 -0
  213. package/tests/kernels/reference/bias-add.js +20 -0
  214. package/tests/kernels/reference/conv2d.js +58 -0
  215. package/tests/kernels/reference/cross-entropy.js +26 -0
  216. package/tests/kernels/reference/dequant.js +312 -0
  217. package/tests/kernels/reference/embed.js +15 -0
  218. package/tests/kernels/reference/gather.js +61 -0
  219. package/tests/kernels/reference/gelu.js +37 -0
  220. package/tests/kernels/reference/groupnorm.js +36 -0
  221. package/tests/kernels/reference/index.js +89 -0
  222. package/tests/kernels/reference/layernorm.js +22 -0
  223. package/tests/kernels/reference/matmul.js +57 -0
  224. package/tests/kernels/reference/modulate.js +24 -0
  225. package/tests/kernels/reference/moe-gather.js +83 -0
  226. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  227. package/tests/kernels/reference/residual.js +55 -0
  228. package/tests/kernels/reference/rmsnorm.js +52 -0
  229. package/tests/kernels/reference/rope.js +79 -0
  230. package/tests/kernels/reference/sample.js +97 -0
  231. package/tests/kernels/reference/scatter-add.js +58 -0
  232. package/tests/kernels/reference/silu.js +52 -0
  233. package/tests/kernels/reference/softmax.js +86 -0
  234. package/tests/kernels/reference/split-qkv.js +58 -0
  235. package/tests/kernels/reference/topk.js +88 -0
  236. package/tests/kernels/reference/upsample2d.js +37 -0
  237. package/BRANDING.md +0 -14
  238. package/SECURITY.md +0 -19
  239. package/assets/doppler-runtime-map.svg +0 -82
  240. package/src/bootstrap.js +0 -30
  241. package/src/cli/doppler-cli.d.ts +0 -46
  242. package/src/cli/doppler-quickstart.d.ts +0 -45
  243. package/src/cli/doppler-serve.d.ts +0 -210
  244. package/src/cli/runtime-profiles.d.ts +0 -40
  245. package/src/client/doppler-api.browser.d.ts +0 -1
  246. package/src/client/doppler-api.browser.js +0 -25
  247. package/src/client/doppler-registry.d.ts +0 -54
  248. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  249. package/src/client/runtime/source-runtime.d.ts +0 -26
  250. package/src/client/runtime/storage.d.ts +0 -5
  251. package/src/config/README.md +0 -61
  252. package/src/config/conversion/README.md +0 -62
  253. package/src/config/goal-completion-matrix.json +0 -552
  254. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  255. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  256. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  257. package/src/config/kernels/registry.d.ts +0 -51
  258. package/src/config/merge-contract-check.d.ts +0 -16
  259. package/src/config/merge-contract-check.js +0 -319
  260. package/src/config/merge-helpers.d.ts +0 -60
  261. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  262. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  263. package/src/config/param-categories.d.ts +0 -8
  264. package/src/config/param-validator.d.ts +0 -33
  265. package/src/config/platforms/loader.d.ts +0 -38
  266. package/src/config/quantization-contract-check.d.ts +0 -12
  267. package/src/config/quantization-contract-check.js +0 -91
  268. package/src/config/runtime/README.md +0 -28
  269. package/src/config/runtime-merge.d.ts +0 -5
  270. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  271. package/src/config/schema/command-surface-contract.schema.json +0 -81
  272. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  273. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  274. package/src/config/schema/platform.schema.js +0 -1
  275. package/src/config/schema/policy-schema-registry.json +0 -31
  276. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  277. package/src/config/support-tiers/subsystems.json +0 -502
  278. package/src/config/supported-operations.d.ts +0 -14
  279. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  280. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  281. package/src/converter/conversion-plan.d.ts +0 -56
  282. package/src/converter/index.d.ts +0 -98
  283. package/src/converter/index.js +0 -59
  284. package/src/converter/parsers/diffusion.d.ts +0 -50
  285. package/src/converter/parsers/gguf.d.ts +0 -31
  286. package/src/converter/parsers/index.d.ts +0 -21
  287. package/src/converter/parsers/index.js +0 -12
  288. package/src/converter/parsers/transformer.d.ts +0 -31
  289. package/src/converter/quantization-info.d.ts +0 -38
  290. package/src/converter/rope-config.d.ts +0 -16
  291. package/src/converter/tokenizer-utils.d.ts +0 -12
  292. package/src/debug/reference/README.md +0 -114
  293. package/src/debug/reference/hf_attn_debug.py +0 -114
  294. package/src/debug/reference/hf_embed_check.py +0 -89
  295. package/src/debug/reference/hf_layer_out.py +0 -100
  296. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  297. package/src/debug/reference/hf_rope_check.py +0 -116
  298. package/src/debug/reference/hf_weights.py +0 -75
  299. package/src/errors/doppler-error.d.ts +0 -20
  300. package/src/experimental/bridge/extension/background.d.ts +0 -14
  301. package/src/experimental/bridge/extension/background.js +0 -168
  302. package/src/experimental/bridge/extension/manifest.json +0 -34
  303. package/src/experimental/bridge/extension-client.d.ts +0 -114
  304. package/src/experimental/bridge/index.d.ts +0 -69
  305. package/src/experimental/bridge/node-bridge.js +0 -24
  306. package/src/experimental/bridge/protocol.d.ts +0 -96
  307. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  308. package/src/experimental/browser/gguf-importer.js +0 -532
  309. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  310. package/src/experimental/browser/quantization.d.ts +0 -74
  311. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  312. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  313. package/src/experimental/browser/shard-io-browser.js +0 -89
  314. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  315. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  316. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  317. package/src/experimental/distribution/p2p-transport.js +0 -75
  318. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  319. package/src/experimental/hotswap/manifest.d.ts +0 -42
  320. package/src/experimental/hotswap/model-swap.js +0 -42
  321. package/src/experimental/hotswap/runtime.d.ts +0 -31
  322. package/src/experimental/training/README.md +0 -164
  323. package/src/experimental/training/artifacts.d.ts +0 -160
  324. package/src/experimental/training/attention-backward.d.ts +0 -30
  325. package/src/experimental/training/attention-forward.d.ts +0 -22
  326. package/src/experimental/training/attention-forward.js +0 -82
  327. package/src/experimental/training/autograd.d.ts +0 -51
  328. package/src/experimental/training/backward-kernels.js +0 -39
  329. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  330. package/src/experimental/training/checkpoint.d.ts +0 -36
  331. package/src/experimental/training/clip.d.ts +0 -9
  332. package/src/experimental/training/dataloader.d.ts +0 -8
  333. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  334. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  335. package/src/experimental/training/datasets/index.d.ts +0 -24
  336. package/src/experimental/training/datasets/index.js +0 -13
  337. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  338. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  339. package/src/experimental/training/datasets/reploid.js +0 -36
  340. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  341. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  342. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  343. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  344. package/src/experimental/training/datasets/token-batch.js +0 -52
  345. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  346. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  347. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  348. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  349. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  350. package/src/experimental/training/distillation/eval.d.ts +0 -34
  351. package/src/experimental/training/distillation/index.d.ts +0 -29
  352. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  353. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  354. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  355. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  356. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  357. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  358. package/src/experimental/training/export.d.ts +0 -41
  359. package/src/experimental/training/index.d.ts +0 -74
  360. package/src/experimental/training/index.js +0 -63
  361. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  362. package/src/experimental/training/lora.d.ts +0 -19
  363. package/src/experimental/training/loss-scaling.d.ts +0 -21
  364. package/src/experimental/training/loss.d.ts +0 -10
  365. package/src/experimental/training/objectives/base.d.ts +0 -58
  366. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  367. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  368. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  369. package/src/experimental/training/objectives/index.d.ts +0 -12
  370. package/src/experimental/training/objectives/index.js +0 -6
  371. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  372. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  373. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  374. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  375. package/src/experimental/training/operator-command.d.ts +0 -5
  376. package/src/experimental/training/operator-eval.d.ts +0 -48
  377. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  378. package/src/experimental/training/optimizer.d.ts +0 -22
  379. package/src/experimental/training/runner.d.ts +0 -249
  380. package/src/experimental/training/suite.d.ts +0 -299
  381. package/src/experimental/training/tensor-factory.d.ts +0 -9
  382. package/src/experimental/training/trainer.d.ts +0 -89
  383. package/src/experimental/training/ul_dataset.d.ts +0 -47
  384. package/src/experimental/training/ul_schedule.d.ts +0 -6
  385. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  386. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  387. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  388. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  389. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  390. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  391. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  392. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  393. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  394. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  395. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  396. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  397. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  398. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  399. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  400. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  401. package/src/experimental/training/workload-packs/registry.json +0 -278
  402. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  403. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  404. package/src/experimental/training/workloads.d.ts +0 -201
  405. package/src/formats/gguf/types.d.ts +0 -74
  406. package/src/formats/index.d.ts +0 -75
  407. package/src/formats/index.js +0 -19
  408. package/src/formats/litert/types.d.ts +0 -74
  409. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  410. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  411. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  412. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  413. package/src/formats/safetensors/types.d.ts +0 -67
  414. package/src/formats/tflite/types.d.ts +0 -61
  415. package/src/formats/tokenizer/index.d.ts +0 -5
  416. package/src/formats/tokenizer/index.js +0 -3
  417. package/src/formats/tokenizer/types.d.ts +0 -9
  418. package/src/gpu/kernel-runtime.d.ts +0 -13
  419. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  420. package/src/gpu/kernel-selection-log.d.ts +0 -4
  421. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  422. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  423. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  424. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  425. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  426. package/src/gpu/kernel-tuner/types.js +0 -4
  427. package/src/gpu/kernel-tuner.d.ts +0 -33
  428. package/src/gpu/kernels/README.md +0 -166
  429. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  430. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  431. package/src/gpu/kernels/check-stop.d.ts +0 -31
  432. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  433. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  434. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  435. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  436. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  437. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  438. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  439. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  440. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  441. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  442. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  443. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  444. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  445. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  446. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  447. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  448. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  449. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  450. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  451. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  452. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  453. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  454. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  455. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  456. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  457. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  458. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  459. package/src/gpu/kernels/constants.d.ts +0 -129
  460. package/src/gpu/kernels/dispatch.d.ts +0 -72
  461. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  462. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  463. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  464. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  465. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  466. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  467. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  468. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  469. package/src/gpu/kernels/types.js +0 -4
  470. package/src/gpu/perf-guards.d.ts +0 -13
  471. package/src/gpu/profiler.d.ts +0 -93
  472. package/src/gpu/readback-utils.d.ts +0 -16
  473. package/src/gpu/readback-utils.js +0 -49
  474. package/src/gpu/submit-probe.d.ts +0 -12
  475. package/src/index-browser.d.ts +0 -2
  476. package/src/index-browser.js +0 -2
  477. package/src/inference/README.md +0 -593
  478. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  479. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  480. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  481. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  482. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  483. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  484. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  485. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  486. package/src/inference/network-evolution.js +0 -79
  487. package/src/inference/pipelines/context.d.ts +0 -21
  488. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  489. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  490. package/src/inference/pipelines/diffusion/index.js +0 -3
  491. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  492. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  493. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  494. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  495. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  496. package/src/inference/pipelines/diffusion/types.js +0 -1
  497. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  498. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  499. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  500. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  501. package/src/inference/pipelines/dream/pipeline.js +0 -8
  502. package/src/inference/pipelines/energy/index.d.ts +0 -1
  503. package/src/inference/pipelines/energy/index.js +0 -1
  504. package/src/inference/pipelines/energy/types.js +0 -1
  505. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  506. package/src/inference/pipelines/energy-head/index.js +0 -6
  507. package/src/inference/pipelines/factory.d.ts +0 -10
  508. package/src/inference/pipelines/index.d.ts +0 -22
  509. package/src/inference/pipelines/index.js +0 -19
  510. package/src/inference/pipelines/registry.d.ts +0 -15
  511. package/src/inference/pipelines/rng.d.ts +0 -2
  512. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  513. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  514. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  515. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  516. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  517. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  518. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  519. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  520. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  521. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  522. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  523. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  524. package/src/inference/pipelines/text/attention.d.ts +0 -23
  525. package/src/inference/pipelines/text/attention.js +0 -12
  526. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  527. package/src/inference/pipelines/text/buffer-types.js +0 -4
  528. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  529. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  530. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  531. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  532. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  533. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  534. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  535. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  536. package/src/inference/pipelines/text/embed.d.ts +0 -69
  537. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  538. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  539. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  540. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  541. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  542. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  543. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  544. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  545. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  546. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  547. package/src/inference/pipelines/text/ffn.js +0 -18
  548. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  549. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  550. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  551. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  552. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  553. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  554. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  555. package/src/inference/pipelines/text/index.d.ts +0 -5
  556. package/src/inference/pipelines/text/index.js +0 -6
  557. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  558. package/src/inference/pipelines/text/layer.d.ts +0 -118
  559. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  560. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  561. package/src/inference/pipelines/text/logits/types.js +0 -4
  562. package/src/inference/pipelines/text/logits.d.ts +0 -27
  563. package/src/inference/pipelines/text/logits.js +0 -16
  564. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  565. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  566. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  567. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  568. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  569. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  570. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  571. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  572. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  573. package/src/inference/pipelines/text/ops.d.ts +0 -222
  574. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  575. package/src/inference/pipelines/text/probes.d.ts +0 -28
  576. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  577. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  578. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  579. package/src/inference/pipelines/text/types.js +0 -4
  580. package/src/inference/tokenizers/base.d.ts +0 -44
  581. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  582. package/src/inference/tokenizers/bpe.d.ts +0 -27
  583. package/src/inference/tokenizers/bundled.d.ts +0 -67
  584. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  585. package/src/inference/tokenizers/types.js +0 -7
  586. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  587. package/src/loader/per-layer-input-loader.d.ts +0 -30
  588. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  589. package/src/loader/tensors/source-transform.d.ts +0 -65
  590. package/src/memory/address-table.d.ts +0 -96
  591. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  592. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  593. package/src/rules/rule-registry.d.ts +0 -77
  594. package/src/storage/artifact-storage-context.d.ts +0 -63
  595. package/src/storage/backends/idb-store.d.ts +0 -52
  596. package/src/storage/backends/memory-store.d.ts +0 -36
  597. package/src/storage/backends/opfs-store.d.ts +0 -41
  598. package/src/storage/blake3.d.ts +0 -17
  599. package/src/storage/download-types.d.ts +0 -157
  600. package/src/storage/downloader.d.ts +0 -103
  601. package/src/storage/index.d.ts +0 -274
  602. package/src/storage/index.js +0 -201
  603. package/src/storage/preflight.d.ts +0 -144
  604. package/src/storage/preflight.js +0 -360
  605. package/src/storage/quickstart-downloader.d.ts +0 -157
  606. package/src/storage/quickstart-downloader.js +0 -289
  607. package/src/storage/source-artifact-store.d.ts +0 -59
  608. package/src/sw.js +0 -184
  609. package/src/tooling/command-api-constants.d.ts +0 -9
  610. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  611. package/src/tooling/command-api-helpers.d.ts +0 -23
  612. package/src/tooling/command-runner-shared.d.ts +0 -89
  613. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  614. package/src/tooling/conversion-config-materializer.js +0 -96
  615. package/src/tooling/diagnose-runner.d.ts +0 -28
  616. package/src/tooling/hf-registry-utils.d.ts +0 -55
  617. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  618. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  619. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  620. package/src/tooling/kernel-path-builder/index.js +0 -1361
  621. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  622. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  623. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  624. package/src/tooling/lean-execution-contract-runner.js +0 -158
  625. package/src/tooling/lean-execution-contract.d.ts +0 -16
  626. package/src/tooling/lean-execution-contract.js +0 -81
  627. package/src/tooling/litert-package-runtime.d.ts +0 -46
  628. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  629. package/src/tooling/node-convert-worker.d.ts +0 -1
  630. package/src/tooling/node-converter.d.ts +0 -1
  631. package/src/tooling/node-file-fetch.d.ts +0 -1
  632. package/src/tooling/node-source-runtime.d.ts +0 -24
  633. package/src/tooling/node-webgpu.d.ts +0 -17
  634. package/src/tooling/precision-replay-math.js +0 -174
  635. package/src/tooling/program-bundle-host.d.ts +0 -16
  636. package/src/tooling/program-bundle-host.js +0 -20
  637. package/src/tooling/runtime-input-composition.d.ts +0 -37
  638. package/src/tooling/source-package-profiles.d.ts +0 -50
  639. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  640. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  641. package/src/tooling/source-runtime-materializer.js +0 -89
  642. package/src/tooling-exports.browser.d.ts +0 -7
  643. package/src/types/chrome.d.ts +0 -36
  644. package/src/types/chrome.js +0 -1
  645. package/src/types/index.d.ts +0 -1
  646. package/src/types/index.js +0 -1
  647. package/src/utils/array-buffer.d.ts +0 -4
  648. package/src/utils/canonical-hash.d.ts +0 -7
  649. package/src/utils/clone-json.d.ts +0 -1
  650. package/src/utils/hf-resolve-url.d.ts +0 -16
  651. package/src/utils/index.d.ts +0 -10
  652. package/src/utils/index.js +0 -10
  653. package/src/utils/load-json.d.ts +0 -5
  654. package/src/utils/plain-object.d.ts +0 -1
  655. package/src/utils/sha256.d.ts +0 -5
  656. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -1,4 +1,5 @@
1
1
  import { getDevice } from '../../../gpu/device.js';
2
+ import { Q4K_BLOCK_BYTES, q4kBlockCount } from '../../../config/schema/index.js';
2
3
  import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
3
4
  import { getKernelPathMatmulVariant } from '../../../config/kernel-path-loader.js';
4
5
  import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
@@ -24,7 +25,7 @@ function kernelPathVariantRequiresQ4KWeights(variant) {
24
25
  return shaderFile.startsWith('fused_matmul_q4');
25
26
  }
26
27
 
27
- function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
28
+ function qkvProjectionRequiresQ4KWeights(layerIdx, kernelPath) {
28
29
  if (!kernelPath) {
29
30
  return false;
30
31
  }
@@ -36,7 +37,82 @@ function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
36
37
  return kernelPathVariantRequiresQ4KWeights(decodeVariant);
37
38
  }
38
39
 
39
- export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
40
+ function normalizeProjectionDtype(weight) {
41
+ const dtype = typeof weight?.dtype === 'string'
42
+ ? weight.dtype.toLowerCase()
43
+ : null;
44
+ if (dtype === 'bf16') return 'f16';
45
+ if (dtype === 'f16' || dtype === 'f32' || dtype === 'q4k') return dtype;
46
+ return null;
47
+ }
48
+
49
+ function resolveProjectionStorageFormat(weight, expectedRows, hiddenSize) {
50
+ const dtype = normalizeProjectionDtype(weight);
51
+ if (dtype === 'q4k') {
52
+ const layout = String(weight?.layout ?? 'row').toLowerCase();
53
+ if (layout !== 'row') {
54
+ return null;
55
+ }
56
+ const bytesPerRow = q4kBlockCount(hiddenSize) * Q4K_BLOCK_BYTES;
57
+ const byteLength = expectedRows * bytesPerRow;
58
+ if (weight?.buffer?.size < byteLength) {
59
+ return null;
60
+ }
61
+ return {
62
+ dtype,
63
+ layout,
64
+ byteLength,
65
+ bytesPerElement: null,
66
+ bytesPerRow,
67
+ };
68
+ }
69
+
70
+ if (dtype === 'f16' || dtype === 'f32') {
71
+ const bytesPerElement = dtype === 'f16' ? 2 : 4;
72
+ const byteLength = expectedRows * hiddenSize * bytesPerElement;
73
+ if (weight?.buffer?.size < byteLength) {
74
+ return null;
75
+ }
76
+ return {
77
+ dtype,
78
+ layout: weight?.layout ?? null,
79
+ byteLength,
80
+ bytesPerElement,
81
+ bytesPerRow: hiddenSize * bytesPerElement,
82
+ };
83
+ }
84
+
85
+ const minF16Bytes = expectedRows * hiddenSize * 2;
86
+ const minF32Bytes = expectedRows * hiddenSize * 4;
87
+ if (weight?.buffer?.size >= minF32Bytes) {
88
+ return {
89
+ dtype: 'f32',
90
+ layout: weight?.layout ?? null,
91
+ byteLength: minF32Bytes,
92
+ bytesPerElement: 4,
93
+ bytesPerRow: hiddenSize * 4,
94
+ };
95
+ }
96
+ if (weight?.buffer?.size >= minF16Bytes) {
97
+ return {
98
+ dtype: 'f16',
99
+ layout: weight?.layout ?? null,
100
+ byteLength: minF16Bytes,
101
+ bytesPerElement: 2,
102
+ bytesPerRow: hiddenSize * 2,
103
+ };
104
+ }
105
+ return null;
106
+ }
107
+
108
+ function formatsMatch(a, b) {
109
+ return a?.dtype === b?.dtype
110
+ && a?.layout === b?.layout
111
+ && a?.bytesPerElement === b?.bytesPerElement
112
+ && a?.bytesPerRow === b?.bytesPerRow;
113
+ }
114
+
115
+ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, options = {}) {
40
116
  const device = getDevice();
41
117
  if (!device) {
42
118
  log.debug('QKV Fusion', 'No GPU device, skipping fusion');
@@ -52,12 +128,14 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
52
128
  );
53
129
  return;
54
130
  }
55
-
56
131
  const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
57
132
  const qSize = numHeads * headDim;
133
+ const hasAttentionOutputGate = modelConfig?.attentionOutputGate === true;
134
+ const qProjSize = hasAttentionOutputGate ? qSize * 2 : qSize;
58
135
  const kSize = numKVHeads * headDim;
59
136
  const vSize = numKVHeads * headDim;
60
137
  const qkvSize = qSize + kSize + vSize;
138
+ const allowQ4K = options.allowQ4K === true;
61
139
 
62
140
  const resolveWeight = (value) => {
63
141
  if (isWeightBuffer(value)) {
@@ -81,19 +159,6 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
81
159
 
82
160
  log.debug('QKV Fusion', `Fusing Q/K/V weights for ${numLayers} layers (${qSize}+${kSize}+${vSize}=${qkvSize})`);
83
161
 
84
- const resolveBytesPerElement = (weight, expectedElements) => {
85
- const dtype = typeof weight?.dtype === 'string'
86
- ? weight.dtype.toLowerCase()
87
- : null;
88
- if (dtype === 'f16' || dtype === 'bf16') return 2;
89
- if (dtype === 'f32') return 4;
90
- const minF16Bytes = expectedElements * 2;
91
- const minF32Bytes = expectedElements * 4;
92
- if (weight?.buffer?.size >= minF32Bytes) return 4;
93
- if (weight?.buffer?.size >= minF16Bytes) return 2;
94
- return 0;
95
- };
96
-
97
162
  let fusedCount = 0;
98
163
  for (let l = 0; l < numLayers; l++) {
99
164
  const weights = layerWeights.get(`layer_${l}`);
@@ -101,13 +166,7 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
101
166
 
102
167
  // Skip if already fused or if weights are not GPUBuffers
103
168
  if (weights.qkvProj) continue;
104
- if (shouldSkipQKVFusionForLayer(l, kernelPath)) {
105
- log.debug(
106
- 'QKV Fusion',
107
- `Layer ${l}: skipped because active kernel path requires Q4K weights for qkv_proj`
108
- );
109
- continue;
110
- }
169
+ const requiresQ4K = qkvProjectionRequiresQ4KWeights(l, kernelPath);
111
170
  const qProj = resolveWeight(weights.qProj);
112
171
  const kProj = resolveWeight(weights.kProj);
113
172
  const vProj = resolveWeight(weights.vProj);
@@ -115,72 +174,97 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
115
174
  continue;
116
175
  }
117
176
 
118
- const qExpectedElements = qSize * hiddenSize;
119
- const kExpectedElements = kSize * hiddenSize;
120
- const vExpectedElements = vSize * hiddenSize;
121
- const bytesPerElement = resolveBytesPerElement(qProj, qExpectedElements);
122
- const kBytesPerElement = resolveBytesPerElement(kProj, kExpectedElements);
123
- const vBytesPerElement = resolveBytesPerElement(vProj, vExpectedElements);
177
+ const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
178
+ const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
179
+ const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
124
180
 
125
181
  // Pool allocation can round GPUBuffer.size up, so infer logical dtype first and
126
182
  // only use buffer size as a minimum-size inference.
127
- if ((bytesPerElement !== 2 && bytesPerElement !== 4)
128
- || kBytesPerElement !== bytesPerElement
129
- || vBytesPerElement !== bytesPerElement) {
183
+ if (!qFormat || !formatsMatch(qFormat, kFormat) || !formatsMatch(qFormat, vFormat)) {
130
184
  log.debug(
131
185
  'QKV Fusion',
132
- `Layer ${l}: inconsistent projection dtypes (q=${bytesPerElement}, k=${kBytesPerElement}, v=${vBytesPerElement}), skipping`
186
+ `Layer ${l}: inconsistent projection storage formats, skipping`
133
187
  );
134
188
  continue;
135
189
  }
136
190
 
137
- const normalizedDtype = typeof qProj.dtype === 'string'
138
- ? qProj.dtype.toLowerCase()
139
- : null;
140
- const dtype = normalizedDtype === 'bf16'
141
- ? 'f16'
142
- : (
143
- normalizedDtype === 'f16' || normalizedDtype === 'f32'
144
- ? normalizedDtype
145
- : selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement })
146
- );
191
+ if (requiresQ4K && qFormat.dtype !== 'q4k') {
192
+ log.debug('QKV Fusion', `Layer ${l}: qkv_proj requires Q4K weights, skipping ${qFormat.dtype} pack`);
193
+ continue;
194
+ }
195
+ if (qFormat.dtype === 'q4k' && !allowQ4K) {
196
+ log.debug('QKV Fusion', `Layer ${l}: Q4K QKV fusion requires explicit runtime opt-in`);
197
+ continue;
198
+ }
199
+
200
+ const dtype = qFormat.dtype ?? selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', {
201
+ bytesPerElement: qFormat.bytesPerElement,
202
+ });
147
203
  const layout = qProj.layout ?? kProj.layout ?? vProj.layout ?? 'row';
148
204
  let fusedShape = [qkvSize, hiddenSize];
149
- if (Array.isArray(qProj.shape) && qProj.shape.length === 2) {
150
- if (qProj.shape[0] === qSize && qProj.shape[1] === hiddenSize) {
205
+ if (qFormat.dtype !== 'q4k' && Array.isArray(qProj.shape) && qProj.shape.length === 2) {
206
+ if (qProj.shape[0] === qProjSize && qProj.shape[1] === hiddenSize) {
151
207
  fusedShape = [qkvSize, hiddenSize];
152
- } else if (qProj.shape[1] === qSize && qProj.shape[0] === hiddenSize) {
208
+ } else if (qProj.shape[1] === qProjSize && qProj.shape[0] === hiddenSize) {
153
209
  fusedShape = [hiddenSize, qkvSize];
154
210
  }
155
211
  }
156
212
 
157
- // Create fused QKV buffer: [qkvSize, hiddenSize] row-major
158
- // Each row is concatenated: [q_row, k_row, v_row]
213
+ const qBytes = qSize * qFormat.bytesPerRow;
214
+ const qGateBytes = hasAttentionOutputGate ? qBytes : 0;
215
+
216
+ // Create fused QKV buffer: [qkvSize, hiddenSize] row-major.
217
+ // attentionOutputGate models keep Q rows in qkvProj and gate rows in qGateProj.
159
218
  const qkvBuffer = device.createBuffer({
160
219
  label: `layer_${l}_qkv_proj`,
161
- size: qkvSize * hiddenSize * bytesPerElement,
220
+ size: qBytes + kFormat.byteLength + vFormat.byteLength,
162
221
  usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
163
222
  });
223
+ const qGateBuffer = hasAttentionOutputGate
224
+ ? device.createBuffer({
225
+ label: `layer_${l}_q_gate_proj`,
226
+ size: qGateBytes,
227
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
228
+ })
229
+ : null;
164
230
 
165
- // Copy Q, K, V weights into fused buffer
166
- // Q: [qSize, hiddenSize] -> offset 0
167
- // K: [kSize, hiddenSize] -> offset qSize * hiddenSize * bytesPerElement
168
- // V: [vSize, hiddenSize] -> offset (qSize + kSize) * hiddenSize * bytesPerElement
169
231
  const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
170
- encoder.copyBufferToBuffer(
171
- qProj.buffer, 0,
172
- qkvBuffer, 0,
173
- qSize * hiddenSize * bytesPerElement
174
- );
232
+ if (hasAttentionOutputGate) {
233
+ for (let head = 0; head < numHeads; head++) {
234
+ const srcHeadOffset = head * headDim * 2 * qFormat.bytesPerRow;
235
+ const dstHeadOffset = head * headDim * qFormat.bytesPerRow;
236
+ const headBytes = headDim * qFormat.bytesPerRow;
237
+ encoder.copyBufferToBuffer(
238
+ qProj.buffer,
239
+ srcHeadOffset,
240
+ qkvBuffer,
241
+ dstHeadOffset,
242
+ headBytes
243
+ );
244
+ encoder.copyBufferToBuffer(
245
+ qProj.buffer,
246
+ srcHeadOffset + headBytes,
247
+ qGateBuffer,
248
+ dstHeadOffset,
249
+ headBytes
250
+ );
251
+ }
252
+ } else {
253
+ encoder.copyBufferToBuffer(
254
+ qProj.buffer, 0,
255
+ qkvBuffer, 0,
256
+ qBytes
257
+ );
258
+ }
175
259
  encoder.copyBufferToBuffer(
176
260
  kProj.buffer, 0,
177
- qkvBuffer, qSize * hiddenSize * bytesPerElement,
178
- kSize * hiddenSize * bytesPerElement
261
+ qkvBuffer, qBytes,
262
+ kFormat.byteLength
179
263
  );
180
264
  encoder.copyBufferToBuffer(
181
265
  vProj.buffer, 0,
182
- qkvBuffer, (qSize + kSize) * hiddenSize * bytesPerElement,
183
- vSize * hiddenSize * bytesPerElement
266
+ qkvBuffer, qBytes + kFormat.byteLength,
267
+ vFormat.byteLength
184
268
  );
185
269
  device.queue.submit([encoder.finish()]);
186
270
 
@@ -194,6 +278,15 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
194
278
  );
195
279
  weights.qkvSizes = [qSize, kSize, vSize];
196
280
  weights.qkvDtype = dtype;
281
+ if (qGateBuffer) {
282
+ weights.qGateProj = createWeightBuffer(
283
+ qGateBuffer,
284
+ dtype,
285
+ layout,
286
+ [qSize, hiddenSize],
287
+ `layer_${l}_q_gate_proj`
288
+ );
289
+ }
197
290
  fusedCount++;
198
291
  }
199
292
 
@@ -44,6 +44,7 @@ import type {
44
44
  import type { LoaderDebugConfigSchema } from '../../../config/schema/debug.schema.js';
45
45
  import type { KernelPathSource } from '../../../config/kernel-path-loader.js';
46
46
  import type { ExecutionV1PerLayerInputsSessionSchema } from '../../../config/schema/execution-v1.schema.js';
47
+ import type { DopplerLoader } from '../../../loader/doppler-loader.js';
47
48
 
48
49
  export interface PipelineStorageContext {
49
50
  preflight?: () => Promise<void>;
@@ -67,6 +68,15 @@ export interface PipelineStorageContext {
67
68
  close?: () => Promise<void>;
68
69
  }
69
70
 
71
+ export interface EmulationContext {
72
+ config: {
73
+ topology: { gpuCount: number };
74
+ timingMode: string;
75
+ };
76
+ getStats(): unknown;
77
+ destroy(): void | Promise<void>;
78
+ }
79
+
70
80
  export function createNodeFileShardStorageContext(
71
81
  baseUrl: string | null | undefined,
72
82
  manifest: Manifest
@@ -90,6 +100,10 @@ export interface PipelineContexts {
90
100
  runtimeConfig?: Partial<RuntimeConfigSchema> | RuntimeConfigSchema;
91
101
  /** Progress callback for weight loading */
92
102
  onProgress?: (progress: { percent: number; message?: string }) => void;
103
+ /** Optional caller-owned loader instance for isolated model residency */
104
+ loader?: DopplerLoader;
105
+ /** True when the pipeline should release the injected loader on unload */
106
+ ownsLoader?: boolean;
93
107
  }
94
108
 
95
109
  /**
@@ -219,6 +233,7 @@ export function initTokenizer(
219
233
  * Weight loading result.
220
234
  */
221
235
  export interface WeightLoadResult {
236
+ loader: DopplerLoader;
222
237
  layerWeights: Map<string, LayerWeights>;
223
238
  embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
224
239
  lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
@@ -250,6 +265,7 @@ export interface LoadWeightsOptions {
250
265
  keepF32Weights?: boolean;
251
266
  loaderDebug?: LoaderDebugConfigSchema | null;
252
267
  perLayerInputSession?: ExecutionV1PerLayerInputsSessionSchema | null;
268
+ loader?: DopplerLoader;
253
269
  }
254
270
 
255
271
  /**
@@ -334,7 +350,10 @@ export function initSpeculativeDecoder(
334
350
  export function fuseQKVWeights(
335
351
  layerWeights: Map<string, LayerWeights>,
336
352
  modelConfig: ParsedModelConfig,
337
- kernelPath?: KernelPathSchema | null
353
+ kernelPath?: KernelPathSchema | null,
354
+ options?: {
355
+ allowQ4K?: boolean;
356
+ }
338
357
  ): void;
339
358
 
340
359
  /**
@@ -349,7 +368,7 @@ export function fuseQKVWeights(
349
368
  */
350
369
  export function initEmulation(
351
370
  runtimeConfig: RuntimeConfigSchema
352
- ): Promise<import('/proto/simulator/index.js').EmulationContext | null>;
371
+ ): Promise<EmulationContext | null>;
353
372
 
354
373
  /**
355
374
  * Destroy emulation context and clean up resources.
@@ -357,5 +376,5 @@ export function initEmulation(
357
376
  * @param emulation - Emulation context to destroy
358
377
  */
359
378
  export function destroyEmulation(
360
- emulation: import('/proto/simulator/index.js').EmulationContext | null
379
+ emulation: EmulationContext | null
361
380
  ): Promise<void>;
@@ -879,7 +879,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
879
879
  throw new Error('runtime.loading.shardCache.verifyHashes is required.');
880
880
  }
881
881
 
882
- const dopplerLoader = getDopplerLoader(loadingConfig);
882
+ const dopplerLoader = options.loader ?? getDopplerLoader(loadingConfig);
883
883
  const keepF32Weights = options.keepF32Weights === true;
884
884
  dopplerLoader.setQ4KConfig(
885
885
  resolveQ4KConfig(
@@ -997,7 +997,6 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
997
997
  layerWeights.set(`layer_${l}`, weights);
998
998
  }
999
999
  }
1000
-
1001
1000
  // Collect per-layer router weights for MoE
1002
1001
 
1003
1002
  const layerRouterWeights = new Map();
@@ -1017,6 +1016,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1017
1016
  }
1018
1017
 
1019
1018
  return {
1019
+ loader: dopplerLoader,
1020
1020
  layerWeights,
1021
1021
  embeddings: dopplerLoader.embeddings,
1022
1022
  lmHead: dopplerLoader.lmHead,
@@ -2,7 +2,7 @@
2
2
 
3
3
  import { log, trace } from '../../../debug/index.js';
4
4
  import { getRuntimeConfig } from '../../../config/runtime.js';
5
- import { getDevice } from '../../../gpu/device.js';
5
+ import { getDevice, getKernelCapabilities } from '../../../gpu/device.js';
6
6
  import { releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
7
7
  import { allowReadback } from '../../../gpu/perf-guards.js';
8
8
  import { createTensor } from '../../../gpu/tensor.js';
@@ -144,7 +144,7 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
144
144
  return false;
145
145
  }
146
146
  const nextLayerType = config.layerTypes?.[nextLayerIdx];
147
- if (isConvLayerType(nextLayerType) || isLinearLayerType(nextLayerType)) {
147
+ if (isConvLayerType(nextLayerType)) {
148
148
  return false;
149
149
  }
150
150
  if (!sandwichNorm.useSandwichNorm || !sandwichNorm.hasPostFeedforwardNorm) {
@@ -164,6 +164,56 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
164
164
  return true;
165
165
  }
166
166
 
167
+ function shouldUseStandardPostFfnNextInputRMSNormPairFusion({
168
+ context,
169
+ config,
170
+ sandwichNorm,
171
+ layerIdx,
172
+ nextLayerWeights,
173
+ numTokens,
174
+ hiddenSize,
175
+ activationDtype,
176
+ layerScalar,
177
+ residualBranchScale,
178
+ }) {
179
+ const mergedSession = getRuntimeConfig()?.inference?.session;
180
+ if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
181
+ return false;
182
+ }
183
+ if (numTokens !== 1 || context.phase !== 'decode' || context.diffusionGemmaDecoder === true) {
184
+ return false;
185
+ }
186
+ if (context.debug === true || context.debugProbes?.length || context.operatorDiagnostics?.enabled === true) {
187
+ return false;
188
+ }
189
+ if (!context.decodeBuffers || context.pipelinePlan || hasPerLayerInputBlock(config)) {
190
+ return false;
191
+ }
192
+ if (sandwichNorm.useSandwichNorm || layerScalar !== 1 || residualBranchScale !== 1 || activationDtype !== 'f32') {
193
+ return false;
194
+ }
195
+ if (getKernelCapabilities()?.hasSubgroups !== true) {
196
+ return false;
197
+ }
198
+ const nextLayerIdx = layerIdx + 1;
199
+ if (nextLayerIdx >= config.numLayers) {
200
+ return false;
201
+ }
202
+ const nextLayerType = config.layerTypes?.[nextLayerIdx];
203
+ if (isConvLayerType(nextLayerType)) {
204
+ return false;
205
+ }
206
+ if (!nextLayerWeights?.inputNorm) {
207
+ return false;
208
+ }
209
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0 || hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
210
+ throw new Error(
211
+ `usePostFfnNextInputRMSNormPairFusion requires hiddenSize in 1..${RMSNORM_PAIR_CACHE_LIMIT}; got ${String(hiddenSize)}.`
212
+ );
213
+ }
214
+ return true;
215
+ }
216
+
167
217
 
168
218
  export function isMoELayer(layerIdx, config) {
169
219
  if (!config.useMoE) return false;
@@ -395,6 +445,7 @@ export async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens,
395
445
  let activatedTensor = null;
396
446
  let projectedTensor = null;
397
447
  let normalizedTensor = null;
448
+ let residualAddTensor = null;
398
449
  let outputTensor = null;
399
450
 
400
451
  try {
@@ -501,11 +552,19 @@ export async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens,
501
552
  releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
502
553
  projectedTensor = null;
503
554
 
504
- outputTensor = await doResidualAdd(normalizedTensor, residualTensor, size, recorder, {
555
+ residualAddTensor = normalizedTensor;
556
+ if (residualAddTensor.dtype !== residualTensor.dtype) {
557
+ residualAddTensor = await doCast(residualAddTensor, residualTensor.dtype, recorder);
558
+ }
559
+ outputTensor = await doResidualAdd(residualAddTensor, residualTensor, size, recorder, {
505
560
  label: `L${layerIdx}.per_layer_input_residual`,
506
561
  layerIdx,
507
562
  executionPolicies: context.executionPolicies ?? null,
508
563
  });
564
+ if (residualAddTensor.buffer !== normalizedTensor.buffer) {
565
+ releaseOrTrack(recorder, residualAddTensor.buffer, decodeBuffers);
566
+ }
567
+ residualAddTensor = null;
509
568
  releaseOrTrack(recorder, normalizedTensor.buffer, decodeBuffers);
510
569
  normalizedTensor = null;
511
570
 
@@ -523,6 +582,9 @@ export async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens,
523
582
  return outputTensor;
524
583
  } catch (error) {
525
584
  if (outputTensor?.buffer) releaseOrTrack(recorder, outputTensor.buffer, decodeBuffers);
585
+ if (residualAddTensor?.buffer && residualAddTensor.buffer !== normalizedTensor?.buffer) {
586
+ releaseOrTrack(recorder, residualAddTensor.buffer, decodeBuffers);
587
+ }
526
588
  if (normalizedTensor?.buffer) releaseOrTrack(recorder, normalizedTensor.buffer, decodeBuffers);
527
589
  if (projectedTensor?.buffer) releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
528
590
  if (activatedTensor?.buffer) releaseOrTrack(recorder, activatedTensor.buffer, decodeBuffers);
@@ -753,6 +815,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
753
815
  linearRuntime: context.linearAttentionRuntime ?? null,
754
816
  getWeightBuffer: (weight, label) => getWeightBuffer(weight, label),
755
817
  getNormWeightBuffer: (weight, label) => getNormWeightBuffer(weight, label, weightConfig, debugFlags),
818
+ precomputedInputNorm: takePrecomputedInputNorm(context, layerIdx, recorder),
756
819
  debugProbes: context.debugProbes,
757
820
  operatorDiagnostics: context.operatorDiagnostics,
758
821
  recorder: recorder ?? null,
@@ -826,6 +889,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
826
889
  operatorDiagnostics: context.operatorDiagnostics,
827
890
  linearRuntime: context.linearAttentionRuntime ?? null,
828
891
  executionPolicies: context.executionPolicies ?? null,
892
+ skipKVCacheWrites: context.skipKVCacheWrites === true,
829
893
  };
830
894
 
831
895
  const attnResult = await doAttention(
@@ -1058,6 +1122,18 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
1058
1122
  activationDtype,
1059
1123
  layerScalar,
1060
1124
  });
1125
+ const useStandardPostFfnNextInputNormPair = shouldUseStandardPostFfnNextInputRMSNormPairFusion({
1126
+ context,
1127
+ config,
1128
+ sandwichNorm,
1129
+ layerIdx,
1130
+ nextLayerWeights,
1131
+ numTokens,
1132
+ hiddenSize,
1133
+ activationDtype,
1134
+ layerScalar,
1135
+ residualBranchScale,
1136
+ });
1061
1137
  let layerScalarFused = false;
1062
1138
  if (sandwichNorm.useSandwichNorm) {
1063
1139
  context.__postFfnNextInputNorm = usePostFfnNextInputNormPair
@@ -1087,11 +1163,14 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
1087
1163
  numTokens,
1088
1164
  size,
1089
1165
  context,
1090
- layerWeights,
1091
- fusedResidualForFFN,
1092
- requestFfnLayerScalarFusion ? layerScalar : 1,
1093
- residualBranchScale
1094
- );
1166
+ layerWeights,
1167
+ fusedResidualForFFN,
1168
+ requestFfnLayerScalarFusion ? layerScalar : 1,
1169
+ residualBranchScale,
1170
+ useStandardPostFfnNextInputNormPair
1171
+ ? { layerIdx: nextLayerIdx, weight: nextLayerWeights.inputNorm }
1172
+ : null
1173
+ );
1095
1174
  layerScalarFused = context.__layerScalarFusedFired === true;
1096
1175
  context.__layerScalarFusedFired = false;
1097
1176
  }