doppler-gpu 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1589 -0
  4. package/package.json +44 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  16. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  17. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  18. package/src/config/kernel-path-loader.js +1 -0
  19. package/src/config/kernels/backward-registry.json +13 -2
  20. package/src/config/kernels/kernel-ref-digests.js +34 -21
  21. package/src/config/kernels/registry.json +1402 -2623
  22. package/src/config/merge.js +11 -0
  23. package/src/config/param-validator.js +25 -0
  24. package/src/config/runtime/default.json +3 -1
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  29. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  30. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  31. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  32. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  33. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  34. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  35. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  36. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  37. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  38. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  39. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  40. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  41. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  42. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  43. package/src/config/schema/doppler.schema.js +40 -5
  44. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  45. package/src/config/schema/execution-v1.schema.js +10 -0
  46. package/src/config/schema/index.d.ts +3 -1
  47. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  48. package/src/config/schema/training-metrics.schema.js +4 -0
  49. package/src/config/transforms/capability-transform-resolver.js +7 -1
  50. package/src/config/transforms/execution-graph-transforms.js +2 -2
  51. package/src/debug/config.d.ts +4 -0
  52. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  53. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  54. package/src/experimental/orchestration/index.d.ts +10 -1
  55. package/src/experimental/training/attention-backward.js +123 -55
  56. package/src/experimental/training/autograd.js +101 -5
  57. package/src/experimental/training/clip.js +6 -1
  58. package/src/experimental/training/datasets/text-pairs.js +80 -8
  59. package/src/experimental/training/distillation/student-fixture.js +470 -114
  60. package/src/experimental/training/lora-pipeline.js +202 -27
  61. package/src/experimental/training/lora.js +10 -4
  62. package/src/experimental/training/optimizer.js +9 -1
  63. package/src/experimental/training/runner.js +18 -4
  64. package/src/experimental/training/suite.js +1 -1
  65. package/src/experimental/training/workloads.js +11 -0
  66. package/src/formats/rdrr/classification.js +3 -0
  67. package/src/formats/rdrr/integrity.d.ts +0 -2
  68. package/src/formats/rdrr/integrity.js +0 -2
  69. package/src/formats/rdrr/types.d.ts +1 -1
  70. package/src/gpu/kernels/attention.d.ts +2 -0
  71. package/src/gpu/kernels/attention.js +75 -16
  72. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  73. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  74. package/src/gpu/kernels/backward/adam.js +6 -4
  75. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  76. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  77. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  78. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  79. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  80. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  81. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  82. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  83. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  84. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  85. package/src/gpu/kernels/backward/index.d.ts +3 -0
  86. package/src/gpu/kernels/backward/index.js +3 -0
  87. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  88. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  89. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  90. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  91. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  92. package/src/gpu/kernels/backward/utils.js +9 -2
  93. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  94. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  95. package/src/gpu/kernels/fused_ffn.js +540 -19
  96. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  97. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  98. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  99. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  100. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  101. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  102. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  103. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  104. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  105. package/src/gpu/kernels/index.d.ts +26 -0
  106. package/src/gpu/kernels/index.js +18 -0
  107. package/src/gpu/kernels/linear-attention-core.js +413 -15
  108. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  109. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  110. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  111. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  112. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  113. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  114. package/src/gpu/kernels/matmul-selection.js +10 -1
  115. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  116. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  117. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  118. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  119. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  120. package/src/gpu/kernels/silu.d.ts +1 -0
  121. package/src/gpu/kernels/silu.js +19 -4
  122. package/src/gpu/kernels/silu.wgsl +7 -1
  123. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  124. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  125. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  126. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  127. package/src/gpu/tensor.d.ts +1 -0
  128. package/src/index.d.ts +1 -1
  129. package/src/index.js +1 -1
  130. package/src/inference/browser-harness-suite-helpers.js +1 -1
  131. package/src/inference/browser-harness-text-helpers.js +251 -20
  132. package/src/inference/browser-harness.js +221 -3
  133. package/src/inference/kv-cache/base.js +42 -1
  134. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  135. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  136. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  137. package/src/inference/pipelines/text/attention/projections.js +174 -4
  138. package/src/inference/pipelines/text/attention/record.js +44 -21
  139. package/src/inference/pipelines/text/attention/run.js +18 -3
  140. package/src/inference/pipelines/text/config.d.ts +18 -0
  141. package/src/inference/pipelines/text/config.js +100 -0
  142. package/src/inference/pipelines/text/embed.js +1 -1
  143. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  144. package/src/inference/pipelines/text/execution-plan.js +37 -0
  145. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  146. package/src/inference/pipelines/text/execution-v1.js +130 -10
  147. package/src/inference/pipelines/text/ffn/dense.js +241 -21
  148. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  149. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  152. package/src/inference/pipelines/text/generator-steps.js +2 -2
  153. package/src/inference/pipelines/text/generator.d.ts +18 -1
  154. package/src/inference/pipelines/text/generator.js +326 -19
  155. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  156. package/src/inference/pipelines/text/init.d.ts +22 -3
  157. package/src/inference/pipelines/text/init.js +2 -2
  158. package/src/inference/pipelines/text/layer.js +87 -8
  159. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  160. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  161. package/src/inference/pipelines/text/linear-attention.js +173 -16
  162. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  163. package/src/inference/pipelines/text/logits/index.js +62 -1
  164. package/src/inference/pipelines/text/lora-apply.js +4 -4
  165. package/src/inference/pipelines/text/model-load.js +6 -0
  166. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  167. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  168. package/src/inference/pipelines/text/ops.js +22 -0
  169. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  170. package/src/inference/pipelines/text/probes.js +9 -4
  171. package/src/inference/pipelines/text/state.d.ts +4 -4
  172. package/src/inference/pipelines/text/state.js +4 -0
  173. package/src/inference/pipelines/text/types.d.ts +27 -2
  174. package/src/inference/pipelines/text.d.ts +21 -1
  175. package/src/inference/pipelines/text.js +40 -14
  176. package/src/inference/tokenizers/bundled.js +150 -55
  177. package/src/inference/tokenizers/types.d.ts +7 -4
  178. package/src/loader/doppler-loader.js +25 -2
  179. package/src/loader/index.d.ts +3 -1
  180. package/src/loader/index.js +2 -0
  181. package/src/loader/loader-types.d.ts +3 -1
  182. package/src/loader/manifest-config.d.ts +18 -0
  183. package/src/loader/manifest-config.js +11 -3
  184. package/src/loader/shard-cache.d.ts +2 -0
  185. package/src/loader/shard-cache.js +29 -1
  186. package/src/models/gemma4.js +8 -0
  187. package/src/rules/inference/capability-transforms.rules.json +145 -20
  188. package/src/rules/inference/ffn.rules.json +33 -0
  189. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  190. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  191. package/src/rules/kernels/matmul.rules.json +2 -0
  192. package/src/rules/kernels/silu.rules.json +1 -0
  193. package/src/rules/rule-registry.js +2 -0
  194. package/src/storage/artifact-storage-context.js +36 -0
  195. package/src/tooling/node-browser-command-runner.js +42 -7
  196. package/src/tooling/node-converter.js +20 -5
  197. package/src/tooling/opfs-cache.d.ts +21 -1
  198. package/src/tooling/opfs-cache.js +168 -32
  199. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  200. package/src/tooling-experimental-exports.shared.js +18 -0
  201. package/src/tooling-exports/storage.d.ts +1 -1
  202. package/src/tooling-exports/storage.js +1 -1
  203. package/src/types/gpu.d.ts +25 -0
  204. package/src/version.js +1 -1
  205. package/tests/kernels/browser/kernel-suite.js +1593 -0
  206. package/tests/kernels/browser/test-page.js +2629 -0
  207. package/tests/kernels/harness/benchmark.js +225 -0
  208. package/tests/kernels/harness/buffer-utils.js +88 -0
  209. package/tests/kernels/harness/tolerance.js +174 -0
  210. package/tests/kernels/reference/adam.js +15 -0
  211. package/tests/kernels/reference/attention.js +158 -0
  212. package/tests/kernels/reference/backward.js +96 -0
  213. package/tests/kernels/reference/bias-add.js +20 -0
  214. package/tests/kernels/reference/conv2d.js +58 -0
  215. package/tests/kernels/reference/cross-entropy.js +26 -0
  216. package/tests/kernels/reference/dequant.js +312 -0
  217. package/tests/kernels/reference/embed.js +15 -0
  218. package/tests/kernels/reference/gather.js +61 -0
  219. package/tests/kernels/reference/gelu.js +37 -0
  220. package/tests/kernels/reference/groupnorm.js +36 -0
  221. package/tests/kernels/reference/index.js +89 -0
  222. package/tests/kernels/reference/layernorm.js +22 -0
  223. package/tests/kernels/reference/matmul.js +57 -0
  224. package/tests/kernels/reference/modulate.js +24 -0
  225. package/tests/kernels/reference/moe-gather.js +83 -0
  226. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  227. package/tests/kernels/reference/residual.js +55 -0
  228. package/tests/kernels/reference/rmsnorm.js +52 -0
  229. package/tests/kernels/reference/rope.js +79 -0
  230. package/tests/kernels/reference/sample.js +97 -0
  231. package/tests/kernels/reference/scatter-add.js +58 -0
  232. package/tests/kernels/reference/silu.js +52 -0
  233. package/tests/kernels/reference/softmax.js +86 -0
  234. package/tests/kernels/reference/split-qkv.js +58 -0
  235. package/tests/kernels/reference/topk.js +88 -0
  236. package/tests/kernels/reference/upsample2d.js +37 -0
  237. package/BRANDING.md +0 -14
  238. package/SECURITY.md +0 -19
  239. package/assets/doppler-runtime-map.svg +0 -82
  240. package/src/bootstrap.js +0 -30
  241. package/src/cli/doppler-cli.d.ts +0 -46
  242. package/src/cli/doppler-quickstart.d.ts +0 -45
  243. package/src/cli/doppler-serve.d.ts +0 -210
  244. package/src/cli/runtime-profiles.d.ts +0 -40
  245. package/src/client/doppler-api.browser.d.ts +0 -1
  246. package/src/client/doppler-api.browser.js +0 -25
  247. package/src/client/doppler-registry.d.ts +0 -54
  248. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  249. package/src/client/runtime/source-runtime.d.ts +0 -26
  250. package/src/client/runtime/storage.d.ts +0 -5
  251. package/src/config/README.md +0 -61
  252. package/src/config/conversion/README.md +0 -62
  253. package/src/config/goal-completion-matrix.json +0 -552
  254. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  255. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  256. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  257. package/src/config/kernels/registry.d.ts +0 -51
  258. package/src/config/merge-contract-check.d.ts +0 -16
  259. package/src/config/merge-contract-check.js +0 -319
  260. package/src/config/merge-helpers.d.ts +0 -60
  261. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  262. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  263. package/src/config/param-categories.d.ts +0 -8
  264. package/src/config/param-validator.d.ts +0 -33
  265. package/src/config/platforms/loader.d.ts +0 -38
  266. package/src/config/quantization-contract-check.d.ts +0 -12
  267. package/src/config/quantization-contract-check.js +0 -91
  268. package/src/config/runtime/README.md +0 -28
  269. package/src/config/runtime-merge.d.ts +0 -5
  270. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  271. package/src/config/schema/command-surface-contract.schema.json +0 -81
  272. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  273. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  274. package/src/config/schema/platform.schema.js +0 -1
  275. package/src/config/schema/policy-schema-registry.json +0 -31
  276. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  277. package/src/config/support-tiers/subsystems.json +0 -502
  278. package/src/config/supported-operations.d.ts +0 -14
  279. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  280. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  281. package/src/converter/conversion-plan.d.ts +0 -56
  282. package/src/converter/index.d.ts +0 -98
  283. package/src/converter/index.js +0 -59
  284. package/src/converter/parsers/diffusion.d.ts +0 -50
  285. package/src/converter/parsers/gguf.d.ts +0 -31
  286. package/src/converter/parsers/index.d.ts +0 -21
  287. package/src/converter/parsers/index.js +0 -12
  288. package/src/converter/parsers/transformer.d.ts +0 -31
  289. package/src/converter/quantization-info.d.ts +0 -38
  290. package/src/converter/rope-config.d.ts +0 -16
  291. package/src/converter/tokenizer-utils.d.ts +0 -12
  292. package/src/debug/reference/README.md +0 -114
  293. package/src/debug/reference/hf_attn_debug.py +0 -114
  294. package/src/debug/reference/hf_embed_check.py +0 -89
  295. package/src/debug/reference/hf_layer_out.py +0 -100
  296. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  297. package/src/debug/reference/hf_rope_check.py +0 -116
  298. package/src/debug/reference/hf_weights.py +0 -75
  299. package/src/errors/doppler-error.d.ts +0 -20
  300. package/src/experimental/bridge/extension/background.d.ts +0 -14
  301. package/src/experimental/bridge/extension/background.js +0 -168
  302. package/src/experimental/bridge/extension/manifest.json +0 -34
  303. package/src/experimental/bridge/extension-client.d.ts +0 -114
  304. package/src/experimental/bridge/index.d.ts +0 -69
  305. package/src/experimental/bridge/node-bridge.js +0 -24
  306. package/src/experimental/bridge/protocol.d.ts +0 -96
  307. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  308. package/src/experimental/browser/gguf-importer.js +0 -532
  309. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  310. package/src/experimental/browser/quantization.d.ts +0 -74
  311. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  312. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  313. package/src/experimental/browser/shard-io-browser.js +0 -89
  314. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  315. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  316. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  317. package/src/experimental/distribution/p2p-transport.js +0 -75
  318. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  319. package/src/experimental/hotswap/manifest.d.ts +0 -42
  320. package/src/experimental/hotswap/model-swap.js +0 -42
  321. package/src/experimental/hotswap/runtime.d.ts +0 -31
  322. package/src/experimental/training/README.md +0 -164
  323. package/src/experimental/training/artifacts.d.ts +0 -160
  324. package/src/experimental/training/attention-backward.d.ts +0 -30
  325. package/src/experimental/training/attention-forward.d.ts +0 -22
  326. package/src/experimental/training/attention-forward.js +0 -82
  327. package/src/experimental/training/autograd.d.ts +0 -51
  328. package/src/experimental/training/backward-kernels.js +0 -39
  329. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  330. package/src/experimental/training/checkpoint.d.ts +0 -36
  331. package/src/experimental/training/clip.d.ts +0 -9
  332. package/src/experimental/training/dataloader.d.ts +0 -8
  333. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  334. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  335. package/src/experimental/training/datasets/index.d.ts +0 -24
  336. package/src/experimental/training/datasets/index.js +0 -13
  337. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  338. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  339. package/src/experimental/training/datasets/reploid.js +0 -36
  340. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  341. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  342. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  343. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  344. package/src/experimental/training/datasets/token-batch.js +0 -52
  345. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  346. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  347. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  348. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  349. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  350. package/src/experimental/training/distillation/eval.d.ts +0 -34
  351. package/src/experimental/training/distillation/index.d.ts +0 -29
  352. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  353. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  354. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  355. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  356. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  357. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  358. package/src/experimental/training/export.d.ts +0 -41
  359. package/src/experimental/training/index.d.ts +0 -74
  360. package/src/experimental/training/index.js +0 -63
  361. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  362. package/src/experimental/training/lora.d.ts +0 -19
  363. package/src/experimental/training/loss-scaling.d.ts +0 -21
  364. package/src/experimental/training/loss.d.ts +0 -10
  365. package/src/experimental/training/objectives/base.d.ts +0 -58
  366. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  367. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  368. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  369. package/src/experimental/training/objectives/index.d.ts +0 -12
  370. package/src/experimental/training/objectives/index.js +0 -6
  371. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  372. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  373. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  374. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  375. package/src/experimental/training/operator-command.d.ts +0 -5
  376. package/src/experimental/training/operator-eval.d.ts +0 -48
  377. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  378. package/src/experimental/training/optimizer.d.ts +0 -22
  379. package/src/experimental/training/runner.d.ts +0 -249
  380. package/src/experimental/training/suite.d.ts +0 -299
  381. package/src/experimental/training/tensor-factory.d.ts +0 -9
  382. package/src/experimental/training/trainer.d.ts +0 -89
  383. package/src/experimental/training/ul_dataset.d.ts +0 -47
  384. package/src/experimental/training/ul_schedule.d.ts +0 -6
  385. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  386. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  387. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  388. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  389. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  390. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  391. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  392. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  393. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  394. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  395. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  396. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  397. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  398. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  399. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  400. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  401. package/src/experimental/training/workload-packs/registry.json +0 -278
  402. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  403. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  404. package/src/experimental/training/workloads.d.ts +0 -201
  405. package/src/formats/gguf/types.d.ts +0 -74
  406. package/src/formats/index.d.ts +0 -75
  407. package/src/formats/index.js +0 -19
  408. package/src/formats/litert/types.d.ts +0 -74
  409. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  410. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  411. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  412. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  413. package/src/formats/safetensors/types.d.ts +0 -67
  414. package/src/formats/tflite/types.d.ts +0 -61
  415. package/src/formats/tokenizer/index.d.ts +0 -5
  416. package/src/formats/tokenizer/index.js +0 -3
  417. package/src/formats/tokenizer/types.d.ts +0 -9
  418. package/src/gpu/kernel-runtime.d.ts +0 -13
  419. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  420. package/src/gpu/kernel-selection-log.d.ts +0 -4
  421. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  422. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  423. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  424. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  425. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  426. package/src/gpu/kernel-tuner/types.js +0 -4
  427. package/src/gpu/kernel-tuner.d.ts +0 -33
  428. package/src/gpu/kernels/README.md +0 -166
  429. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  430. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  431. package/src/gpu/kernels/check-stop.d.ts +0 -31
  432. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  433. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  434. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  435. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  436. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  437. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  438. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  439. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  440. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  441. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  442. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  443. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  444. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  445. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  446. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  447. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  448. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  449. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  450. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  451. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  452. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  453. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  454. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  455. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  456. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  457. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  458. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  459. package/src/gpu/kernels/constants.d.ts +0 -129
  460. package/src/gpu/kernels/dispatch.d.ts +0 -72
  461. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  462. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  463. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  464. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  465. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  466. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  467. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  468. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  469. package/src/gpu/kernels/types.js +0 -4
  470. package/src/gpu/perf-guards.d.ts +0 -13
  471. package/src/gpu/profiler.d.ts +0 -93
  472. package/src/gpu/readback-utils.d.ts +0 -16
  473. package/src/gpu/readback-utils.js +0 -49
  474. package/src/gpu/submit-probe.d.ts +0 -12
  475. package/src/index-browser.d.ts +0 -2
  476. package/src/index-browser.js +0 -2
  477. package/src/inference/README.md +0 -593
  478. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  479. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  480. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  481. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  482. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  483. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  484. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  485. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  486. package/src/inference/network-evolution.js +0 -79
  487. package/src/inference/pipelines/context.d.ts +0 -21
  488. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  489. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  490. package/src/inference/pipelines/diffusion/index.js +0 -3
  491. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  492. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  493. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  494. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  495. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  496. package/src/inference/pipelines/diffusion/types.js +0 -1
  497. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  498. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  499. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  500. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  501. package/src/inference/pipelines/dream/pipeline.js +0 -8
  502. package/src/inference/pipelines/energy/index.d.ts +0 -1
  503. package/src/inference/pipelines/energy/index.js +0 -1
  504. package/src/inference/pipelines/energy/types.js +0 -1
  505. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  506. package/src/inference/pipelines/energy-head/index.js +0 -6
  507. package/src/inference/pipelines/factory.d.ts +0 -10
  508. package/src/inference/pipelines/index.d.ts +0 -22
  509. package/src/inference/pipelines/index.js +0 -19
  510. package/src/inference/pipelines/registry.d.ts +0 -15
  511. package/src/inference/pipelines/rng.d.ts +0 -2
  512. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  513. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  514. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  515. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  516. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  517. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  518. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  519. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  520. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  521. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  522. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  523. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  524. package/src/inference/pipelines/text/attention.d.ts +0 -23
  525. package/src/inference/pipelines/text/attention.js +0 -12
  526. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  527. package/src/inference/pipelines/text/buffer-types.js +0 -4
  528. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  529. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  530. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  531. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  532. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  533. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  534. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  535. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  536. package/src/inference/pipelines/text/embed.d.ts +0 -69
  537. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  538. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  539. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  540. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  541. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  542. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  543. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  544. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  545. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  546. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  547. package/src/inference/pipelines/text/ffn.js +0 -18
  548. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  549. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  550. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  551. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  552. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  553. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  554. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  555. package/src/inference/pipelines/text/index.d.ts +0 -5
  556. package/src/inference/pipelines/text/index.js +0 -6
  557. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  558. package/src/inference/pipelines/text/layer.d.ts +0 -118
  559. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  560. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  561. package/src/inference/pipelines/text/logits/types.js +0 -4
  562. package/src/inference/pipelines/text/logits.d.ts +0 -27
  563. package/src/inference/pipelines/text/logits.js +0 -16
  564. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  565. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  566. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  567. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  568. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  569. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  570. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  571. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  572. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  573. package/src/inference/pipelines/text/ops.d.ts +0 -222
  574. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  575. package/src/inference/pipelines/text/probes.d.ts +0 -28
  576. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  577. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  578. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  579. package/src/inference/pipelines/text/types.js +0 -4
  580. package/src/inference/tokenizers/base.d.ts +0 -44
  581. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  582. package/src/inference/tokenizers/bpe.d.ts +0 -27
  583. package/src/inference/tokenizers/bundled.d.ts +0 -67
  584. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  585. package/src/inference/tokenizers/types.js +0 -7
  586. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  587. package/src/loader/per-layer-input-loader.d.ts +0 -30
  588. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  589. package/src/loader/tensors/source-transform.d.ts +0 -65
  590. package/src/memory/address-table.d.ts +0 -96
  591. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  592. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  593. package/src/rules/rule-registry.d.ts +0 -77
  594. package/src/storage/artifact-storage-context.d.ts +0 -63
  595. package/src/storage/backends/idb-store.d.ts +0 -52
  596. package/src/storage/backends/memory-store.d.ts +0 -36
  597. package/src/storage/backends/opfs-store.d.ts +0 -41
  598. package/src/storage/blake3.d.ts +0 -17
  599. package/src/storage/download-types.d.ts +0 -157
  600. package/src/storage/downloader.d.ts +0 -103
  601. package/src/storage/index.d.ts +0 -274
  602. package/src/storage/index.js +0 -201
  603. package/src/storage/preflight.d.ts +0 -144
  604. package/src/storage/preflight.js +0 -360
  605. package/src/storage/quickstart-downloader.d.ts +0 -157
  606. package/src/storage/quickstart-downloader.js +0 -289
  607. package/src/storage/source-artifact-store.d.ts +0 -59
  608. package/src/sw.js +0 -184
  609. package/src/tooling/command-api-constants.d.ts +0 -9
  610. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  611. package/src/tooling/command-api-helpers.d.ts +0 -23
  612. package/src/tooling/command-runner-shared.d.ts +0 -89
  613. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  614. package/src/tooling/conversion-config-materializer.js +0 -96
  615. package/src/tooling/diagnose-runner.d.ts +0 -28
  616. package/src/tooling/hf-registry-utils.d.ts +0 -55
  617. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  618. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  619. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  620. package/src/tooling/kernel-path-builder/index.js +0 -1361
  621. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  622. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  623. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  624. package/src/tooling/lean-execution-contract-runner.js +0 -158
  625. package/src/tooling/lean-execution-contract.d.ts +0 -16
  626. package/src/tooling/lean-execution-contract.js +0 -81
  627. package/src/tooling/litert-package-runtime.d.ts +0 -46
  628. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  629. package/src/tooling/node-convert-worker.d.ts +0 -1
  630. package/src/tooling/node-converter.d.ts +0 -1
  631. package/src/tooling/node-file-fetch.d.ts +0 -1
  632. package/src/tooling/node-source-runtime.d.ts +0 -24
  633. package/src/tooling/node-webgpu.d.ts +0 -17
  634. package/src/tooling/precision-replay-math.js +0 -174
  635. package/src/tooling/program-bundle-host.d.ts +0 -16
  636. package/src/tooling/program-bundle-host.js +0 -20
  637. package/src/tooling/runtime-input-composition.d.ts +0 -37
  638. package/src/tooling/source-package-profiles.d.ts +0 -50
  639. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  640. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  641. package/src/tooling/source-runtime-materializer.js +0 -89
  642. package/src/tooling-exports.browser.d.ts +0 -7
  643. package/src/types/chrome.d.ts +0 -36
  644. package/src/types/chrome.js +0 -1
  645. package/src/types/index.d.ts +0 -1
  646. package/src/types/index.js +0 -1
  647. package/src/utils/array-buffer.d.ts +0 -4
  648. package/src/utils/canonical-hash.d.ts +0 -7
  649. package/src/utils/clone-json.d.ts +0 -1
  650. package/src/utils/hf-resolve-url.d.ts +0 -16
  651. package/src/utils/index.d.ts +0 -10
  652. package/src/utils/index.js +0 -10
  653. package/src/utils/load-json.d.ts +0 -5
  654. package/src/utils/plain-object.d.ts +0 -1
  655. package/src/utils/sha256.d.ts +0 -5
  656. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -87,6 +87,66 @@ function resolvePipelineLoadTimings(pipeline) {
87
87
  }
88
88
  }
89
89
 
90
+ function getNestedSampleValue(sample, key) {
91
+ if (!sample || typeof sample !== 'object' || typeof key !== 'string' || key.length === 0) {
92
+ return null;
93
+ }
94
+ let current = sample;
95
+ for (const part of key.split('.')) {
96
+ if (!current || typeof current !== 'object') {
97
+ return null;
98
+ }
99
+ current = current[part];
100
+ }
101
+ return current;
102
+ }
103
+
104
+ function getNestedPhaseValue(sample, key) {
105
+ const value = getNestedSampleValue(sample, key);
106
+ return Number.isFinite(value) ? value : null;
107
+ }
108
+
109
+ function summarizeTimingPhaseSamples(samples, keys) {
110
+ const rows = Array.isArray(samples) ? samples.filter((sample) => sample && typeof sample === 'object') : [];
111
+ const summary = {};
112
+ for (const key of keys) {
113
+ const values = rows
114
+ .map((sample) => getNestedPhaseValue(sample, key))
115
+ .filter((value) => Number.isFinite(value));
116
+ summary[key] = computeSampleStats(values);
117
+ }
118
+ return summary;
119
+ }
120
+
121
+ function summarizePrefillRecordOps(samples) {
122
+ const rows = Array.isArray(samples) ? samples.filter((sample) => sample && typeof sample === 'object') : [];
123
+ const labelSums = {};
124
+ let labelSampleCount = 0;
125
+ for (const sample of rows) {
126
+ const labelCounts = normalizeDecodeRecordOpLabels(getNestedSampleValue(sample, 'prefillRecordOpLabels'));
127
+ if (!labelCounts) {
128
+ continue;
129
+ }
130
+ labelSampleCount += 1;
131
+ for (const [label, count] of Object.entries(labelCounts)) {
132
+ labelSums[label] = (labelSums[label] ?? 0) + count;
133
+ }
134
+ }
135
+ if (labelSampleCount === 0) {
136
+ return {};
137
+ }
138
+ const meanLabels = {};
139
+ for (const [label, count] of Object.entries(labelSums)) {
140
+ meanLabels[label] = count / labelSampleCount;
141
+ }
142
+ const prefillRecordOps = summarizeTimingPhaseSamples(rows, ['prefillRecordOps']).prefillRecordOps;
143
+ return {
144
+ prefillRecordUniqueOpLabels: Object.keys(meanLabels).length,
145
+ prefillRecordTopOps: buildDecodeRecordTopOps(meanLabels, prefillRecordOps?.mean),
146
+ prefillRecordTopOpGroups: buildDecodeRecordTopOpGroups(meanLabels, prefillRecordOps?.mean),
147
+ };
148
+ }
149
+
90
150
  async function loadTrainingSuiteModule() {
91
151
  if (!trainingSuiteModulePromise) {
92
152
  trainingSuiteModulePromise = import(TRAINING_SUITE_MODULE_PATH);
@@ -184,9 +244,13 @@ function resolveDecodeCadence(runtimeConfig, executionPlan = null) {
184
244
  const executionPlanCadence = resolveExecutionPlanCadence(executionPlan);
185
245
  const batchSize = executionPlanCadence?.batchSize ?? decodeLoop.batchSize ?? batching.batchSize ?? null;
186
246
  const readbackInterval = executionPlanCadence?.readbackInterval ?? decodeLoop.readbackInterval ?? batching.readbackInterval ?? null;
247
+ const maxBatchDecodeTokens = executionPlanCadence?.maxBatchDecodeTokens
248
+ ?? decodeLoop.maxBatchDecodeTokens
249
+ ?? null;
187
250
  return {
188
251
  batchSize,
189
252
  readbackInterval,
253
+ maxBatchDecodeTokens,
190
254
  stopCheckMode: executionPlanCadence?.stopCheckMode ?? decodeLoop.stopCheckMode ?? batching.stopCheckMode ?? null,
191
255
  readbackMode: executionPlanCadence?.readbackMode ?? decodeLoop.readbackMode ?? batching.readbackMode ?? null,
192
256
  disableCommandBatching: executionPlanCadence?.disableCommandBatching ?? decodeLoop.disableCommandBatching ?? null,
@@ -205,6 +269,7 @@ function resolveDecodeCadence(runtimeConfig, executionPlan = null) {
205
269
  decodeLoop: {
206
270
  batchSize: decodeLoop.batchSize ?? null,
207
271
  readbackInterval: decodeLoop.readbackInterval ?? null,
272
+ maxBatchDecodeTokens: decodeLoop.maxBatchDecodeTokens ?? null,
208
273
  stopCheckMode: decodeLoop.stopCheckMode ?? null,
209
274
  readbackMode: decodeLoop.readbackMode ?? null,
210
275
  ringTokens: decodeLoop.ringTokens ?? null,
@@ -217,6 +282,7 @@ function resolveDecodeCadence(runtimeConfig, executionPlan = null) {
217
282
  id: executionPlanCadence.id ?? null,
218
283
  batchSize: executionPlanCadence.batchSize ?? null,
219
284
  readbackInterval: executionPlanCadence.readbackInterval ?? null,
285
+ maxBatchDecodeTokens: executionPlanCadence.maxBatchDecodeTokens ?? null,
220
286
  stopCheckMode: executionPlanCadence.stopCheckMode ?? null,
221
287
  readbackMode: executionPlanCadence.readbackMode ?? null,
222
288
  disableCommandBatching: executionPlanCadence.disableCommandBatching ?? null,
@@ -1010,13 +1076,18 @@ async function runBenchSuite(options = {}) {
1010
1076
  const documentCounts = [];
1011
1077
  const topDocumentScores = [];
1012
1078
  const topDocumentProbabilities = [];
1079
+ const rerankPhases = [];
1080
+ const rerankDocumentPhases = [];
1013
1081
  let invalidRuns = 0;
1014
1082
  let nonFiniteScores = 0;
1015
1083
  let lastRun = null;
1016
1084
 
1017
1085
  for (let i = 0; i < warmupRuns + timedRuns; i++) {
1018
1086
  harness.pipeline.reset?.();
1019
- const run = await runRerank(harness.pipeline, runtimeConfig, benchRun);
1087
+ const run = await runRerank(harness.pipeline, runtimeConfig, {
1088
+ ...benchRun,
1089
+ benchmark: true,
1090
+ });
1020
1091
  if (i >= warmupRuns) {
1021
1092
  timedDurations.push(run.durationMs);
1022
1093
  const finiteScores = run.scores.filter((entry) => (
@@ -1029,6 +1100,16 @@ async function runBenchSuite(options = {}) {
1029
1100
  const hasRanking = Array.isArray(run.ranking) && run.ranking.length === run.documentCount;
1030
1101
  if (finiteScores.length === run.scores.length && hasRanking && run.documentCount > 0) {
1031
1102
  durations.push(run.durationMs);
1103
+ if (run.phase && typeof run.phase === 'object') {
1104
+ rerankPhases.push(run.phase);
1105
+ if (Array.isArray(run.phase.documents)) {
1106
+ for (const documentPhase of run.phase.documents) {
1107
+ if (documentPhase?.phase && typeof documentPhase.phase === 'object') {
1108
+ rerankDocumentPhases.push(documentPhase.phase);
1109
+ }
1110
+ }
1111
+ }
1112
+ }
1032
1113
  documentCounts.push(run.documentCount);
1033
1114
  if (Number.isFinite(run.topDocument?.score)) {
1034
1115
  topDocumentScores.push(run.topDocument.score);
@@ -1051,6 +1132,9 @@ async function runBenchSuite(options = {}) {
1051
1132
  const topProbabilityStats = computeSampleStats(topDocumentProbabilities);
1052
1133
  const avgMs = rerankMsStats.mean;
1053
1134
  const semanticPassed = semantic.passed;
1135
+ const rerankPrefixPhases = rerankPhases
1136
+ .map((phase) => phase?.prefix)
1137
+ .filter((phase) => phase && typeof phase === 'object');
1054
1138
 
1055
1139
  results = [
1056
1140
  {
@@ -1135,6 +1219,53 @@ async function runBenchSuite(options = {}) {
1135
1219
  timedRerankMs: timedRerankMsStats,
1136
1220
  rerankMs: rerankMsStats,
1137
1221
  },
1222
+ phase: summarizeTimingPhaseSamples(rerankPhases, [
1223
+ 'totalMs',
1224
+ 'prefixMs',
1225
+ 'prefixTokens',
1226
+ 'documentCount',
1227
+ 'documentTotalMs',
1228
+ 'maxDocumentMs',
1229
+ 'avgDocumentMs',
1230
+ 'prefix.totalMs',
1231
+ 'prefix.prefillMs',
1232
+ 'prefix.prefillRecordMs',
1233
+ 'prefix.prefillRecordOps',
1234
+ 'prefix.prefillRecordPasses',
1235
+ 'prefix.prefillSubmitWaitMs',
1236
+ ]),
1237
+ prefixPhase: {
1238
+ ...summarizeTimingPhaseSamples(rerankPrefixPhases, [
1239
+ 'totalMs',
1240
+ 'prefillMs',
1241
+ 'prefillRecordMs',
1242
+ 'prefillRecordOps',
1243
+ 'prefillRecordPasses',
1244
+ 'prefillSubmitWaitMs',
1245
+ 'gpuPrefillMs',
1246
+ 'tokens',
1247
+ ]),
1248
+ ...summarizePrefillRecordOps(rerankPrefixPhases),
1249
+ },
1250
+ documentPhase: {
1251
+ ...summarizeTimingPhaseSamples(rerankDocumentPhases, [
1252
+ 'totalMs',
1253
+ 'prefillCallMs',
1254
+ 'inputMs',
1255
+ 'prefillMs',
1256
+ 'prefillRecordMs',
1257
+ 'prefillRecordOps',
1258
+ 'prefillRecordPasses',
1259
+ 'prefillSubmitWaitMs',
1260
+ 'gpuPrefillMs',
1261
+ 'tokens',
1262
+ 'selectedTokenCount',
1263
+ 'prefixTokens',
1264
+ 'suffixTokens',
1265
+ 'promptChars',
1266
+ ]),
1267
+ ...summarizePrefillRecordOps(rerankDocumentPhases),
1268
+ },
1138
1269
  documents: {
1139
1270
  count: documentCountStats,
1140
1271
  },
@@ -1158,12 +1289,16 @@ async function runBenchSuite(options = {}) {
1158
1289
  const embeddingDims = [];
1159
1290
  const embeddingTokenCounts = [];
1160
1291
  const embeddingNorms = [];
1292
+ const embeddingPhases = [];
1161
1293
  let firstTimedEmbeddingMs = null;
1162
1294
  let invalidRuns = 0;
1163
1295
  let totalNonFiniteValues = 0;
1164
1296
  for (let i = 0; i < warmupRuns + timedRuns; i++) {
1165
1297
  harness.pipeline.reset?.();
1166
- const run = await runEmbedding(harness.pipeline, runtimeConfig, benchRun);
1298
+ const run = await runEmbedding(harness.pipeline, runtimeConfig, {
1299
+ ...benchRun,
1300
+ benchmark: true,
1301
+ });
1167
1302
  if (i >= warmupRuns) {
1168
1303
  timedDurations.push(run.durationMs);
1169
1304
  if (firstTimedEmbeddingMs == null) {
@@ -1179,6 +1314,9 @@ async function runBenchSuite(options = {}) {
1179
1314
  if (run.embeddingDim > 0 && run.nonFiniteCount === 0) {
1180
1315
  durations.push(run.durationMs);
1181
1316
  embeddingDims.push(run.embeddingDim);
1317
+ if (run.phase && typeof run.phase === 'object') {
1318
+ embeddingPhases.push(run.phase);
1319
+ }
1182
1320
  } else {
1183
1321
  invalidRuns++;
1184
1322
  }
@@ -1233,6 +1371,26 @@ async function runBenchSuite(options = {}) {
1233
1371
  timedEmbeddingMs: timedEmbeddingMsStats,
1234
1372
  embeddingMs: embeddingMsStats,
1235
1373
  },
1374
+ phase: {
1375
+ ...summarizeTimingPhaseSamples(embeddingPhases, [
1376
+ 'totalMs',
1377
+ 'inputMs',
1378
+ 'prefillMs',
1379
+ 'submitWaitMs',
1380
+ 'readbackMs',
1381
+ 'decodeHiddenMs',
1382
+ 'finalNormMs',
1383
+ 'extractMs',
1384
+ 'hiddenBytes',
1385
+ 'tokens',
1386
+ 'prefillRecordMs',
1387
+ 'prefillRecordOps',
1388
+ 'prefillRecordPasses',
1389
+ 'prefillSubmitWaitMs',
1390
+ 'gpuPrefillMs',
1391
+ ]),
1392
+ ...summarizePrefillRecordOps(embeddingPhases),
1393
+ },
1236
1394
  dimensions: {
1237
1395
  embedding: embeddingDimStats,
1238
1396
  },
@@ -1289,6 +1447,10 @@ async function runBenchSuite(options = {}) {
1289
1447
  const gpuDecodeReadbackCopyMs = [];
1290
1448
  const gpuDecodeOrchestrationMs = [];
1291
1449
  const gpuPrefillRecordMs = [];
1450
+ const gpuPrefillRecordOps = [];
1451
+ const gpuPrefillRecordPasses = [];
1452
+ const gpuPrefillRecordOpLabels = {};
1453
+ let hasGpuPrefillRecordOpLabels = false;
1292
1454
  const gpuPrefillSubmitWaitMs = [];
1293
1455
  const singleTokenSubmitWaitMs = [];
1294
1456
  const singleTokenReadbackWaitMs = [];
@@ -1334,6 +1496,7 @@ async function runBenchSuite(options = {}) {
1334
1496
  },
1335
1497
  () => runTextInference(harness.pipeline, runtimeConfig, {
1336
1498
  ...benchRun,
1499
+ benchmark: true,
1337
1500
  ...(options.inferenceInput ?? {}),
1338
1501
  })
1339
1502
  );
@@ -1411,6 +1574,15 @@ async function runBenchSuite(options = {}) {
1411
1574
  if (Number.isFinite(phaseGpu?.decodeReadbackCleanupMs)) gpuDecodeReadbackCleanupMs.push(phaseGpu.decodeReadbackCleanupMs);
1412
1575
  if (Number.isFinite(phaseGpu?.decodeReadbackCopyMs)) gpuDecodeReadbackCopyMs.push(phaseGpu.decodeReadbackCopyMs);
1413
1576
  if (Number.isFinite(phaseGpu?.prefillRecordMs)) gpuPrefillRecordMs.push(phaseGpu.prefillRecordMs);
1577
+ if (Number.isFinite(phaseGpu?.prefillRecordOps)) gpuPrefillRecordOps.push(phaseGpu.prefillRecordOps);
1578
+ if (Number.isFinite(phaseGpu?.prefillRecordPasses)) gpuPrefillRecordPasses.push(phaseGpu.prefillRecordPasses);
1579
+ const phasePrefillRecordOpLabels = normalizeDecodeRecordOpLabels(phaseGpu?.prefillRecordOpLabels);
1580
+ if (phasePrefillRecordOpLabels) {
1581
+ hasGpuPrefillRecordOpLabels = true;
1582
+ for (const [label, count] of Object.entries(phasePrefillRecordOpLabels)) {
1583
+ gpuPrefillRecordOpLabels[label] = (gpuPrefillRecordOpLabels[label] ?? 0) + count;
1584
+ }
1585
+ }
1414
1586
  if (Number.isFinite(phaseGpu?.prefillSubmitWaitMs)) gpuPrefillSubmitWaitMs.push(phaseGpu.prefillSubmitWaitMs);
1415
1587
  if (Number.isFinite(phaseGpu?.decodeOrchestrationMs)) {
1416
1588
  gpuDecodeOrchestrationMs.push(phaseGpu.decodeOrchestrationMs);
@@ -1454,6 +1626,8 @@ async function runBenchSuite(options = {}) {
1454
1626
  const decodeTokensStats = computeSampleStats(decodeTokens);
1455
1627
  const gpuDecodeRecordOpsStats = computeSampleStats(gpuDecodeRecordOps);
1456
1628
  const gpuDecodeRecordPassesStats = computeSampleStats(gpuDecodeRecordPasses);
1629
+ const gpuPrefillRecordOpsStats = computeSampleStats(gpuPrefillRecordOps);
1630
+ const gpuPrefillRecordPassesStats = computeSampleStats(gpuPrefillRecordPasses);
1457
1631
  const gpuDecodeRecordOpLabelSampleCount = gpuDecodeRecordOps.length > 0
1458
1632
  ? gpuDecodeRecordOps.length
1459
1633
  : 1;
@@ -1463,14 +1637,25 @@ async function runBenchSuite(options = {}) {
1463
1637
  gpuDecodeRecordMeanOpLabels[label] = count / gpuDecodeRecordOpLabelSampleCount;
1464
1638
  }
1465
1639
  }
1640
+ const gpuPrefillRecordOpLabelSampleCount = gpuPrefillRecordOps.length > 0
1641
+ ? gpuPrefillRecordOps.length
1642
+ : 1;
1643
+ const gpuPrefillRecordMeanOpLabels = {};
1644
+ if (hasGpuPrefillRecordOpLabels) {
1645
+ for (const [label, count] of Object.entries(gpuPrefillRecordOpLabels)) {
1646
+ gpuPrefillRecordMeanOpLabels[label] = count / gpuPrefillRecordOpLabelSampleCount;
1647
+ }
1648
+ }
1466
1649
  const hasGpuStats = gpuPrefillMs.length > 0 || gpuDecodeMs.length > 0 || gpuDecodeRecordMs.length > 0
1467
1650
  || gpuDecodeRecordOps.length > 0 || gpuDecodeRecordPasses.length > 0
1651
+ || gpuPrefillRecordOps.length > 0 || gpuPrefillRecordPasses.length > 0
1468
1652
  || gpuDecodeRecordMsPerOp.length > 0 || gpuDecodeRecordMsPerPass.length > 0
1469
1653
  || gpuDecodeRecordPassesPerOp.length > 0
1470
1654
  || gpuDecodeRecordMsPerExecutedBatchToken.length > 0
1471
1655
  || gpuDecodeRecordOpsPerExecutedBatchToken.length > 0
1472
1656
  || gpuDecodeRecordPassesPerExecutedBatchToken.length > 0
1473
1657
  || hasGpuDecodeRecordOpLabels
1658
+ || hasGpuPrefillRecordOpLabels
1474
1659
  || gpuDecodeSubmitWaitMs.length > 0 || gpuDecodeReadbackWaitMs.length > 0
1475
1660
  || gpuDecodeReadbackMapWaitMs.length > 0 || gpuDecodeReadbackCleanupMs.length > 0
1476
1661
  || gpuDecodeReadbackCopyMs.length > 0
@@ -1513,6 +1698,21 @@ async function runBenchSuite(options = {}) {
1513
1698
  decodeReadbackCopyMs: computeSampleStats(gpuDecodeReadbackCopyMs),
1514
1699
  decodeOrchestrationMs: computeSampleStats(gpuDecodeOrchestrationMs),
1515
1700
  prefillRecordMs: computeSampleStats(gpuPrefillRecordMs),
1701
+ prefillRecordOps: gpuPrefillRecordOpsStats,
1702
+ prefillRecordPasses: gpuPrefillRecordPassesStats,
1703
+ prefillRecordUniqueOpLabels: hasGpuPrefillRecordOpLabels ? Object.keys(gpuPrefillRecordOpLabels).length : null,
1704
+ prefillRecordTopOps: hasGpuPrefillRecordOpLabels
1705
+ ? buildDecodeRecordTopOps(
1706
+ gpuPrefillRecordMeanOpLabels,
1707
+ gpuPrefillRecordOpsStats?.mean
1708
+ )
1709
+ : [],
1710
+ prefillRecordTopOpGroups: hasGpuPrefillRecordOpLabels
1711
+ ? buildDecodeRecordTopOpGroups(
1712
+ gpuPrefillRecordMeanOpLabels,
1713
+ gpuPrefillRecordOpsStats?.mean
1714
+ )
1715
+ : [],
1516
1716
  prefillSubmitWaitMs: computeSampleStats(gpuPrefillSubmitWaitMs),
1517
1717
  uniformCache: lastGpuUniformCache,
1518
1718
  singleTokenSubmitWaitMs: computeSampleStats(singleTokenSubmitWaitMs),
@@ -1880,7 +2080,25 @@ export async function runBrowserSuite(options = {}) {
1880
2080
  report.timestamp = suiteTimestamp;
1881
2081
  }
1882
2082
  const reportInfo = await saveReport(modelId, report, { timestamp: report.timestamp });
1883
- return { ...suiteResult, mode, workload, debugSnapshot, report, reportInfo };
2083
+ const requestReceipt = {
2084
+ ...(
2085
+ suiteResult.request && typeof suiteResult.request === 'object'
2086
+ ? suiteResult.request
2087
+ : {}
2088
+ ),
2089
+ runtimeProfile: options.runtimeProfile ?? null,
2090
+ runtimeConfigUrl: options.runtimeConfigUrl ?? null,
2091
+ runtimeConfig: cloneRuntimeConfig(getRuntimeConfig()),
2092
+ };
2093
+ return {
2094
+ ...suiteResult,
2095
+ mode,
2096
+ workload,
2097
+ request: requestReceipt,
2098
+ debugSnapshot,
2099
+ report,
2100
+ reportInfo,
2101
+ };
1884
2102
  });
1885
2103
  }
1886
2104
 
@@ -893,6 +893,46 @@ export class KVCache {
893
893
  return cloned;
894
894
  }
895
895
 
896
+ if (this.useGPU) {
897
+ const device = getDevice();
898
+ if (!device) {
899
+ throw new Error('GPU device not initialized');
900
+ }
901
+ const cloned = new KVCache({
902
+ numLayers: this.numLayers,
903
+ numHeads: this.numHeads,
904
+ headDim: this.headDim,
905
+ maxSeqLen: this.maxSeqLen,
906
+ useGPU: true,
907
+ layout: 'contiguous',
908
+ pageSize: this.pageSize,
909
+ kvDtype: this.kvDtype
910
+ });
911
+
912
+ cloned.currentSeqLen = this.currentSeqLen;
913
+ cloned.totalTokensSeen = this.totalTokensSeen;
914
+
915
+ for (let l = 0; l < this.numLayers; l++) {
916
+ const src = this.layers[l];
917
+ const dst = cloned.layers[l];
918
+ dst.seqLen = src.seqLen;
919
+
920
+ if (!src.keysGPU || !src.valuesGPU || !dst.keysGPU || !dst.valuesGPU) {
921
+ throw new Error(`KVCache GPU clone requires contiguous GPU buffers for layer ${l}.`);
922
+ }
923
+
924
+ const usedBytes = src.seqLen * this.kvSize * this.bytesPerElem;
925
+ if (usedBytes > 0) {
926
+ const encoder = device.createCommandEncoder({ label: `kv_cache_clone_contiguous_${l}` });
927
+ encoder.copyBufferToBuffer(src.keysGPU, 0, dst.keysGPU, 0, usedBytes);
928
+ encoder.copyBufferToBuffer(src.valuesGPU, 0, dst.valuesGPU, 0, usedBytes);
929
+ device.queue.submit([encoder.finish()]);
930
+ }
931
+ }
932
+
933
+ return cloned;
934
+ }
935
+
896
936
  const cloned = new KVCache({
897
937
  numLayers: this.numLayers,
898
938
  numHeads: this.numHeads,
@@ -905,6 +945,7 @@ export class KVCache {
905
945
  });
906
946
 
907
947
  cloned.currentSeqLen = this.currentSeqLen;
948
+ cloned.totalTokensSeen = this.totalTokensSeen;
908
949
 
909
950
  for (let l = 0; l < this.numLayers; l++) {
910
951
  const { keys, values } = this.get(l);
@@ -956,7 +997,7 @@ export class KVCache {
956
997
  this.gpuContext = gpuContext;
957
998
 
958
999
  // Migrate existing cache to GPU if we have data
959
- if (this.currentSeqLen > 0 && gpuContext?.device) {
1000
+ if (this.currentSeqLen > 0 && gpuContext?.device && !this.hasGPUCache()) {
960
1001
  this._migrateToGPU(gpuContext.device);
961
1002
  }
962
1003
  }
@@ -0,0 +1,18 @@
1
+ export interface JsonGrammarMaskTokenizer {
2
+ decode(ids: number[], skipSpecial?: boolean, skipBos?: boolean): string;
3
+ }
4
+
5
+ export interface JsonGrammarMaskOptions {
6
+ tokenizer?: JsonGrammarMaskTokenizer | null;
7
+ cacheBudget?: number;
8
+ }
9
+
10
+ export interface JsonGrammarMaskContext {
11
+ generatedIds: number[];
12
+ tokenizer?: JsonGrammarMaskTokenizer;
13
+ vocabSize?: number;
14
+ }
15
+
16
+ export function createJsonGrammarMask(
17
+ options?: JsonGrammarMaskOptions
18
+ ): (logits: Float32Array, context: JsonGrammarMaskContext) => void;
@@ -26,7 +26,7 @@ export function isSlidingLayerType(layerType) {
26
26
  // KV Cache State Resolution
27
27
  // ============================================================================
28
28
 
29
- export function resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens) {
29
+ export function resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens, options = {}) {
30
30
  const kvState = {
31
31
  cachedK: undefined,
32
32
  cachedV: undefined,
@@ -61,7 +61,10 @@ export function resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSe
61
61
  totalSeqLen: currentSeqLen + numTokens,
62
62
  };
63
63
 
64
- kvState.hasCache = !!state.kvCache?.hasGPUCache?.();
64
+ kvState.forceCurrentTensors = options.forceCurrentTensors === true;
65
+ kvState.hasCache = kvState.forceCurrentTensors
66
+ ? false
67
+ : !!state.kvCache?.hasGPUCache?.();
65
68
 
66
69
  if (!kvState.hasCache) {
67
70
  kvState.cachedK = kTensor.buffer;
@@ -318,7 +321,9 @@ export function buildAttentionDispatchParams(config, state, kTensor, vTensor, kv
318
321
  const {
319
322
  numTokens, slidingWindow, layerType, headDim, queryPreAttnScalar, numKVHeads,
320
323
  } = config;
321
- const resolvedKvCacheDtype = config.kvCacheDtype ?? state.kvCache?.kvDtype ?? null;
324
+ const resolvedKvCacheDtype = kvState.forceCurrentTensors === true
325
+ ? null
326
+ : (config.kvCacheDtype ?? state.kvCache?.kvDtype ?? null);
322
327
 
323
328
  // Tiered prefill fallback: tiered layout does not support prefill (numTokens > 1)
324
329
  let prefillFallbackNeedsCast = false;
@@ -0,0 +1,37 @@
1
+ function kernelPathUsesHead256DecodeAttention(kernelPath) {
2
+ const steps = kernelPath?.decode?.steps;
3
+ if (!Array.isArray(steps)) return false;
4
+ return steps.some((step) => (
5
+ step?.op === 'attention'
6
+ && step?.kernel === 'attention_decode_online_head256_f16kv_output_gate.wgsl'
7
+ && (step?.entry === undefined || step.entry === 'main')
8
+ ));
9
+ }
10
+
11
+ function tensorElementCount(tensor) {
12
+ if (!Array.isArray(tensor?.shape)) return null;
13
+ return tensor.shape.reduce((total, value) => total * value, 1);
14
+ }
15
+
16
+ export function canUseAttentionOutputGateFusion(options = {}) {
17
+ const {
18
+ session,
19
+ qGateTensor,
20
+ numTokens,
21
+ numHeads,
22
+ headDim,
23
+ cachedKDtype,
24
+ cachedVDtype,
25
+ kernelPath,
26
+ diffusionGemmaDecoder,
27
+ } = options;
28
+ if (diffusionGemmaDecoder === true) return false;
29
+ if (session?.attentionDecodeOnline?.useOutputGateFusion !== true) return false;
30
+ if (!qGateTensor?.buffer || qGateTensor.dtype !== 'f32') return false;
31
+ if (numTokens !== 1 || headDim !== 256) return false;
32
+ if (cachedKDtype !== 'f16' || cachedVDtype !== 'f16') return false;
33
+ if (!kernelPathUsesHead256DecodeAttention(kernelPath)) return false;
34
+ const gateElements = tensorElementCount(qGateTensor);
35
+ if (gateElements !== null && gateElements < numTokens * numHeads * headDim) return false;
36
+ return true;
37
+ }