doppler-gpu 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (953) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +106 -87
  6. package/src/capsule-runtime.d.ts +25 -0
  7. package/src/capsule-runtime.js +24 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +2 -0
  24. package/src/client/capsule-host.d.ts +5 -0
  25. package/src/client/capsule-host.js +2 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +38 -1
  28. package/src/client/doppler-api.js +41 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +323 -0
  45. package/src/client/{runtime → model-host}/model-session.js +202 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +69 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +101 -0
  64. package/src/client/runtime/capsule-program-adapter.js +153 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +89 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +77 -0
  72. package/src/client/runtime/composition-root.js +352 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +19 -0
  92. package/src/client/runtime/session-controller.js +92 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +101 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +12 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +63 -0
  109. package/src/config/capsule-operations.json +17 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  128. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +2 -2
  129. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +62 -4
  130. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  132. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +3 -3
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +3 -3
  134. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +3 -3
  135. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +3 -3
  136. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +3 -3
  137. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +3 -3
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +5 -5
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +4 -4
  140. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +6 -6
  141. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +3 -3
  142. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  143. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +6 -6
  145. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +3 -3
  146. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +3 -3
  147. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  148. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  149. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  150. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  151. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  152. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +3 -3
  153. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  154. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  155. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +6 -6
  157. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +6 -6
  158. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +6 -6
  160. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +7 -7
  161. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +6 -6
  162. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  163. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +3 -3
  164. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +3 -3
  165. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  166. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +60 -8
  167. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  168. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  169. package/src/config/dtype-transition-contract.js +40 -0
  170. package/src/config/embedding-contract.js +26 -0
  171. package/src/config/embedding-reference.js +85 -0
  172. package/src/config/evidence/source-intake-policy.json +1 -1
  173. package/src/config/execution-contract-check.d.ts +16 -73
  174. package/src/config/execution-contract-check.js +7 -429
  175. package/src/config/forecast-manifest.js +56 -0
  176. package/src/config/generation-contract.d.ts +39 -0
  177. package/src/config/generation-contract.js +66 -0
  178. package/src/config/generation-contract.json +32 -0
  179. package/src/config/heterogeneous-attention-contract.js +46 -0
  180. package/src/config/initial-execution-identity.d.ts +58 -0
  181. package/src/config/initial-execution-identity.js +331 -0
  182. package/src/config/kernel-path-loader.d.ts +17 -0
  183. package/src/config/kernel-path-loader.js +42 -8
  184. package/src/config/kernel-registry-contract.d.ts +25 -0
  185. package/src/config/kernel-registry-contract.js +46 -0
  186. package/src/config/kernels/kernel-ref-digests.js +66 -18
  187. package/src/config/kernels/registry.js +1 -1
  188. package/src/config/kernels/registry.json +13617 -11377
  189. package/src/config/lora-layouts.d.ts +10 -0
  190. package/src/config/lora-layouts.js +17 -0
  191. package/src/config/lora-layouts.json +9 -0
  192. package/src/config/manifest-inference-validation.js +575 -0
  193. package/src/config/merge/execution-patches.js +12 -0
  194. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  195. package/src/config/merge/value-selection.js +38 -0
  196. package/src/config/merge.js +43 -11
  197. package/src/config/model-ir-v2.d.ts +57 -0
  198. package/src/config/model-ir-v2.js +258 -0
  199. package/src/config/model-ir.d.ts +49 -0
  200. package/src/config/model-ir.js +180 -0
  201. package/src/config/param-validator.js +1 -1
  202. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  203. package/src/config/platforms/loader.js +2 -1
  204. package/src/config/production-release-evidence.d.ts +23 -0
  205. package/src/config/production-release-evidence.js +423 -0
  206. package/src/config/production-release.js +321 -0
  207. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  208. package/src/config/required-inference-fields-contract-check.js +2 -2
  209. package/src/config/rerank-reference.d.ts +28 -0
  210. package/src/config/rerank-reference.js +89 -0
  211. package/src/config/revocation-policy.d.ts +65 -0
  212. package/src/config/revocation-policy.js +290 -0
  213. package/src/config/revocation-registry.json +11 -0
  214. package/src/config/revocation-updates.d.ts +68 -0
  215. package/src/config/revocation-updates.js +440 -0
  216. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  217. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  218. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  219. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  220. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  221. package/src/config/runtime-merge.js +1 -1
  222. package/src/config/runtime.js +1 -1
  223. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  224. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  225. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  226. package/src/config/schema/conversion-report.schema.js +29 -0
  227. package/src/config/schema/converter.schema.d.ts +1 -0
  228. package/src/config/schema/debug.schema.d.ts +1 -0
  229. package/src/config/schema/doppler.schema.js +6 -5
  230. package/src/config/schema/emulation.schema.js +1 -1
  231. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  232. package/src/config/schema/execution-v1.schema.js +19 -0
  233. package/src/config/schema/harness.schema.d.ts +13 -0
  234. package/src/config/schema/harness.schema.js +6 -0
  235. package/src/config/schema/inference.schema.d.ts +21 -2
  236. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  237. package/src/config/schema/manifest.schema.d.ts +26 -7
  238. package/src/config/schema/manifest.schema.js +10 -1
  239. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  240. package/src/config/schema/program-bundle.schema.js +145 -10
  241. package/src/config/sequence-reference.d.ts +26 -0
  242. package/src/config/sequence-reference.js +100 -0
  243. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  244. package/src/config/target-plan.d.ts +108 -0
  245. package/src/config/target-plan.js +411 -0
  246. package/src/config/training-defaults.js +1 -1
  247. package/src/config/transforms/capability-transform-resolver.js +4 -105
  248. package/src/config/transforms/execution-graph/dtype.js +591 -0
  249. package/src/config/transforms/execution-graph/fusion.js +436 -0
  250. package/src/config/transforms/execution-graph/validation.js +414 -0
  251. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  252. package/src/converter/artifact-identity.js +475 -0
  253. package/src/converter/conversion-plan.js +1 -1
  254. package/src/converter/core.js +94 -2096
  255. package/src/converter/forge-candidate-evaluation.js +271 -0
  256. package/src/converter/forge-qualification.js +163 -0
  257. package/src/converter/forge-stages.js +800 -0
  258. package/src/converter/manifest-builder.js +683 -0
  259. package/src/converter/quantization-info.js +4 -1
  260. package/src/converter/quantizer.js +76 -13
  261. package/src/converter/shard-packer.d.ts +1 -1
  262. package/src/converter/shard-packer.js +2 -2
  263. package/src/converter/tensor-transform.js +640 -0
  264. package/src/converter/tokenizer-contract.js +243 -0
  265. package/src/converter/tokenizer-manifest.js +243 -0
  266. package/src/debug/history.js +0 -8
  267. package/src/debug/index.js +0 -10
  268. package/src/debug/tensor.js +1 -1
  269. package/src/debug/trace.js +0 -7
  270. package/src/experimental/adapters/adapter-manager.js +2 -0
  271. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  272. package/src/experimental/adapters/adapter-manifest.js +11 -0
  273. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  274. package/src/experimental/adapters/artifact-contract.js +5 -2
  275. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  276. package/src/experimental/adapters/lora-loader.js +110 -48
  277. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  278. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  279. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  280. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  281. package/src/experimental/browser/browser-converter.js +11 -11
  282. package/src/experimental/diffusion/index.d.ts +1 -1
  283. package/src/experimental/diffusion/index.js +1 -1
  284. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  285. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  286. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  287. package/src/experimental/diffusion/vae.js +596 -0
  288. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  289. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  290. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  291. package/src/experimental/distribution/shard-delivery.js +4 -1709
  292. package/src/experimental/energy/index.d.ts +2 -2
  293. package/src/experimental/energy/index.js +2 -2
  294. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  295. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  296. package/src/experimental/logits/cpu-output.d.ts +27 -0
  297. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  298. package/src/experimental/logits/cpu.d.ts +87 -0
  299. package/src/experimental/logits/cpu.js +95 -0
  300. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  301. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  302. package/src/experimental/orchestration/index.d.ts +1 -1
  303. package/src/experimental/orchestration/index.js +1 -1
  304. package/src/experimental/training/artifacts.js +2 -2
  305. package/src/experimental/training/checkpoint.js +2 -2
  306. package/src/experimental/training/distillation/artifacts.js +1 -1
  307. package/src/experimental/training/distillation/dataset.js +2 -2
  308. package/src/experimental/training/distillation/eval.js +1 -1
  309. package/src/experimental/training/distillation/runtime.js +2 -2
  310. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  311. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  312. package/src/experimental/training/distillation/student-fixture.js +2 -469
  313. package/src/experimental/training/lora/checkpoint.js +746 -0
  314. package/src/experimental/training/lora/execution.js +550 -0
  315. package/src/experimental/training/lora/plan.js +387 -0
  316. package/src/experimental/training/lora/recovery.js +418 -0
  317. package/src/experimental/training/lora-pipeline.js +7 -1949
  318. package/src/experimental/training/operator-artifacts.js +2 -2
  319. package/src/experimental/training/operator-command.js +4 -4
  320. package/src/experimental/training/runner/evaluation.js +62 -0
  321. package/src/experimental/training/runner/lifecycle.js +257 -0
  322. package/src/experimental/training/runner.js +5 -268
  323. package/src/experimental/training/suite/execution.js +879 -0
  324. package/src/experimental/training/suite/plan.js +855 -0
  325. package/src/experimental/training/suite.js +5 -1659
  326. package/src/experimental/training/workloads.d.ts +5 -5
  327. package/src/experimental/training/workloads.js +6 -6
  328. package/src/experimental/turboquant/codebook.js +470 -0
  329. package/src/formats/quantization-constants.d.ts +11 -0
  330. package/src/formats/quantization-constants.js +11 -0
  331. package/src/formats/rdrr/integrity.js +1 -1
  332. package/src/formats/rdrr/lowerings.js +0 -29
  333. package/src/formats/rdrr/merkle.js +1 -1
  334. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  335. package/src/formats/rdrr/validation.js +3 -3
  336. package/src/gpu/command-recorder.d.ts +10 -1
  337. package/src/gpu/command-recorder.js +58 -14
  338. package/src/gpu/constant-buffer.js +29 -0
  339. package/src/gpu/device-state.js +44 -0
  340. package/src/gpu/device.js +11 -41
  341. package/src/gpu/kernel-runtime.js +2 -1
  342. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  343. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  344. package/src/gpu/kernels/attention/executors.js +960 -0
  345. package/src/gpu/kernels/attention/plan.js +472 -0
  346. package/src/gpu/kernels/attention.d.ts +4 -0
  347. package/src/gpu/kernels/attention.js +145 -1545
  348. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  349. package/src/gpu/kernels/backward/adam.js +2 -1
  350. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  351. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  352. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  353. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  354. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  355. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  356. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  357. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  358. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  359. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  360. package/src/gpu/kernels/backward/executor.js +158 -0
  361. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  362. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  363. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  364. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  365. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  366. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  367. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  368. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  369. package/src/gpu/kernels/backward/matmul.js +133 -0
  370. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  371. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  372. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  374. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  376. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  377. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  378. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  379. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  380. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  381. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  382. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  383. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  384. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  385. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  386. package/src/gpu/kernels/backward/utils.js +2 -392
  387. package/src/gpu/kernels/bind-group-validation.js +16 -0
  388. package/src/gpu/kernels/cast.js +2 -1
  389. package/src/gpu/kernels/check-finiteness.js +13 -49
  390. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  391. package/src/gpu/kernels/check-stop.js +14 -33
  392. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  393. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  394. package/src/gpu/kernels/check_stop.wgsl +20 -0
  395. package/src/gpu/kernels/clamp.js +1 -1
  396. package/src/gpu/kernels/conv2d.js +1 -1
  397. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  398. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  399. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  400. package/src/gpu/kernels/dequant.js +7 -2
  401. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  402. package/src/gpu/kernels/dispatch.js +12 -1
  403. package/src/gpu/kernels/embedding-pool.js +72 -0
  404. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  405. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  406. package/src/gpu/kernels/energy.js +2 -1
  407. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  408. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  409. package/src/gpu/kernels/fused_ffn.js +10 -220
  410. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  411. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  412. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  413. package/src/gpu/kernels/gated-short-conv.js +11 -69
  414. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  415. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  416. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  417. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  418. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  419. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  420. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  421. package/src/gpu/kernels/gather.d.ts +1 -1
  422. package/src/gpu/kernels/gather.js +30 -6
  423. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  424. package/src/gpu/kernels/gelu.d.ts +4 -0
  425. package/src/gpu/kernels/gelu.js +12 -2
  426. package/src/gpu/kernels/gelu.wgsl +16 -0
  427. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  428. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  429. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  430. package/src/gpu/kernels/groupnorm.js +1 -1
  431. package/src/gpu/kernels/index.d.ts +38 -5
  432. package/src/gpu/kernels/index.js +18 -4
  433. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  434. package/src/gpu/kernels/kernel-base.js +1 -1
  435. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  436. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  437. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  438. package/src/gpu/kernels/kernel-configs.js +26 -54
  439. package/src/gpu/kernels/kernel-execution.js +141 -0
  440. package/src/gpu/kernels/kv-cache-write.js +4 -4
  441. package/src/gpu/kernels/kv-quantize.js +4 -1
  442. package/src/gpu/kernels/l2-normalize.js +36 -0
  443. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  444. package/src/gpu/kernels/layernorm.d.ts +4 -4
  445. package/src/gpu/kernels/layernorm.js +44 -7
  446. package/src/gpu/kernels/linear-attention-core.js +29 -449
  447. package/src/gpu/kernels/linear_attention.js +1 -1
  448. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  449. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  450. package/src/gpu/kernels/logit-finalize.js +103 -0
  451. package/src/gpu/kernels/logit-merge.js +13 -125
  452. package/src/gpu/kernels/logit-scatter.js +40 -0
  453. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  454. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  455. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  456. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  457. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  458. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  459. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  460. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  461. package/src/gpu/kernels/matmul-selection.js +3 -2
  462. package/src/gpu/kernels/matmul.js +4 -15
  463. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  464. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  465. package/src/gpu/kernels/modulate.js +1 -1
  466. package/src/gpu/kernels/moe.js +3 -1
  467. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  468. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  469. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  470. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  471. package/src/gpu/kernels/patch_context.wgsl +29 -0
  472. package/src/gpu/kernels/pipeline-cache.js +3 -1
  473. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  474. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  475. package/src/gpu/kernels/relu.js +1 -1
  476. package/src/gpu/kernels/rep-penalty.js +30 -5
  477. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  478. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  479. package/src/gpu/kernels/repeat_channels.js +1 -1
  480. package/src/gpu/kernels/residual.js +2 -2
  481. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  482. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  483. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  484. package/src/gpu/kernels/rmsnorm.js +1 -1
  485. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  486. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  487. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  488. package/src/gpu/kernels/rope-precompute.js +160 -0
  489. package/src/gpu/kernels/rope-qk.js +1 -1
  490. package/src/gpu/kernels/rope.js +2 -2
  491. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  492. package/src/gpu/kernels/sample.d.ts +1 -0
  493. package/src/gpu/kernels/sample.js +36 -17
  494. package/src/gpu/kernels/sample.wgsl +112 -157
  495. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  496. package/src/gpu/kernels/scale.js +1 -1
  497. package/src/gpu/kernels/shader-cache.js +8 -4
  498. package/src/gpu/kernels/shader-source-scope.js +68 -0
  499. package/src/gpu/kernels/silu.js +2 -1
  500. package/src/gpu/kernels/soft-embedding.js +3 -1
  501. package/src/gpu/kernels/softmax.js +4 -2
  502. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  503. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  504. package/src/gpu/kernels/split_qg.js +1 -1
  505. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  506. package/src/gpu/kernels/split_qkv.js +18 -3
  507. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  508. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  509. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  510. package/src/gpu/kernels/transpose.js +1 -1
  511. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  512. package/src/gpu/kernels/uniform-utils.js +7 -2
  513. package/src/gpu/kernels/upsample2d.js +1 -1
  514. package/src/gpu/kernels/utils.js +9 -330
  515. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  516. package/src/gpu/kernels/vision-average-pool.js +68 -0
  517. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  518. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  519. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  520. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  521. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  522. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  523. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  524. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  525. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  526. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  527. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  528. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  529. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  530. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  531. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  532. package/src/gpu/readback-buffer.js +20 -0
  533. package/src/gpu/submit-probe.js +2 -26
  534. package/src/gpu/weight-buffer.d.ts +1 -0
  535. package/src/gpu/weight-buffer.js +7 -0
  536. package/src/index.d.ts +12 -0
  537. package/src/index.js +2 -0
  538. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  539. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  540. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  541. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  542. package/src/inference/browser-harness/inference-execution.js +543 -0
  543. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  544. package/src/inference/browser-harness/model-resolution.js +137 -0
  545. package/src/inference/browser-harness/report.js +338 -0
  546. package/src/inference/browser-harness/request.js +263 -0
  547. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  548. package/src/inference/browser-harness/runtime-config.js +255 -0
  549. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  550. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  551. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  552. package/src/inference/browser-harness/runtime-report.js +117 -0
  553. package/src/inference/browser-harness/suite-summary.js +39 -0
  554. package/src/inference/browser-harness/text-evidence.js +694 -0
  555. package/src/inference/browser-harness/text-execution.js +757 -0
  556. package/src/inference/browser-harness/text-input.js +725 -0
  557. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  558. package/src/inference/browser-harness-contract-helpers.js +1 -1
  559. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  560. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  561. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  562. package/src/inference/browser-harness-text-helpers.js +31 -2199
  563. package/src/inference/browser-harness.d.ts +15 -2
  564. package/src/inference/browser-harness.js +38 -1952
  565. package/src/inference/kv-cache/base.js +67 -381
  566. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  567. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  568. package/src/inference/kv-cache/quantized.d.ts +2 -2
  569. package/src/inference/kv-cache/quantized.js +4 -8
  570. package/src/inference/kv-cache/update.js +309 -0
  571. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  572. package/src/inference/model-checkpoint-evidence.js +264 -0
  573. package/src/inference/moe-router.js +36 -216
  574. package/src/inference/multi-model-network.js +5 -341
  575. package/src/inference/pipelines/audio/index.js +1 -20
  576. package/src/inference/pipelines/audio/mel.js +0 -38
  577. package/src/inference/pipelines/context.js +6 -2
  578. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  579. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  580. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  581. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  582. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  583. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  584. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  585. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  586. package/src/inference/pipelines/factory.js +14 -3
  587. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  588. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  589. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -124
  590. package/src/inference/pipelines/text/abort-contract.js +22 -0
  591. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  592. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  593. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  594. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  595. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  596. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  597. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  598. package/src/inference/pipelines/text/attention/plan.js +17 -0
  599. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  600. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  601. package/src/inference/pipelines/text/attention/projections.js +20 -537
  602. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  603. package/src/inference/pipelines/text/attention/record.js +0 -6
  604. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  605. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  606. package/src/inference/pipelines/text/attention/run.js +0 -6
  607. package/src/inference/pipelines/text/attention/types.js +3 -31
  608. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  609. package/src/inference/pipelines/text/chat-format.js +42 -0
  610. package/src/inference/pipelines/text/config/normalization.js +729 -0
  611. package/src/inference/pipelines/text/config/validation.js +9 -0
  612. package/src/inference/pipelines/text/config.d.ts +12 -1
  613. package/src/inference/pipelines/text/config.js +14 -1228
  614. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  615. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  616. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  617. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  618. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  619. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  620. package/src/inference/pipelines/text/embed.js +178 -241
  621. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  622. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  623. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  624. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  625. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  626. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  627. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  628. package/src/inference/pipelines/text/execution-v1.js +7 -283
  629. package/src/inference/pipelines/text/execution.js +517 -0
  630. package/src/inference/pipelines/text/ffn/dense-executor.js +723 -0
  631. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  632. package/src/inference/pipelines/text/ffn/dense.js +6 -970
  633. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  634. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  635. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  636. package/src/inference/pipelines/text/generator/decode-runtime.js +985 -0
  637. package/src/inference/pipelines/text/generator/decode.js +964 -0
  638. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  639. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  640. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  641. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  642. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  643. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  644. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  645. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  646. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  647. package/src/inference/pipelines/text/generator/text.js +448 -0
  648. package/src/inference/pipelines/text/generator/training.js +753 -0
  649. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  650. package/src/inference/pipelines/text/generator-logits-observation.js +48 -0
  651. package/src/inference/pipelines/text/generator-runtime.js +23 -154
  652. package/src/inference/pipelines/text/generator-steps.js +45 -980
  653. package/src/inference/pipelines/text/generator.js +179 -3552
  654. package/src/inference/pipelines/text/image-transcription.js +304 -0
  655. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  656. package/src/inference/pipelines/text/init-rope.js +37 -189
  657. package/src/inference/pipelines/text/init.d.ts +8 -3
  658. package/src/inference/pipelines/text/init.js +6 -216
  659. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  660. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  661. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  662. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  663. package/src/inference/pipelines/text/layer-plan.js +45 -3
  664. package/src/inference/pipelines/text/layer.js +52 -522
  665. package/src/inference/pipelines/text/lifecycle.js +839 -0
  666. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  667. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  668. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  669. package/src/inference/pipelines/text/linear-attention.js +28 -534
  670. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  671. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  672. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  673. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  674. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  675. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  676. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  677. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  678. package/src/inference/pipelines/text/logits/index.js +62 -102
  679. package/src/inference/pipelines/text/logits/output-transform.js +54 -0
  680. package/src/inference/pipelines/text/logits/plan.js +184 -0
  681. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  682. package/src/inference/pipelines/text/logits/readback.js +9 -0
  683. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  684. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  685. package/src/inference/pipelines/text/lora-apply.js +12 -4
  686. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  687. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  688. package/src/inference/pipelines/text/model-load.js +0 -20
  689. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  690. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  691. package/src/inference/pipelines/text/moe/plan.js +47 -0
  692. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  693. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  694. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  695. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  696. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  697. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  698. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  699. package/src/inference/pipelines/text/probes.js +20 -0
  700. package/src/inference/pipelines/text/sampling-config.js +22 -113
  701. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  702. package/src/inference/pipelines/text/sampling.js +18 -375
  703. package/src/inference/pipelines/text/state.d.ts +8 -0
  704. package/src/inference/pipelines/text/state.js +1 -0
  705. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  706. package/src/inference/pipelines/text/types.d.ts +7 -3
  707. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  708. package/src/inference/pipelines/text.d.ts +2 -2
  709. package/src/inference/pipelines/text.js +46 -1633
  710. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  711. package/src/inference/pipelines/video/index.js +0 -21
  712. package/src/inference/pipelines/vision/attention.js +59 -0
  713. package/src/inference/pipelines/vision/encoder.js +254 -369
  714. package/src/inference/pipelines/vision/gemma4.js +204 -318
  715. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  716. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  717. package/src/inference/pipelines/vision/glmocr.js +557 -0
  718. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  719. package/src/inference/pipelines/vision/index.js +8 -27
  720. package/src/inference/pipelines/vision/ops.js +11 -33
  721. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  722. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  723. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  724. package/src/inference/refactor-receipt.js +2 -2
  725. package/src/inference/rerank.js +182 -0
  726. package/src/inference/token-sampling.js +319 -0
  727. package/src/inference/tokenizer.js +1 -1
  728. package/src/inference/tokenizers/behavior-flags.js +6 -3
  729. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  730. package/src/inference/tokenizers/bundled/registry.js +216 -0
  731. package/src/inference/tokenizers/bundled.js +23 -353
  732. package/src/loader/doppler-loader.d.ts +4 -0
  733. package/src/loader/doppler-loader.js +119 -930
  734. package/src/loader/embedding-limit-preflight.js +11 -4
  735. package/src/loader/embedding-loader.js +1 -3
  736. package/src/loader/gpu-tensor-materialization.js +20 -0
  737. package/src/loader/layer-loader.js +15 -119
  738. package/src/loader/layer-tensor-aliases.js +99 -0
  739. package/src/loader/layer-weight-shape.js +29 -0
  740. package/src/loader/load-timing.js +64 -0
  741. package/src/loader/loader-types.d.ts +3 -0
  742. package/src/loader/model-executor.js +819 -0
  743. package/src/loader/model-load-validation.js +15 -0
  744. package/src/loader/quantization-constants.d.ts +11 -29
  745. package/src/loader/quantization-constants.js +1 -2
  746. package/src/loader/shard-resolver.js +0 -14
  747. package/src/loader/tensor-load-error.js +14 -0
  748. package/src/loader/tensor-shard-indices.js +8 -0
  749. package/src/loader/tensors/bf16-loader.js +125 -0
  750. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  751. package/src/loader/tensors/load-executor.js +577 -0
  752. package/src/loader/tensors/source-transform/executor.js +123 -0
  753. package/src/loader/tensors/source-transform/plan.js +107 -0
  754. package/src/loader/tensors/source-transform.js +15 -264
  755. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  756. package/src/loader/tensors/tensor-loader.js +7 -653
  757. package/src/memory/buffer-pool.js +7 -29
  758. package/src/rules/capability-transform-resolver.js +80 -0
  759. package/src/rules/execution-contract.d.ts +73 -0
  760. package/src/rules/execution-contract.js +429 -0
  761. package/src/rules/execution-rules-contract-check.js +1 -1
  762. package/src/rules/generated/rule-bundle.json +4305 -0
  763. package/src/rules/inference/capability-transforms.rules.json +3 -3
  764. package/src/rules/layer-pattern-contract-check.js +1 -1
  765. package/src/rules/rule-registry.js +70 -57
  766. package/src/storage/artifact-storage-context.js +2 -2
  767. package/src/storage/backend-selection.js +43 -0
  768. package/src/storage/backends/opfs-store.d.ts +41 -0
  769. package/src/storage/distribution-transport.js +5 -0
  770. package/src/storage/download/integrity.d.ts +33 -0
  771. package/src/storage/download/integrity.js +130 -0
  772. package/src/storage/download/resume-inspection.d.ts +22 -0
  773. package/src/storage/download/resume-inspection.js +134 -0
  774. package/src/storage/download/retry.js +57 -0
  775. package/src/storage/download/state.js +131 -0
  776. package/src/storage/download/transport.js +99 -0
  777. package/src/storage/downloader.js +74 -453
  778. package/src/storage/model-cache.d.ts +1 -0
  779. package/src/storage/model-cache.js +51 -11
  780. package/src/storage/shard-manager.d.ts +5 -0
  781. package/src/storage/shard-manager.js +50 -369
  782. package/src/storage/shards/index.js +60 -0
  783. package/src/storage/shards/integrity.js +225 -0
  784. package/src/storage/shards/lifecycle.js +22 -0
  785. package/src/storage/source-artifact-store.js +14 -4
  786. package/src/tooling/boundary-evidence.d.ts +4 -4
  787. package/src/tooling/boundary-evidence.js +20 -20
  788. package/src/tooling/browser-command-runner.js +13 -6
  789. package/src/tooling/capsule-v2.d.ts +11 -0
  790. package/src/tooling/capsule-v2.js +58 -0
  791. package/src/tooling/capsule.js +20 -0
  792. package/src/tooling/command-api-constants.js +1 -1
  793. package/src/tooling/command-api-family-normalizers.js +163 -5
  794. package/src/tooling/command-api.d.ts +42 -3
  795. package/src/tooling/command-api.js +7 -2
  796. package/src/tooling/command-envelope.js +1 -1
  797. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  798. package/src/tooling/command-runner.html +49 -1
  799. package/src/tooling/command-validation.js +1 -24
  800. package/src/tooling/conversion-run-timing.js +24 -0
  801. package/src/tooling/diagnose-runner.js +11 -9
  802. package/src/tooling/distribution-shard-transport.js +11 -0
  803. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  804. package/src/tooling/execution-cost-ledger.js +197 -15
  805. package/src/tooling/hf-registry/catalog-io.js +63 -0
  806. package/src/tooling/litert/package.js +863 -0
  807. package/src/tooling/litert-package-runtime.js +3 -849
  808. package/src/tooling/model-capsule-forge.js +478 -0
  809. package/src/tooling/node-browser/execution.d.ts +40 -0
  810. package/src/tooling/node-browser/execution.js +308 -0
  811. package/src/tooling/node-browser/static-server.js +189 -0
  812. package/src/tooling/node-browser/transport.d.ts +80 -0
  813. package/src/tooling/node-browser/transport.js +622 -0
  814. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  815. package/src/tooling/node-browser-command-runner.js +10 -1112
  816. package/src/tooling/node-capsule-artifact-store.js +44 -0
  817. package/src/tooling/node-command-execution.js +237 -0
  818. package/src/tooling/node-command-runner.js +26 -205
  819. package/src/tooling/node-converter/input.js +521 -0
  820. package/src/tooling/node-converter/output.js +244 -0
  821. package/src/tooling/node-converter-chunk-batches.js +21 -0
  822. package/src/tooling/node-converter.js +7 -720
  823. package/src/tooling/node-source-runtime.js +16 -276
  824. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  825. package/src/tooling/node-webgpu.js +178 -453
  826. package/src/tooling/opfs-cache.d.ts +5 -0
  827. package/src/tooling/opfs-cache.js +4 -0
  828. package/src/tooling/production-release.d.ts +19 -0
  829. package/src/tooling/production-release.js +647 -0
  830. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  831. package/src/tooling/program-bundle/materialize.js +324 -0
  832. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  833. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  834. package/src/tooling/program-bundle/validation.js +31 -0
  835. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  836. package/src/tooling/program-bundle-parity.d.ts +11 -4
  837. package/src/tooling/program-bundle-parity.js +262 -109
  838. package/src/tooling/program-bundle.d.ts +7 -0
  839. package/src/tooling/program-bundle.js +186 -591
  840. package/src/tooling/provider-v1-contract.d.ts +61 -0
  841. package/src/tooling/provider-v1-contract.js +192 -0
  842. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  843. package/src/tooling/registered-variant-calibration-job.js +210 -0
  844. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  845. package/src/tooling/registered-variant-calibration.js +57 -7
  846. package/src/tooling/runtime-optimization/candidates.js +664 -0
  847. package/src/tooling/runtime-optimization-campaign.js +166 -0
  848. package/src/tooling/runtime-optimization-index.js +16 -1
  849. package/src/tooling/runtime-optimization.d.ts +44 -0
  850. package/src/tooling/runtime-optimization.js +13 -656
  851. package/src/tooling/source-intake.d.ts +1 -1
  852. package/src/tooling/source-intake.js +34 -17
  853. package/src/tooling/source-package-profiles.js +1 -1
  854. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  855. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  856. package/src/tooling/source-runtime/node-loader.js +258 -0
  857. package/src/tooling/source-runtime-bundle.js +26 -411
  858. package/src/tooling/source-runtime-converter-config.js +1 -1
  859. package/src/tooling-experimental-exports.js +0 -5
  860. package/src/tooling-exports/evidence.d.ts +14 -3
  861. package/src/tooling-exports/evidence.js +14 -3
  862. package/src/tooling-exports/storage.d.ts +7 -0
  863. package/src/tooling-exports/storage.js +5 -0
  864. package/src/tooling-exports.d.ts +3 -0
  865. package/src/tooling-exports.js +2 -0
  866. package/src/tooling-exports.shared.d.ts +17 -3
  867. package/src/tooling-exports.shared.js +9 -3
  868. package/src/training.d.ts +6 -6
  869. package/src/training.js +4 -4
  870. package/src/version.js +1 -1
  871. package/tests/kernels/browser/kernel-suite.js +512 -0
  872. package/tests/kernels/browser/test-page.js +170 -14
  873. package/tests/kernels/harness/tolerance.js +1 -0
  874. package/tests/kernels/reference/index.js +13 -1
  875. package/tests/kernels/reference/vision.js +160 -0
  876. package/src/converter/quantizer.d.ts +0 -121
  877. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  878. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  879. package/src/gpu/kernels/utils.d.ts +0 -147
  880. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  881. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  882. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  883. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  884. package/src/rules/converter/execution.rules.json +0 -20
  885. package/src/rules/converter/tensor-roles.rules.json +0 -13
  886. package/src/rules/converter/tokenizer.rules.json +0 -12
  887. package/src/rules/inference/attention.rules.json +0 -69
  888. package/src/rules/inference/config.rules.json +0 -82
  889. package/src/rules/inference/dtype.rules.json +0 -113
  890. package/src/rules/inference/execution.rules.json +0 -195
  891. package/src/rules/inference/ffn.rules.json +0 -123
  892. package/src/rules/inference/layer-pattern.rules.json +0 -16
  893. package/src/rules/inference/layer.rules.json +0 -7
  894. package/src/rules/inference/moe.rules.json +0 -145
  895. package/src/rules/kernels/attention.rules.json +0 -68
  896. package/src/rules/kernels/conv2d.rules.json +0 -6
  897. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  898. package/src/rules/kernels/dequant.rules.json +0 -58
  899. package/src/rules/kernels/energy.rules.json +0 -22
  900. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  901. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  902. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  903. package/src/rules/kernels/gather.rules.json +0 -93
  904. package/src/rules/kernels/gelu.rules.json +0 -11
  905. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  906. package/src/rules/kernels/groupnorm.rules.json +0 -10
  907. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  908. package/src/rules/kernels/layernorm.rules.json +0 -6
  909. package/src/rules/kernels/linear-attention.rules.json +0 -6
  910. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  911. package/src/rules/kernels/matmul.rules.json +0 -95
  912. package/src/rules/kernels/modulate.rules.json +0 -6
  913. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  914. package/src/rules/kernels/moe.rules.json +0 -14
  915. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  916. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  917. package/src/rules/kernels/relu.rules.json +0 -6
  918. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  919. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  920. package/src/rules/kernels/residual.rules.json +0 -12
  921. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  922. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  923. package/src/rules/kernels/rope-qk.rules.json +0 -6
  924. package/src/rules/kernels/rope.rules.json +0 -6
  925. package/src/rules/kernels/sample.rules.json +0 -6
  926. package/src/rules/kernels/scale.rules.json +0 -8
  927. package/src/rules/kernels/silu.rules.json +0 -22
  928. package/src/rules/kernels/softmax.rules.json +0 -25
  929. package/src/rules/kernels/split-qg.rules.json +0 -6
  930. package/src/rules/kernels/split-qkv.rules.json +0 -6
  931. package/src/rules/kernels/upsample2d.rules.json +0 -6
  932. package/src/rules/loader/tensor-loader.rules.json +0 -42
  933. package/src/rules/loader/weights.rules.json +0 -42
  934. package/src/rules/tooling/command-runtime.rules.json +0 -65
  935. package/src/tooling/hf-registry-utils.js +0 -395
  936. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  937. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  938. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  939. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  940. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  941. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  942. /package/src/{utils → formats}/array-buffer.js +0 -0
  943. /package/src/{utils → formats}/canonical-hash.js +0 -0
  944. /package/src/{utils → formats}/clone-json.js +0 -0
  945. /package/src/{utils → formats}/encode-utf8.js +0 -0
  946. /package/src/{utils → formats}/load-json.js +0 -0
  947. /package/src/{utils → formats}/plain-object.js +0 -0
  948. /package/src/{utils → formats}/sha256.js +0 -0
  949. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  950. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  951. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  952. /package/src/{utils → storage}/runtime-env.js +0 -0
  953. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -1,90 +1,57 @@
1
-
2
-
3
1
  import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
4
2
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
5
- import { runMatmul, runRMSNorm, runScale, recordScale } from '../../../../gpu/kernel-selector.js';
6
- import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
7
- import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
8
- import { recordLmHeadArgmax } from '../../../../gpu/kernels/lm-head-argmax.js';
3
+ import { runMatmul, runRMSNorm, runScale } from '../../../../gpu/kernel-selector.js';
9
4
  import { createTensor } from '../../../../gpu/tensor.js';
5
+ import { castF16ToF32, castF32ToF16 } from '../../../../gpu/kernels/cast.js';
10
6
  import {
11
- castF16ToF32,
12
- castF32ToF16,
13
- recordCastF16ToF32,
14
- recordCastF32ToF16,
15
- } from '../../../../gpu/kernels/cast.js';
16
- import { createWeightBuffer, createSplitWeightBuffer, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../../gpu/weight-buffer.js';
7
+ createWeightBuffer,
8
+ createSplitWeightBuffer,
9
+ isWeightBuffer,
10
+ isCpuWeightBuffer,
11
+ isGpuBufferInstance,
12
+ isSplitWeightBuffer,
13
+ } from '../../../../gpu/weight-buffer.js';
17
14
  import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
18
15
  import { getRuntimeConfig } from '../../../../config/runtime.js';
19
16
  import { getKernelPathMatmulPrecision, getKernelPathStepPrecision } from '../../../../config/kernel-path-loader.js';
20
17
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
21
18
  import { runProbes } from '../probes.js';
22
19
  import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
23
- import { f16BufferToF32 } from './cpu.js';
24
- import { readBufferWithCleanup, resolveLogitInputScale } from './utils.js';
25
- import { f16ToF32 } from '../../../../loader/dtype-utils.js';
20
+ import { readBufferWithCleanup } from './readback.js';
21
+ import { resolveLogitInputScale } from './scale-policy.js';
22
+ import { finalizeLogitOutputTensor } from './output-transform.js';
23
+ import { runLogitScatter } from '../../../../gpu/kernels/logit-scatter.js';
26
24
  import { shouldForceStableF32Logits, createStableF32LogitsKernelPath } from './precision-policy.js';
25
+ import {
26
+ extractLmHeadChunk,
27
+ isRangeBackedCpuWeightSource,
28
+ normalizeRangeBytes,
29
+ resolveCpuWeightDims,
30
+ shouldMaterializeSplitLmHeadGPU,
31
+ } from './plan.js';
32
+ import {
33
+ coerceTensorDtype,
34
+ recordGreedyLmHeadArgmaxGPU,
35
+ recordLogitsGPU,
36
+ resolveFinalNormGpuBuffer,
37
+ resolveLmHeadMatmulRole,
38
+ resolveMatmulStepDtype,
39
+ resolvePostLayerStepDtype,
40
+ } from './gpu-executor.js';
41
+
42
+ export {
43
+ extractLmHeadChunk,
44
+ resolveCpuWeightDims,
45
+ shouldMaterializeSplitLmHeadGPU,
46
+ writeChunkLogits,
47
+ } from './plan.js';
48
+ export {
49
+ recordGreedyLmHeadArgmaxGPU,
50
+ recordLogitsGPU,
51
+ } from './gpu-executor.js';
27
52
 
28
- function resolvePrecisionFieldDtype(precision, fallback, field) {
29
- const requested = precision?.[field] ?? fallback;
30
- if (requested == null) {
31
- return fallback;
32
- }
33
- return selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: requested });
34
- }
35
-
36
- function resolveMatmulStepDtype(role, phase, kernelPath, fallback, field) {
37
- const precision = getKernelPathMatmulPrecision(role, phase, 0, kernelPath);
38
- return resolvePrecisionFieldDtype(precision, fallback, field);
39
- }
40
-
41
- function resolvePostLayerStepDtype(op, phase, kernelPath, fallback, field) {
42
- const precision = getKernelPathStepPrecision(op, 'postLayer', phase, 0, kernelPath);
43
- return resolvePrecisionFieldDtype(precision, fallback, field);
44
- }
45
-
46
- function resolveLmHeadMatmulRole(phase) {
47
- return phase === 'prefill' ? 'lm_head_prefill' : 'lm_head';
48
- }
49
-
50
- async function coerceTensorDtype(tensor, targetDtype, recorder = null, options = {}) {
51
- if (!targetDtype || tensor.dtype === targetDtype) {
52
- return tensor;
53
- }
54
- assertImplicitDtypeTransitionAllowed({
55
- executionPolicies: options.executionPolicies ?? null,
56
- fromDtype: tensor.dtype,
57
- toDtype: targetDtype,
58
- op: options.op ?? 'logits',
59
- detail: 'The execution graph must declare this cast explicitly.',
60
- transitionDeclaredBy: options.transitionDeclaredBy ?? null,
61
- });
62
- if (tensor.dtype === 'f32' && targetDtype === 'f16') {
63
- return recorder ? await recordCastF32ToF16(recorder, tensor) : await castF32ToF16(tensor);
64
- }
65
- if (tensor.dtype === 'f16' && targetDtype === 'f32') {
66
- return recorder ? await recordCastF16ToF32(recorder, tensor) : await castF16ToF32(tensor);
67
- }
68
- throw new Error(`Unsupported logits matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
69
- }
70
-
71
- const bf16ScratchU32 = new Uint32Array(1);
72
- const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
73
53
  const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
74
54
 
75
- function bf16ToF32(value) {
76
- bf16ScratchU32[0] = (value & 0xffff) << 16;
77
- return bf16ScratchF32[0];
78
- }
79
-
80
- function isRangeBackedCpuWeightSource(value) {
81
- return (
82
- typeof value === 'object'
83
- && value !== null
84
- && value.kind === 'tensor_range_source'
85
- && typeof value.loadRange === 'function'
86
- );
87
- }
88
55
 
89
56
  function alignByteLength(byteLength) {
90
57
  return Math.ceil(byteLength / 4) * 4;
@@ -97,116 +64,6 @@ function writeBufferInChunks(queue, buffer, bytes) {
97
64
  }
98
65
  }
99
66
 
100
- function resolveFinalNormGpuBuffer(finalNorm, queue, label) {
101
- if (isWeightBuffer(finalNorm)) {
102
- return { buffer: finalNorm.buffer, owned: false };
103
- }
104
- if (isGpuBufferInstance(finalNorm)) {
105
- return { buffer: finalNorm, owned: false };
106
- }
107
- if (!ArrayBuffer.isView(finalNorm)) {
108
- throw new Error('[Logits] final_norm must be a GPU buffer, typed array, or WeightBuffer.');
109
- }
110
- const buffer = acquireBuffer(finalNorm.byteLength, undefined, label);
111
- try {
112
- queue.writeBuffer(buffer, 0, finalNorm);
113
- return { buffer, owned: true };
114
- } catch (error) {
115
- releaseBuffer(buffer);
116
- throw error;
117
- }
118
- }
119
-
120
- function normalizeRangeBytes(value, label) {
121
- if (value instanceof Uint8Array) return value;
122
- if (value instanceof ArrayBuffer) return new Uint8Array(value);
123
- if (ArrayBuffer.isView(value)) {
124
- return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
125
- }
126
- throw new Error(
127
- `[Logits] ${label} returned unsupported byte payload type "${value?.constructor?.name ?? typeof value}".`
128
- );
129
- }
130
-
131
- function decodeChunkIntoOutput(bytes, sourceDtype, output, dstOffset, valueCount) {
132
- if (sourceDtype === 'f16') {
133
- const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
134
- for (let index = 0; index < valueCount; index += 1) {
135
- output[dstOffset + index] = f16ToF32(values[index]);
136
- }
137
- return;
138
- }
139
- if (sourceDtype === 'bf16') {
140
- const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
141
- for (let index = 0; index < valueCount; index += 1) {
142
- output[dstOffset + index] = bf16ToF32(values[index]);
143
- }
144
- return;
145
- }
146
- if (((bytes.byteOffset % 4) === 0) && ((bytes.byteLength % 4) === 0)) {
147
- output.set(new Float32Array(bytes.buffer, bytes.byteOffset, valueCount), dstOffset);
148
- return;
149
- }
150
- const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
151
- for (let index = 0; index < valueCount; index += 1) {
152
- output[dstOffset + index] = view.getFloat32(index * 4, true);
153
- }
154
- }
155
-
156
- function readTypedLmHeadChunk(data, layout, hiddenSize, vocabSize, rowOffset, rowCount, sourceDtype) {
157
- if (data instanceof Float32Array) {
158
- if (layout === 'row') {
159
- const start = rowOffset * hiddenSize;
160
- return data.subarray(start, start + rowCount * hiddenSize);
161
- }
162
- const chunk = new Float32Array(hiddenSize * rowCount);
163
- for (let k = 0; k < hiddenSize; k++) {
164
- const srcOffset = k * vocabSize + rowOffset;
165
- const dstOffset = k * rowCount;
166
- chunk.set(data.subarray(srcOffset, srcOffset + rowCount), dstOffset);
167
- }
168
- return chunk;
169
- }
170
-
171
- if (!(data instanceof Uint16Array)) {
172
- throw new Error(
173
- `[Logits] Unsupported CPU LM head chunk source type "${data?.constructor?.name ?? typeof data}".`
174
- );
175
- }
176
-
177
- const chunk = new Float32Array(hiddenSize * rowCount);
178
- if (layout === 'row') {
179
- const start = rowOffset * hiddenSize;
180
- for (let index = 0; index < rowCount * hiddenSize; index += 1) {
181
- const raw = data[start + index];
182
- chunk[index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
183
- }
184
- return chunk;
185
- }
186
-
187
- for (let k = 0; k < hiddenSize; k += 1) {
188
- const srcOffset = k * vocabSize + rowOffset;
189
- const dstOffset = k * rowCount;
190
- for (let index = 0; index < rowCount; index += 1) {
191
- const raw = data[srcOffset + index];
192
- chunk[dstOffset + index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
193
- }
194
- }
195
- return chunk;
196
- }
197
-
198
-
199
- export function resolveCpuWeightDims(lmHead) {
200
- if (lmHead.shape.length !== 2) {
201
- throw new Error(`[Logits] CPU LM head shape must be 2D, got [${lmHead.shape.join(', ')}]`);
202
- }
203
- if (lmHead.layout === 'column') {
204
- return { hiddenSize: lmHead.shape[0], vocabSize: lmHead.shape[1] };
205
- }
206
- return { vocabSize: lmHead.shape[0], hiddenSize: lmHead.shape[1] };
207
- }
208
-
209
-
210
67
  export function resolveLmHeadChunkRows(
211
68
  device,
212
69
  numTokens,
@@ -239,86 +96,6 @@ export function resolveLmHeadChunkRows(
239
96
  }
240
97
 
241
98
 
242
- export async function extractLmHeadChunk(
243
- data,
244
- layout,
245
- hiddenSize,
246
- vocabSize,
247
- rowOffset,
248
- rowCount,
249
- sourceDtype
250
- ) {
251
- if (typeof sourceDtype !== 'string' || sourceDtype.trim() === '') {
252
- throw new Error('[Logits] CPU LM head source dtype is required.');
253
- }
254
- const normalizedSourceDtype = sourceDtype.trim().toLowerCase();
255
- if (normalizedSourceDtype !== 'f32' && normalizedSourceDtype !== 'f16' && normalizedSourceDtype !== 'bf16') {
256
- throw new Error(`[Logits] Unsupported CPU LM head source dtype "${sourceDtype}".`);
257
- }
258
- if (!isRangeBackedCpuWeightSource(data)) {
259
- return readTypedLmHeadChunk(
260
- data,
261
- layout,
262
- hiddenSize,
263
- vocabSize,
264
- rowOffset,
265
- rowCount,
266
- normalizedSourceDtype
267
- );
268
- }
269
-
270
- const bytesPerElement = normalizedSourceDtype === 'f32' ? 4 : 2;
271
- const chunk = new Float32Array(hiddenSize * rowCount);
272
- if (layout === 'row') {
273
- const byteOffset = rowOffset * hiddenSize * bytesPerElement;
274
- const byteLength = rowCount * hiddenSize * bytesPerElement;
275
- const bytes = normalizeRangeBytes(
276
- await data.loadRange(byteOffset, byteLength),
277
- 'CPU LM head range source'
278
- );
279
- if (bytes.byteLength !== byteLength) {
280
- throw new Error(
281
- `[Logits] CPU LM head range source returned ${bytes.byteLength} bytes, expected ${byteLength}.`
282
- );
283
- }
284
- decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, 0, rowCount * hiddenSize);
285
- return chunk;
286
- }
287
-
288
- for (let k = 0; k < hiddenSize; k += 1) {
289
- const byteOffset = (k * vocabSize + rowOffset) * bytesPerElement;
290
- const byteLength = rowCount * bytesPerElement;
291
- const bytes = normalizeRangeBytes(
292
- await data.loadRange(byteOffset, byteLength),
293
- `CPU LM head range source column ${k}`
294
- );
295
- if (bytes.byteLength !== byteLength) {
296
- throw new Error(
297
- `[Logits] CPU LM head range source returned ${bytes.byteLength} bytes for column ${k}, expected ${byteLength}.`
298
- );
299
- }
300
- decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, k * rowCount, rowCount);
301
- }
302
- return chunk;
303
- }
304
-
305
-
306
- export function writeChunkLogits(
307
- target,
308
- chunk,
309
- numTokens,
310
- vocabSize,
311
- rowOffset,
312
- rowCount
313
- ) {
314
- for (let t = 0; t < numTokens; t++) {
315
- const srcOffset = t * rowCount;
316
- const dstOffset = t * vocabSize + rowOffset;
317
- target.set(chunk.subarray(srcOffset, srcOffset + rowCount), dstOffset);
318
- }
319
- }
320
-
321
-
322
99
  function resolveSplitLmHeadRows(device, hiddenSize, largeWeightConfig) {
323
100
  if (largeWeightConfig.safetyRatio == null) {
324
101
  throw new Error('runtime.inference.largeWeights.safetyRatio is required.');
@@ -340,14 +117,6 @@ function resolveSplitLmHeadRows(device, hiddenSize, largeWeightConfig) {
340
117
  return rows;
341
118
  }
342
119
 
343
- export function shouldMaterializeSplitLmHeadGPU(lmHead, largeWeightConfig) {
344
- const overrides = largeWeightConfig?.gpuResidentOverrides;
345
- if (!Array.isArray(overrides) || overrides.length === 0) {
346
- return false;
347
- }
348
- const label = lmHead?.label;
349
- return typeof label === 'string' && overrides.includes(label);
350
- }
351
120
 
352
121
  function destroySplitWeightBuffer(splitWeight) {
353
122
  if (!splitWeight) {
@@ -454,7 +223,7 @@ async function materializeSplitLmHeadGPU(lmHead, hiddenSize, weightVocabSize, la
454
223
  }
455
224
 
456
225
 
457
- export async function computeChunkedLogitsGPU(
226
+ export async function computeChunkedLogitsTensorGPU(
458
227
  normedTensor,
459
228
  lmHead,
460
229
  numTokens,
@@ -477,7 +246,7 @@ export async function computeChunkedLogitsGPU(
477
246
 
478
247
  const splitLmHead = await materializeSplitLmHeadGPU(lmHead, hiddenSize, weightVocabSize, largeWeightConfig);
479
248
  if (splitLmHead) {
480
- return computeSplitLogitsGPU(
249
+ return computeSplitLogitsTensorGPU(
481
250
  normedTensor,
482
251
  splitLmHead,
483
252
  numTokens,
@@ -503,7 +272,11 @@ export async function computeChunkedLogitsGPU(
503
272
  hasF16: caps.hasF16,
504
273
  });
505
274
  const preferF16 = weightDtype === 'f16';
506
- const logits = new Float32Array(numTokens * vocabSize);
275
+ const logitsBuffer = acquireBuffer(
276
+ numTokens * vocabSize * Float32Array.BYTES_PER_ELEMENT,
277
+ undefined,
278
+ 'lm_head_chunked_logits'
279
+ );
507
280
 
508
281
  if (isTraceEnabled('logits')) {
509
282
  trace.logits(`LM_HEAD_CHUNKED: vocab=${vocabSize}, chunkRows=${chunkRows}, layout=${lmHead.layout}, f16=${preferF16}`);
@@ -517,8 +290,9 @@ export async function computeChunkedLogitsGPU(
517
290
  })
518
291
  : normedTensor;
519
292
 
520
- for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
521
- const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
293
+ try {
294
+ for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
295
+ const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
522
296
  const chunkShape = lmHead.layout === 'column'
523
297
  ? [hiddenSize, rowCount]
524
298
  : [rowCount, hiddenSize];
@@ -580,30 +354,32 @@ export async function computeChunkedLogitsGPU(
580
354
  });
581
355
  }
582
356
 
583
- const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsTensor.dtype });
584
- const chunkLogitsData = await readBufferWithCleanup(
585
- logitsTensor.buffer,
586
- numTokens * rowCount * logitsBytes,
587
- () => {
357
+ try {
358
+ await runLogitScatter(logitsTensor, logitsBuffer, {
359
+ rowCount: numTokens,
360
+ chunkColumns: rowCount,
361
+ targetColumns: vocabSize,
362
+ columnOffset: rowOffset,
363
+ });
364
+ } finally {
588
365
  releaseBuffer(logitsTensor.buffer);
589
366
  releaseBuffer(weightBuffer.buffer);
590
367
  }
591
- );
592
- const chunkLogits = logitsTensor.dtype === 'f16'
593
- ? f16BufferToF32(chunkLogitsData)
594
- : new Float32Array(chunkLogitsData);
595
- writeChunkLogits(logits, chunkLogits, numTokens, vocabSize, rowOffset, rowCount);
596
- }
368
+ }
597
369
 
598
- if (matmulInput !== normedTensor) {
599
- releaseBuffer(matmulInput.buffer);
370
+ return createTensor(logitsBuffer, 'f32', [numTokens, vocabSize], 'lm_head_chunked_logits');
371
+ } catch (error) {
372
+ releaseBuffer(logitsBuffer);
373
+ throw error;
374
+ } finally {
375
+ if (matmulInput !== normedTensor) {
376
+ releaseBuffer(matmulInput.buffer);
377
+ }
600
378
  }
601
-
602
- return logits;
603
379
  }
604
380
 
605
381
 
606
- export async function computeSplitLogitsGPU(
382
+ export async function computeSplitLogitsTensorGPU(
607
383
  normedTensor,
608
384
  lmHead,
609
385
  numTokens,
@@ -627,7 +403,11 @@ export async function computeSplitLogitsGPU(
627
403
  const lmHeadRole = resolveLmHeadMatmulRole(phase);
628
404
  const lmHeadInputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'inputDtype');
629
405
  const lmHeadOutputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'outputDtype');
630
- const logits = new Float32Array(numTokens * vocabSize);
406
+ const logitsBuffer = acquireBuffer(
407
+ numTokens * vocabSize * Float32Array.BYTES_PER_ELEMENT,
408
+ undefined,
409
+ 'lm_head_split_logits'
410
+ );
631
411
  let matmulInput = normedTensor;
632
412
  let matmulInputOwned = false;
633
413
 
@@ -678,26 +458,47 @@ export async function computeSplitLogitsGPU(
678
458
  });
679
459
  }
680
460
 
681
- const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsTensor.dtype });
682
- const chunkLogitsData = await readBufferWithCleanup(
683
- logitsTensor.buffer,
684
- numTokens * rowCount * logitsBytes,
685
- () => {
686
- releaseBuffer(logitsTensor.buffer);
687
- }
688
- );
689
- const chunkLogits = logitsTensor.dtype === 'f16'
690
- ? f16BufferToF32(chunkLogitsData)
691
- : new Float32Array(chunkLogitsData);
692
- writeChunkLogits(logits, chunkLogits, numTokens, vocabSize, section.rowStart, rowCount);
461
+ try {
462
+ await runLogitScatter(logitsTensor, logitsBuffer, {
463
+ rowCount: numTokens,
464
+ chunkColumns: rowCount,
465
+ targetColumns: vocabSize,
466
+ columnOffset: section.rowStart,
467
+ });
468
+ } finally {
469
+ releaseBuffer(logitsTensor.buffer);
470
+ }
693
471
  }
472
+ return createTensor(logitsBuffer, 'f32', [numTokens, vocabSize], 'lm_head_split_logits');
473
+ } catch (error) {
474
+ releaseBuffer(logitsBuffer);
475
+ throw error;
694
476
  } finally {
695
477
  if (matmulInputOwned) {
696
478
  releaseBuffer(matmulInput.buffer);
697
479
  }
698
480
  }
699
481
 
700
- return logits;
482
+ }
483
+
484
+ export async function computeChunkedLogitsGPU(...args) {
485
+ const tensor = await computeChunkedLogitsTensorGPU(...args);
486
+ const data = await readBufferWithCleanup(
487
+ tensor.buffer,
488
+ tensor.shape[0] * tensor.shape[1] * Float32Array.BYTES_PER_ELEMENT,
489
+ () => releaseBuffer(tensor.buffer)
490
+ );
491
+ return new Float32Array(data);
492
+ }
493
+
494
+ export async function computeSplitLogitsGPU(...args) {
495
+ const tensor = await computeSplitLogitsTensorGPU(...args);
496
+ const data = await readBufferWithCleanup(
497
+ tensor.buffer,
498
+ tensor.shape[0] * tensor.shape[1] * Float32Array.BYTES_PER_ELEMENT,
499
+ () => releaseBuffer(tensor.buffer)
500
+ );
501
+ return new Float32Array(data);
701
502
  }
702
503
 
703
504
 
@@ -708,6 +509,7 @@ export async function computeLogitsGPU(
708
509
  config,
709
510
  debugFlags,
710
511
  operatorDiagnostics = null,
512
+ options = null,
711
513
  ) {
712
514
  const {
713
515
  hiddenSize,
@@ -717,7 +519,7 @@ export async function computeLogitsGPU(
717
519
  embeddingVocabSize,
718
520
  activationDtype,
719
521
  } = config;
720
- const { finalNorm, lmHead } = weights;
522
+ const { finalNorm, lmHead, lmHeadBias = null } = weights;
721
523
  const device = getDevice();
722
524
 
723
525
  if (!device) {
@@ -882,18 +684,20 @@ export async function computeLogitsGPU(
882
684
  ? embeddingVocabSize
883
685
  : vocabSize;
884
686
 
885
- const logitsTensor = await runMatmul(lmHeadInputTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
687
+ let logitsTensor = await runMatmul(lmHeadInputTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
886
688
  transposeB: 'auto',
887
689
  role: lmHeadRole,
888
690
  kernelPath: stableKernelPath,
889
691
  outputDtype: lmHeadOutputDtype,
890
692
  executionPolicies: config.executionPolicies ?? null,
891
693
  });
892
- await runProbes('logits', logitsTensor.buffer, {
694
+ logitsTensor = await finalizeLogitOutputTensor(logitsTensor, config, {
893
695
  numTokens,
894
- hiddenSize: matmulVocabSize,
696
+ vocabSize: matmulVocabSize,
697
+ targetVocabSize: vocabSize,
698
+ bias: lmHeadBias,
699
+ applySoftcap: options?.applySoftcap === true,
895
700
  operatorDiagnostics,
896
- dtype: logitsTensor.dtype,
897
701
  });
898
702
 
899
703
  // Cleanup intermediate buffers (but keep logitsBuffer)
@@ -908,7 +712,7 @@ export async function computeLogitsGPU(
908
712
  if (normWeightBufferOwned) { releaseBuffer(normWeightBuffer); normWeightBufferOwned = false; }
909
713
  if (lmHeadBufferOwned) { releaseBuffer(isWeightBuffer(lmHeadBuffer) ? lmHeadBuffer.buffer : lmHeadBuffer); lmHeadBufferOwned = false; }
910
714
 
911
- return { logitsBuffer: logitsTensor.buffer, vocabSize: matmulVocabSize, logitsDtype: logitsTensor.dtype };
715
+ return { logitsBuffer: logitsTensor.buffer, vocabSize, logitsDtype: logitsTensor.dtype };
912
716
  } finally {
913
717
  if (inputBufferOwned && inputBuffer) releaseBuffer(inputBuffer);
914
718
  if (normInputOwned && normInputTensor) releaseBuffer(normInputTensor.buffer);
@@ -920,270 +724,3 @@ export async function computeLogitsGPU(
920
724
  if (lmHeadBufferOwned && lmHeadBuffer) releaseBuffer(isWeightBuffer(lmHeadBuffer) ? lmHeadBuffer.buffer : lmHeadBuffer);
921
725
  }
922
726
  }
923
-
924
-
925
- async function recordLogitsTailGPU(
926
- recorder,
927
- hiddenStates,
928
- numTokens,
929
- weights,
930
- config,
931
- operatorDiagnostics = null,
932
- ) {
933
- const {
934
- hiddenSize,
935
- vocabSize,
936
- rmsNormEps,
937
- useTiedEmbeddings,
938
- embeddingVocabSize,
939
- activationDtype = 'f32',
940
- } = config;
941
- const { finalNorm, lmHead } = weights;
942
- const matmulVocabSize = useTiedEmbeddings && embeddingVocabSize ? embeddingVocabSize : vocabSize;
943
-
944
- if (!finalNorm || !lmHead) {
945
- throw new Error('[recordLogitsGPU] Final norm or LM head not loaded');
946
- }
947
- if (isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead)) {
948
- throw new Error('[recordLogitsGPU] CPU-resident or split LM head not supported in recorded path');
949
- }
950
-
951
- // Get norm weight buffer
952
-
953
- let normWeightBuffer;
954
- let normWeightOwned = false;
955
- const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, recorder.device.queue, 'final_norm_w');
956
- normWeightBuffer = resolvedFinalNorm.buffer;
957
- normWeightOwned = resolvedFinalNorm.owned;
958
-
959
-
960
- const inputDtype = activationDtype;
961
- // Wrap input buffer as Tensor for RMSNorm
962
- const inputTensor = createTensor(hiddenStates, inputDtype, [numTokens, hiddenSize], 'logits_input');
963
- const phase = numTokens === 1 ? 'decode' : 'prefill';
964
- const kernelPath = config.kernelPath ?? null;
965
- const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
966
- const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
967
- await runProbes('pre_final_norm', hiddenStates, {
968
- numTokens,
969
- hiddenSize,
970
- recorder,
971
- operatorDiagnostics,
972
- dtype: inputDtype,
973
- });
974
- const forceStableF32Logits = !hasExplicitFinalNormPrecision && shouldForceStableF32Logits(config, inputDtype);
975
- const stableKernelPath = forceStableF32Logits
976
- ? createStableF32LogitsKernelPath(kernelPath)
977
- : kernelPath;
978
- let normInputTensor = inputTensor;
979
- let normInputOwned = false;
980
- if (forceStableF32Logits) {
981
- assertImplicitDtypeTransitionAllowed({
982
- executionPolicies: config.executionPolicies ?? null,
983
- fromDtype: inputTensor.dtype,
984
- toDtype: 'f32',
985
- op: 'logits_final_norm',
986
- detail: 'Stable logits mode would widen activations implicitly before final RMSNorm.',
987
- });
988
- normInputTensor = await recordCastF16ToF32(recorder, inputTensor);
989
- normInputOwned = true;
990
- } else {
991
- const finalNormInputDtype = resolvePostLayerStepDtype('final_norm', phase, stableKernelPath, inputTensor.dtype, 'inputDtype');
992
- normInputTensor = finalNormInputDtype !== inputTensor.dtype
993
- ? await coerceTensorDtype(inputTensor, finalNormInputDtype, recorder, {
994
- executionPolicies: config.executionPolicies ?? null,
995
- op: 'final_norm',
996
- transitionDeclaredBy: 'step_precision',
997
- })
998
- : inputTensor;
999
- normInputOwned = normInputTensor !== inputTensor;
1000
- }
1001
- // Record RMSNorm (no submit)
1002
- const normedTensor = await recordRMSNorm(recorder, normInputTensor, normWeightBuffer, rmsNormEps, {
1003
- batchSize: numTokens,
1004
- hiddenSize,
1005
- rmsNormWeightOffset: config.rmsNormWeightOffset,
1006
- label: 'final_norm',
1007
- });
1008
- let finalNormTensor = normedTensor;
1009
- if (!forceStableF32Logits) {
1010
- const finalNormOutputDtype = resolvePostLayerStepDtype(
1011
- 'final_norm',
1012
- phase,
1013
- stableKernelPath,
1014
- normedTensor.dtype,
1015
- 'outputDtype'
1016
- );
1017
- finalNormTensor = finalNormOutputDtype !== normedTensor.dtype
1018
- ? await coerceTensorDtype(normedTensor, finalNormOutputDtype, recorder, {
1019
- executionPolicies: config.executionPolicies ?? null,
1020
- op: 'final_norm',
1021
- transitionDeclaredBy: 'step_precision',
1022
- })
1023
- : normedTensor;
1024
- }
1025
- await runProbes('final_norm', finalNormTensor.buffer, {
1026
- numTokens,
1027
- hiddenSize,
1028
- recorder,
1029
- operatorDiagnostics,
1030
- dtype: finalNormTensor.dtype,
1031
- });
1032
- const logitInputScale = resolveLogitInputScale(config);
1033
- let logitInputTensor = finalNormTensor;
1034
- let logitInputOwned = false;
1035
- if (logitInputScale !== 1) {
1036
- logitInputTensor = await recordScale(recorder, finalNormTensor, logitInputScale, {
1037
- count: numTokens * hiddenSize,
1038
- });
1039
- logitInputOwned = true;
1040
- }
1041
- const lmHeadRole = resolveLmHeadMatmulRole(phase);
1042
- const lmHeadInputDtype = forceStableF32Logits
1043
- ? logitInputTensor.dtype
1044
- : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'inputDtype');
1045
- const lmHeadOutputDtype = forceStableF32Logits
1046
- ? logitInputTensor.dtype
1047
- : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'outputDtype');
1048
- const lmHeadInputTensor = lmHeadInputDtype !== logitInputTensor.dtype
1049
- ? await coerceTensorDtype(logitInputTensor, lmHeadInputDtype, recorder, {
1050
- executionPolicies: config.executionPolicies ?? null,
1051
- op: 'lm_head',
1052
- transitionDeclaredBy: 'step_precision',
1053
- })
1054
- : logitInputTensor;
1055
-
1056
- // Get LM head buffer
1057
-
1058
- let lmHeadBuffer;
1059
- let lmHeadBufferOwned = false;
1060
- if (isGpuBufferInstance(lmHead)) {
1061
- lmHeadBuffer = lmHead;
1062
- } else if (isWeightBuffer(lmHead)) {
1063
- lmHeadBuffer = lmHead;
1064
- } else {
1065
- const rawBuffer = acquireBuffer( (lmHead).byteLength, undefined, 'lm_head_w');
1066
- recorder.device.queue.writeBuffer(rawBuffer, 0, (lmHead));
1067
- lmHeadBuffer = rawBuffer;
1068
- lmHeadBufferOwned = true;
1069
- }
1070
-
1071
- return {
1072
- hiddenSize,
1073
- matmulVocabSize,
1074
- phase,
1075
- stableKernelPath,
1076
- lmHeadRole,
1077
- lmHeadOutputDtype,
1078
- normedTensor,
1079
- finalNormTensor,
1080
- logitInputTensor,
1081
- logitInputOwned,
1082
- normInputTensor,
1083
- normInputOwned,
1084
- normWeightBuffer,
1085
- normWeightOwned,
1086
- lmHeadInputTensor,
1087
- lmHeadBuffer,
1088
- lmHeadBufferOwned,
1089
- };
1090
- }
1091
-
1092
- function trackRecordedLogitsTail(recorder, tail) {
1093
- const trackedTempBuffers = new Set();
1094
- const trackTempBufferOnce = (buffer) => {
1095
- if (!buffer || trackedTempBuffers.has(buffer)) {
1096
- return;
1097
- }
1098
- trackedTempBuffers.add(buffer);
1099
- recorder.trackTemporaryBuffer(buffer);
1100
- };
1101
- if (tail.finalNormTensor !== tail.normedTensor) {
1102
- trackTempBufferOnce(tail.normedTensor.buffer);
1103
- }
1104
- trackTempBufferOnce(tail.finalNormTensor.buffer);
1105
- if (tail.logitInputOwned) {
1106
- trackTempBufferOnce(tail.logitInputTensor.buffer);
1107
- }
1108
- if (tail.lmHeadInputTensor !== tail.logitInputTensor) {
1109
- trackTempBufferOnce(tail.lmHeadInputTensor.buffer);
1110
- }
1111
- if (tail.normWeightOwned) {
1112
- recorder.trackTemporaryBuffer(tail.normWeightBuffer);
1113
- }
1114
- if (tail.normInputOwned) {
1115
- recorder.trackTemporaryBuffer(tail.normInputTensor.buffer);
1116
- }
1117
- if (tail.lmHeadBufferOwned) {
1118
- recorder.trackTemporaryBuffer(isWeightBuffer(tail.lmHeadBuffer) ? tail.lmHeadBuffer.buffer : tail.lmHeadBuffer);
1119
- }
1120
- }
1121
-
1122
- export async function recordLogitsGPU(
1123
- recorder,
1124
- hiddenStates,
1125
- numTokens,
1126
- weights,
1127
- config,
1128
- operatorDiagnostics = null,
1129
- ) {
1130
- const tail = await recordLogitsTailGPU(
1131
- recorder,
1132
- hiddenStates,
1133
- numTokens,
1134
- weights,
1135
- config,
1136
- operatorDiagnostics
1137
- );
1138
-
1139
- const logitsTensor = await recordMatmul(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, numTokens, tail.matmulVocabSize, tail.hiddenSize, {
1140
- transposeB: 'auto',
1141
- role: tail.lmHeadRole,
1142
- kernelPath: tail.stableKernelPath,
1143
- outputDtype: tail.lmHeadOutputDtype,
1144
- executionPolicies: config.executionPolicies ?? null,
1145
- });
1146
- await runProbes('logits', logitsTensor.buffer, {
1147
- numTokens,
1148
- hiddenSize: tail.matmulVocabSize,
1149
- recorder,
1150
- operatorDiagnostics,
1151
- dtype: logitsTensor.dtype,
1152
- });
1153
-
1154
- trackRecordedLogitsTail(recorder, tail);
1155
-
1156
- return { logitsBuffer: logitsTensor.buffer, vocabSize: tail.matmulVocabSize, logitsDtype: logitsTensor.dtype };
1157
- }
1158
-
1159
- export async function recordGreedyLmHeadArgmaxGPU(
1160
- recorder,
1161
- hiddenStates,
1162
- numTokens,
1163
- weights,
1164
- config,
1165
- options,
1166
- operatorDiagnostics = null,
1167
- ) {
1168
- if (numTokens !== 1) {
1169
- throw new Error(`[recordGreedyLmHeadArgmaxGPU] expected numTokens=1, got ${numTokens}.`);
1170
- }
1171
- const tail = await recordLogitsTailGPU(
1172
- recorder,
1173
- hiddenStates,
1174
- numTokens,
1175
- weights,
1176
- config,
1177
- operatorDiagnostics
1178
- );
1179
- const outputBuffer = await recordLmHeadArgmax(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, {
1180
- vocabSize: tail.matmulVocabSize,
1181
- hiddenSize: tail.hiddenSize,
1182
- padTokenId: options.padTokenId,
1183
- logitSoftcap: options.logitSoftcap,
1184
- outputBuffer: options.outputBuffer,
1185
- outputIndex: options.outputIndex,
1186
- });
1187
- trackRecordedLogitsTail(recorder, tail);
1188
- return outputBuffer;
1189
- }