doppler-gpu 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (927) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +99 -86
  6. package/src/capsule-runtime.d.ts +23 -0
  7. package/src/capsule-runtime.js +23 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +1 -0
  24. package/src/client/capsule-host.d.ts +3 -0
  25. package/src/client/capsule-host.js +1 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +36 -1
  28. package/src/client/doppler-api.js +40 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +322 -0
  45. package/src/client/{runtime → model-host}/model-session.js +190 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +68 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +100 -0
  64. package/src/client/runtime/capsule-program-adapter.js +151 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +85 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +74 -0
  72. package/src/client/runtime/composition-root.js +344 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +30 -0
  92. package/src/client/runtime/session-controller.js +170 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +11 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +58 -0
  109. package/src/config/capsule-operations.json +18 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  128. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
  129. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  130. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
  132. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
  134. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
  135. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
  136. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
  137. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
  140. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
  141. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
  142. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  143. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  145. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  146. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
  147. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  148. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  149. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  150. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
  151. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
  152. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  153. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
  154. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
  155. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  157. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
  158. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  160. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
  161. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  162. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  163. package/src/config/dtype-transition-contract.js +40 -0
  164. package/src/config/embedding-contract.js +26 -0
  165. package/src/config/embedding-reference.js +85 -0
  166. package/src/config/evidence/source-intake-policy.json +1 -1
  167. package/src/config/execution-contract-check.d.ts +16 -73
  168. package/src/config/execution-contract-check.js +7 -429
  169. package/src/config/forecast-manifest.js +56 -0
  170. package/src/config/heterogeneous-attention-contract.js +46 -0
  171. package/src/config/initial-execution-identity.d.ts +58 -0
  172. package/src/config/initial-execution-identity.js +331 -0
  173. package/src/config/kernel-path-loader.d.ts +10 -0
  174. package/src/config/kernel-path-loader.js +23 -8
  175. package/src/config/kernel-registry-contract.d.ts +25 -0
  176. package/src/config/kernel-registry-contract.js +46 -0
  177. package/src/config/kernels/kernel-ref-digests.js +50 -2
  178. package/src/config/kernels/registry.js +1 -1
  179. package/src/config/kernels/registry.json +13592 -11377
  180. package/src/config/lora-layouts.d.ts +10 -0
  181. package/src/config/lora-layouts.js +17 -0
  182. package/src/config/lora-layouts.json +9 -0
  183. package/src/config/manifest-inference-validation.js +575 -0
  184. package/src/config/merge/execution-patches.js +12 -0
  185. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  186. package/src/config/merge/value-selection.js +38 -0
  187. package/src/config/merge.js +43 -11
  188. package/src/config/model-ir-v2.d.ts +57 -0
  189. package/src/config/model-ir-v2.js +258 -0
  190. package/src/config/model-ir.d.ts +49 -0
  191. package/src/config/model-ir.js +180 -0
  192. package/src/config/param-validator.js +1 -1
  193. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  194. package/src/config/platforms/loader.js +2 -1
  195. package/src/config/production-release-evidence.d.ts +23 -0
  196. package/src/config/production-release-evidence.js +423 -0
  197. package/src/config/production-release.js +321 -0
  198. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  199. package/src/config/required-inference-fields-contract-check.js +2 -2
  200. package/src/config/rerank-reference.d.ts +28 -0
  201. package/src/config/rerank-reference.js +89 -0
  202. package/src/config/revocation-policy.d.ts +65 -0
  203. package/src/config/revocation-policy.js +290 -0
  204. package/src/config/revocation-registry.json +11 -0
  205. package/src/config/revocation-updates.d.ts +68 -0
  206. package/src/config/revocation-updates.js +440 -0
  207. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  208. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  209. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  210. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  211. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  212. package/src/config/runtime-merge.js +1 -1
  213. package/src/config/runtime.js +1 -1
  214. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  215. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  216. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  217. package/src/config/schema/conversion-report.schema.js +29 -0
  218. package/src/config/schema/converter.schema.d.ts +1 -0
  219. package/src/config/schema/debug.schema.d.ts +1 -0
  220. package/src/config/schema/doppler.schema.js +5 -5
  221. package/src/config/schema/emulation.schema.js +1 -1
  222. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  223. package/src/config/schema/execution-v1.schema.js +19 -0
  224. package/src/config/schema/harness.schema.d.ts +13 -0
  225. package/src/config/schema/harness.schema.js +6 -0
  226. package/src/config/schema/inference.schema.d.ts +21 -2
  227. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  228. package/src/config/schema/manifest.schema.d.ts +26 -7
  229. package/src/config/schema/manifest.schema.js +10 -1
  230. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  231. package/src/config/schema/program-bundle.schema.js +145 -10
  232. package/src/config/sequence-reference.d.ts +26 -0
  233. package/src/config/sequence-reference.js +100 -0
  234. package/src/config/target-plan.d.ts +108 -0
  235. package/src/config/target-plan.js +411 -0
  236. package/src/config/training-defaults.js +1 -1
  237. package/src/config/transforms/capability-transform-resolver.js +4 -105
  238. package/src/config/transforms/execution-graph/dtype.js +591 -0
  239. package/src/config/transforms/execution-graph/fusion.js +436 -0
  240. package/src/config/transforms/execution-graph/validation.js +414 -0
  241. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  242. package/src/converter/artifact-identity.js +475 -0
  243. package/src/converter/conversion-plan.js +1 -1
  244. package/src/converter/core.js +94 -2096
  245. package/src/converter/forge-candidate-evaluation.js +271 -0
  246. package/src/converter/forge-qualification.js +162 -0
  247. package/src/converter/forge-stages.js +798 -0
  248. package/src/converter/manifest-builder.js +683 -0
  249. package/src/converter/quantization-info.js +4 -1
  250. package/src/converter/quantizer.js +58 -9
  251. package/src/converter/shard-packer.d.ts +1 -1
  252. package/src/converter/shard-packer.js +2 -2
  253. package/src/converter/tensor-transform.js +640 -0
  254. package/src/converter/tokenizer-contract.js +243 -0
  255. package/src/converter/tokenizer-manifest.js +243 -0
  256. package/src/debug/history.js +0 -8
  257. package/src/debug/index.js +0 -10
  258. package/src/debug/tensor.js +1 -1
  259. package/src/debug/trace.js +0 -7
  260. package/src/experimental/adapters/adapter-manager.js +2 -0
  261. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  262. package/src/experimental/adapters/adapter-manifest.js +11 -0
  263. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  264. package/src/experimental/adapters/artifact-contract.js +5 -2
  265. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  266. package/src/experimental/adapters/lora-loader.js +110 -48
  267. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  268. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  269. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  270. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  271. package/src/experimental/browser/browser-converter.js +11 -11
  272. package/src/experimental/diffusion/index.d.ts +1 -1
  273. package/src/experimental/diffusion/index.js +1 -1
  274. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  275. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  276. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  277. package/src/experimental/diffusion/vae.js +596 -0
  278. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  279. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  280. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  281. package/src/experimental/distribution/shard-delivery.js +4 -1709
  282. package/src/experimental/energy/index.d.ts +2 -2
  283. package/src/experimental/energy/index.js +2 -2
  284. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  285. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  286. package/src/experimental/logits/cpu-output.d.ts +27 -0
  287. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  288. package/src/experimental/logits/cpu.d.ts +87 -0
  289. package/src/experimental/logits/cpu.js +95 -0
  290. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  291. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  292. package/src/experimental/orchestration/index.d.ts +1 -1
  293. package/src/experimental/orchestration/index.js +1 -1
  294. package/src/experimental/training/artifacts.js +2 -2
  295. package/src/experimental/training/checkpoint.js +2 -2
  296. package/src/experimental/training/distillation/artifacts.js +1 -1
  297. package/src/experimental/training/distillation/dataset.js +2 -2
  298. package/src/experimental/training/distillation/eval.js +1 -1
  299. package/src/experimental/training/distillation/runtime.js +2 -2
  300. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  301. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  302. package/src/experimental/training/distillation/student-fixture.js +2 -469
  303. package/src/experimental/training/lora/checkpoint.js +746 -0
  304. package/src/experimental/training/lora/execution.js +550 -0
  305. package/src/experimental/training/lora/plan.js +387 -0
  306. package/src/experimental/training/lora/recovery.js +418 -0
  307. package/src/experimental/training/lora-pipeline.js +7 -1949
  308. package/src/experimental/training/operator-artifacts.js +2 -2
  309. package/src/experimental/training/operator-command.js +4 -4
  310. package/src/experimental/training/runner/evaluation.js +62 -0
  311. package/src/experimental/training/runner/lifecycle.js +257 -0
  312. package/src/experimental/training/runner.js +5 -268
  313. package/src/experimental/training/suite/execution.js +879 -0
  314. package/src/experimental/training/suite/plan.js +855 -0
  315. package/src/experimental/training/suite.js +5 -1659
  316. package/src/experimental/training/workloads.d.ts +5 -5
  317. package/src/experimental/training/workloads.js +6 -6
  318. package/src/experimental/turboquant/codebook.js +470 -0
  319. package/src/formats/quantization-constants.d.ts +11 -0
  320. package/src/formats/quantization-constants.js +11 -0
  321. package/src/formats/rdrr/integrity.js +1 -1
  322. package/src/formats/rdrr/lowerings.js +0 -29
  323. package/src/formats/rdrr/merkle.js +1 -1
  324. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  325. package/src/formats/rdrr/validation.js +3 -3
  326. package/src/gpu/command-recorder.d.ts +10 -1
  327. package/src/gpu/command-recorder.js +58 -14
  328. package/src/gpu/constant-buffer.js +29 -0
  329. package/src/gpu/device-state.js +36 -0
  330. package/src/gpu/device.js +6 -41
  331. package/src/gpu/kernel-runtime.js +2 -1
  332. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  333. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  334. package/src/gpu/kernels/attention/executors.js +960 -0
  335. package/src/gpu/kernels/attention/plan.js +472 -0
  336. package/src/gpu/kernels/attention.d.ts +4 -0
  337. package/src/gpu/kernels/attention.js +145 -1545
  338. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  339. package/src/gpu/kernels/backward/adam.js +2 -1
  340. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  341. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  342. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  343. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  344. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  345. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  346. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  347. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  348. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  349. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  350. package/src/gpu/kernels/backward/executor.js +158 -0
  351. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  352. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  353. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  354. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  355. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  356. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  357. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  358. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  359. package/src/gpu/kernels/backward/matmul.js +133 -0
  360. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  361. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  362. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  363. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  364. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  365. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  366. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  367. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  368. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  369. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  370. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  371. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  372. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  374. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  376. package/src/gpu/kernels/backward/utils.js +2 -392
  377. package/src/gpu/kernels/bind-group-validation.js +16 -0
  378. package/src/gpu/kernels/cast.js +2 -1
  379. package/src/gpu/kernels/check-finiteness.js +13 -49
  380. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  381. package/src/gpu/kernels/check-stop.js +14 -33
  382. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  383. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  384. package/src/gpu/kernels/check_stop.wgsl +20 -0
  385. package/src/gpu/kernels/clamp.js +1 -1
  386. package/src/gpu/kernels/conv2d.js +1 -1
  387. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  388. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  389. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  390. package/src/gpu/kernels/dequant.js +7 -2
  391. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  392. package/src/gpu/kernels/dispatch.js +12 -1
  393. package/src/gpu/kernels/embedding-pool.js +72 -0
  394. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  395. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  396. package/src/gpu/kernels/energy.js +2 -1
  397. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  398. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  399. package/src/gpu/kernels/fused_ffn.js +10 -220
  400. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  401. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  402. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  403. package/src/gpu/kernels/gated-short-conv.js +11 -69
  404. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  405. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  406. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  407. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  408. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  409. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  410. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  411. package/src/gpu/kernels/gather.d.ts +1 -1
  412. package/src/gpu/kernels/gather.js +30 -6
  413. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  414. package/src/gpu/kernels/gelu.js +1 -1
  415. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  416. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  417. package/src/gpu/kernels/groupnorm.js +1 -1
  418. package/src/gpu/kernels/index.d.ts +38 -5
  419. package/src/gpu/kernels/index.js +18 -4
  420. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  421. package/src/gpu/kernels/kernel-base.js +1 -1
  422. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  423. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  424. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  425. package/src/gpu/kernels/kernel-configs.js +26 -54
  426. package/src/gpu/kernels/kernel-execution.js +141 -0
  427. package/src/gpu/kernels/kv-cache-write.js +4 -4
  428. package/src/gpu/kernels/kv-quantize.js +4 -1
  429. package/src/gpu/kernels/l2-normalize.js +36 -0
  430. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  431. package/src/gpu/kernels/layernorm.d.ts +4 -4
  432. package/src/gpu/kernels/layernorm.js +44 -7
  433. package/src/gpu/kernels/linear-attention-core.js +29 -449
  434. package/src/gpu/kernels/linear_attention.js +1 -1
  435. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  436. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  437. package/src/gpu/kernels/logit-finalize.js +103 -0
  438. package/src/gpu/kernels/logit-merge.js +13 -125
  439. package/src/gpu/kernels/logit-scatter.js +40 -0
  440. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  441. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  442. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  443. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  444. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  445. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  446. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  447. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  448. package/src/gpu/kernels/matmul-selection.js +3 -2
  449. package/src/gpu/kernels/matmul.js +4 -15
  450. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  451. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  452. package/src/gpu/kernels/modulate.js +1 -1
  453. package/src/gpu/kernels/moe.js +3 -1
  454. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  455. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  456. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  457. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  458. package/src/gpu/kernels/patch_context.wgsl +29 -0
  459. package/src/gpu/kernels/pipeline-cache.js +3 -1
  460. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  461. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  462. package/src/gpu/kernels/relu.js +1 -1
  463. package/src/gpu/kernels/repeat_channels.js +1 -1
  464. package/src/gpu/kernels/residual.js +2 -2
  465. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  466. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  467. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  468. package/src/gpu/kernels/rmsnorm.js +1 -1
  469. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  470. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  471. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  472. package/src/gpu/kernels/rope-precompute.js +160 -0
  473. package/src/gpu/kernels/rope-qk.js +1 -1
  474. package/src/gpu/kernels/rope.js +2 -2
  475. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  476. package/src/gpu/kernels/sample.js +5 -1
  477. package/src/gpu/kernels/scale.js +1 -1
  478. package/src/gpu/kernels/shader-cache.js +8 -4
  479. package/src/gpu/kernels/shader-source-scope.js +68 -0
  480. package/src/gpu/kernels/silu.js +2 -1
  481. package/src/gpu/kernels/soft-embedding.js +3 -1
  482. package/src/gpu/kernels/softmax.js +4 -2
  483. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  484. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  485. package/src/gpu/kernels/split_qg.js +1 -1
  486. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  487. package/src/gpu/kernels/split_qkv.js +18 -3
  488. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  489. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  490. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  491. package/src/gpu/kernels/transpose.js +1 -1
  492. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  493. package/src/gpu/kernels/uniform-utils.js +7 -2
  494. package/src/gpu/kernels/upsample2d.js +1 -1
  495. package/src/gpu/kernels/utils.js +9 -330
  496. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  497. package/src/gpu/kernels/vision-average-pool.js +68 -0
  498. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  499. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  500. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  501. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  502. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  503. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  504. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  505. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  506. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  507. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  508. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  509. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  510. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  511. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  512. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  513. package/src/gpu/readback-buffer.js +20 -0
  514. package/src/gpu/submit-probe.js +2 -26
  515. package/src/gpu/weight-buffer.d.ts +1 -0
  516. package/src/gpu/weight-buffer.js +7 -0
  517. package/src/index.d.ts +10 -0
  518. package/src/index.js +1 -0
  519. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  520. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  521. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  522. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  523. package/src/inference/browser-harness/inference-execution.js +543 -0
  524. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  525. package/src/inference/browser-harness/model-resolution.js +137 -0
  526. package/src/inference/browser-harness/report.js +338 -0
  527. package/src/inference/browser-harness/request.js +263 -0
  528. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  529. package/src/inference/browser-harness/runtime-config.js +255 -0
  530. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  531. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  532. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  533. package/src/inference/browser-harness/runtime-report.js +117 -0
  534. package/src/inference/browser-harness/suite-summary.js +39 -0
  535. package/src/inference/browser-harness/text-evidence.js +694 -0
  536. package/src/inference/browser-harness/text-execution.js +757 -0
  537. package/src/inference/browser-harness/text-input.js +725 -0
  538. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  539. package/src/inference/browser-harness-contract-helpers.js +1 -1
  540. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  541. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  542. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  543. package/src/inference/browser-harness-text-helpers.js +31 -2199
  544. package/src/inference/browser-harness.d.ts +15 -2
  545. package/src/inference/browser-harness.js +38 -1952
  546. package/src/inference/kv-cache/base.js +67 -381
  547. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  548. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  549. package/src/inference/kv-cache/quantized.d.ts +2 -2
  550. package/src/inference/kv-cache/quantized.js +4 -8
  551. package/src/inference/kv-cache/update.js +309 -0
  552. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  553. package/src/inference/model-checkpoint-evidence.js +264 -0
  554. package/src/inference/moe-router.js +36 -216
  555. package/src/inference/multi-model-network.js +5 -341
  556. package/src/inference/pipelines/audio/index.js +1 -20
  557. package/src/inference/pipelines/audio/mel.js +0 -38
  558. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  559. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  560. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  561. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  562. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  563. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  564. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  565. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  566. package/src/inference/pipelines/factory.js +14 -3
  567. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  568. package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
  569. package/src/inference/pipelines/text/abort-contract.js +22 -0
  570. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  571. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  572. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  573. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  574. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  575. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  576. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  577. package/src/inference/pipelines/text/attention/plan.js +17 -0
  578. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  579. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  580. package/src/inference/pipelines/text/attention/projections.js +20 -537
  581. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  582. package/src/inference/pipelines/text/attention/record.js +0 -6
  583. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  584. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  585. package/src/inference/pipelines/text/attention/run.js +0 -6
  586. package/src/inference/pipelines/text/attention/types.js +3 -31
  587. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  588. package/src/inference/pipelines/text/chat-format.js +42 -0
  589. package/src/inference/pipelines/text/config/normalization.js +729 -0
  590. package/src/inference/pipelines/text/config/validation.js +9 -0
  591. package/src/inference/pipelines/text/config.d.ts +12 -1
  592. package/src/inference/pipelines/text/config.js +14 -1228
  593. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  594. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  595. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  596. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  597. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  598. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  599. package/src/inference/pipelines/text/embed.js +178 -241
  600. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  601. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  602. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  603. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  604. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  605. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  606. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  607. package/src/inference/pipelines/text/execution-v1.js +7 -283
  608. package/src/inference/pipelines/text/execution.js +517 -0
  609. package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
  610. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  611. package/src/inference/pipelines/text/ffn/dense.js +4 -968
  612. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  613. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  614. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  615. package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
  616. package/src/inference/pipelines/text/generator/decode.js +952 -0
  617. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  618. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  619. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  620. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  621. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  622. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  623. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  624. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  625. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  626. package/src/inference/pipelines/text/generator/text.js +448 -0
  627. package/src/inference/pipelines/text/generator/training.js +752 -0
  628. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  629. package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
  630. package/src/inference/pipelines/text/generator-runtime.js +5 -135
  631. package/src/inference/pipelines/text/generator-steps.js +30 -972
  632. package/src/inference/pipelines/text/generator.js +165 -3544
  633. package/src/inference/pipelines/text/image-transcription.js +304 -0
  634. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  635. package/src/inference/pipelines/text/init-rope.js +37 -189
  636. package/src/inference/pipelines/text/init.d.ts +8 -3
  637. package/src/inference/pipelines/text/init.js +6 -216
  638. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  639. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  640. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  641. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  642. package/src/inference/pipelines/text/layer-plan.js +45 -3
  643. package/src/inference/pipelines/text/layer.js +52 -522
  644. package/src/inference/pipelines/text/lifecycle.js +839 -0
  645. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  646. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  647. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  648. package/src/inference/pipelines/text/linear-attention.js +7 -537
  649. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  650. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  651. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  652. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  653. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  654. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  655. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  656. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  657. package/src/inference/pipelines/text/logits/index.js +62 -102
  658. package/src/inference/pipelines/text/logits/output-transform.js +47 -0
  659. package/src/inference/pipelines/text/logits/plan.js +184 -0
  660. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  661. package/src/inference/pipelines/text/logits/readback.js +9 -0
  662. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  663. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  664. package/src/inference/pipelines/text/lora-apply.js +12 -4
  665. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  666. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  667. package/src/inference/pipelines/text/model-load.js +0 -20
  668. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  669. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  670. package/src/inference/pipelines/text/moe/plan.js +47 -0
  671. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  672. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  673. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  674. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  675. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  676. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  677. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  678. package/src/inference/pipelines/text/probes.js +20 -0
  679. package/src/inference/pipelines/text/state.d.ts +8 -0
  680. package/src/inference/pipelines/text/state.js +1 -0
  681. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  682. package/src/inference/pipelines/text/types.d.ts +4 -0
  683. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  684. package/src/inference/pipelines/text.d.ts +2 -2
  685. package/src/inference/pipelines/text.js +46 -1633
  686. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  687. package/src/inference/pipelines/video/index.js +0 -21
  688. package/src/inference/pipelines/vision/attention.js +59 -0
  689. package/src/inference/pipelines/vision/encoder.js +254 -369
  690. package/src/inference/pipelines/vision/gemma4.js +204 -318
  691. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  692. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  693. package/src/inference/pipelines/vision/glmocr.js +557 -0
  694. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  695. package/src/inference/pipelines/vision/index.js +8 -27
  696. package/src/inference/pipelines/vision/ops.js +11 -33
  697. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  698. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  699. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  700. package/src/inference/refactor-receipt.js +2 -2
  701. package/src/inference/rerank.js +182 -0
  702. package/src/inference/tokenizer.js +1 -1
  703. package/src/inference/tokenizers/behavior-flags.js +6 -3
  704. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  705. package/src/inference/tokenizers/bundled/registry.js +216 -0
  706. package/src/inference/tokenizers/bundled.js +23 -353
  707. package/src/loader/doppler-loader.d.ts +4 -0
  708. package/src/loader/doppler-loader.js +119 -930
  709. package/src/loader/embedding-limit-preflight.js +11 -4
  710. package/src/loader/embedding-loader.js +1 -3
  711. package/src/loader/gpu-tensor-materialization.js +20 -0
  712. package/src/loader/layer-loader.js +15 -119
  713. package/src/loader/layer-tensor-aliases.js +99 -0
  714. package/src/loader/layer-weight-shape.js +29 -0
  715. package/src/loader/load-timing.js +64 -0
  716. package/src/loader/loader-types.d.ts +3 -0
  717. package/src/loader/model-executor.js +819 -0
  718. package/src/loader/model-load-validation.js +15 -0
  719. package/src/loader/quantization-constants.d.ts +11 -29
  720. package/src/loader/quantization-constants.js +1 -2
  721. package/src/loader/shard-resolver.js +0 -14
  722. package/src/loader/tensor-load-error.js +14 -0
  723. package/src/loader/tensor-shard-indices.js +8 -0
  724. package/src/loader/tensors/bf16-loader.js +125 -0
  725. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  726. package/src/loader/tensors/load-executor.js +577 -0
  727. package/src/loader/tensors/source-transform/executor.js +123 -0
  728. package/src/loader/tensors/source-transform/plan.js +107 -0
  729. package/src/loader/tensors/source-transform.js +15 -264
  730. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  731. package/src/loader/tensors/tensor-loader.js +7 -653
  732. package/src/memory/buffer-pool.js +7 -29
  733. package/src/rules/capability-transform-resolver.js +80 -0
  734. package/src/rules/execution-contract.d.ts +73 -0
  735. package/src/rules/execution-contract.js +429 -0
  736. package/src/rules/execution-rules-contract-check.js +1 -1
  737. package/src/rules/generated/rule-bundle.json +4305 -0
  738. package/src/rules/inference/capability-transforms.rules.json +3 -3
  739. package/src/rules/layer-pattern-contract-check.js +1 -1
  740. package/src/rules/rule-registry.js +70 -57
  741. package/src/storage/artifact-storage-context.js +2 -2
  742. package/src/storage/backend-selection.js +43 -0
  743. package/src/storage/distribution-transport.js +5 -0
  744. package/src/storage/download/integrity.d.ts +33 -0
  745. package/src/storage/download/integrity.js +130 -0
  746. package/src/storage/download/resume-inspection.d.ts +22 -0
  747. package/src/storage/download/resume-inspection.js +134 -0
  748. package/src/storage/download/retry.js +57 -0
  749. package/src/storage/download/state.js +131 -0
  750. package/src/storage/download/transport.js +99 -0
  751. package/src/storage/downloader.js +74 -453
  752. package/src/storage/model-cache.d.ts +1 -0
  753. package/src/storage/model-cache.js +51 -11
  754. package/src/storage/shard-manager.d.ts +5 -0
  755. package/src/storage/shard-manager.js +50 -369
  756. package/src/storage/shards/index.js +60 -0
  757. package/src/storage/shards/integrity.js +225 -0
  758. package/src/storage/shards/lifecycle.js +22 -0
  759. package/src/storage/source-artifact-store.js +14 -4
  760. package/src/tooling/boundary-evidence.d.ts +4 -4
  761. package/src/tooling/boundary-evidence.js +20 -20
  762. package/src/tooling/browser-command-runner.js +13 -6
  763. package/src/tooling/capsule-v2.d.ts +11 -0
  764. package/src/tooling/capsule-v2.js +58 -0
  765. package/src/tooling/capsule.js +20 -0
  766. package/src/tooling/command-api-constants.js +1 -1
  767. package/src/tooling/command-api-family-normalizers.js +163 -5
  768. package/src/tooling/command-api.d.ts +42 -3
  769. package/src/tooling/command-api.js +7 -2
  770. package/src/tooling/command-envelope.js +1 -1
  771. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  772. package/src/tooling/command-runner.html +49 -1
  773. package/src/tooling/command-validation.js +1 -24
  774. package/src/tooling/conversion-run-timing.js +24 -0
  775. package/src/tooling/diagnose-runner.js +11 -9
  776. package/src/tooling/distribution-shard-transport.js +11 -0
  777. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  778. package/src/tooling/execution-cost-ledger.js +197 -15
  779. package/src/tooling/hf-registry/catalog-io.js +63 -0
  780. package/src/tooling/litert/package.js +863 -0
  781. package/src/tooling/litert-package-runtime.js +3 -849
  782. package/src/tooling/model-capsule-forge.js +478 -0
  783. package/src/tooling/node-browser/execution.d.ts +40 -0
  784. package/src/tooling/node-browser/execution.js +308 -0
  785. package/src/tooling/node-browser/static-server.js +188 -0
  786. package/src/tooling/node-browser/transport.d.ts +80 -0
  787. package/src/tooling/node-browser/transport.js +622 -0
  788. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  789. package/src/tooling/node-browser-command-runner.js +10 -1112
  790. package/src/tooling/node-capsule-artifact-store.js +44 -0
  791. package/src/tooling/node-command-execution.js +237 -0
  792. package/src/tooling/node-command-runner.js +26 -205
  793. package/src/tooling/node-converter/input.js +521 -0
  794. package/src/tooling/node-converter/output.js +244 -0
  795. package/src/tooling/node-converter-chunk-batches.js +21 -0
  796. package/src/tooling/node-converter.js +7 -720
  797. package/src/tooling/node-source-runtime.js +16 -276
  798. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  799. package/src/tooling/node-webgpu.js +178 -453
  800. package/src/tooling/opfs-cache.d.ts +5 -0
  801. package/src/tooling/opfs-cache.js +4 -0
  802. package/src/tooling/production-release.d.ts +19 -0
  803. package/src/tooling/production-release.js +647 -0
  804. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  805. package/src/tooling/program-bundle/materialize.js +324 -0
  806. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  807. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  808. package/src/tooling/program-bundle/validation.js +31 -0
  809. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  810. package/src/tooling/program-bundle-parity.d.ts +11 -4
  811. package/src/tooling/program-bundle-parity.js +262 -109
  812. package/src/tooling/program-bundle.d.ts +7 -0
  813. package/src/tooling/program-bundle.js +186 -591
  814. package/src/tooling/provider-v1-contract.d.ts +61 -0
  815. package/src/tooling/provider-v1-contract.js +192 -0
  816. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  817. package/src/tooling/registered-variant-calibration-job.js +210 -0
  818. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  819. package/src/tooling/registered-variant-calibration.js +57 -7
  820. package/src/tooling/runtime-optimization/candidates.js +664 -0
  821. package/src/tooling/runtime-optimization-campaign.js +166 -0
  822. package/src/tooling/runtime-optimization-index.js +16 -1
  823. package/src/tooling/runtime-optimization.d.ts +44 -0
  824. package/src/tooling/runtime-optimization.js +13 -656
  825. package/src/tooling/source-intake.d.ts +1 -1
  826. package/src/tooling/source-intake.js +34 -17
  827. package/src/tooling/source-package-profiles.js +1 -1
  828. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  829. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  830. package/src/tooling/source-runtime/node-loader.js +258 -0
  831. package/src/tooling/source-runtime-bundle.js +26 -411
  832. package/src/tooling/source-runtime-converter-config.js +1 -1
  833. package/src/tooling-experimental-exports.js +0 -5
  834. package/src/tooling-exports/evidence.d.ts +14 -3
  835. package/src/tooling-exports/evidence.js +14 -3
  836. package/src/tooling-exports/storage.d.ts +5 -0
  837. package/src/tooling-exports/storage.js +4 -0
  838. package/src/tooling-exports.d.ts +3 -0
  839. package/src/tooling-exports.js +2 -0
  840. package/src/tooling-exports.shared.d.ts +17 -3
  841. package/src/tooling-exports.shared.js +9 -3
  842. package/src/training.d.ts +6 -6
  843. package/src/training.js +4 -4
  844. package/src/version.js +1 -1
  845. package/tests/kernels/browser/kernel-suite.js +512 -0
  846. package/tests/kernels/browser/test-page.js +169 -14
  847. package/tests/kernels/harness/tolerance.js +1 -0
  848. package/tests/kernels/reference/index.js +13 -1
  849. package/tests/kernels/reference/vision.js +160 -0
  850. package/src/converter/quantizer.d.ts +0 -121
  851. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  852. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  853. package/src/gpu/kernels/utils.d.ts +0 -147
  854. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  855. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  856. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  857. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  858. package/src/rules/converter/execution.rules.json +0 -20
  859. package/src/rules/converter/tensor-roles.rules.json +0 -13
  860. package/src/rules/converter/tokenizer.rules.json +0 -12
  861. package/src/rules/inference/attention.rules.json +0 -69
  862. package/src/rules/inference/config.rules.json +0 -82
  863. package/src/rules/inference/dtype.rules.json +0 -113
  864. package/src/rules/inference/execution.rules.json +0 -195
  865. package/src/rules/inference/ffn.rules.json +0 -123
  866. package/src/rules/inference/layer-pattern.rules.json +0 -16
  867. package/src/rules/inference/layer.rules.json +0 -7
  868. package/src/rules/inference/moe.rules.json +0 -145
  869. package/src/rules/kernels/attention.rules.json +0 -68
  870. package/src/rules/kernels/conv2d.rules.json +0 -6
  871. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  872. package/src/rules/kernels/dequant.rules.json +0 -58
  873. package/src/rules/kernels/energy.rules.json +0 -22
  874. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  875. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  876. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  877. package/src/rules/kernels/gather.rules.json +0 -93
  878. package/src/rules/kernels/gelu.rules.json +0 -11
  879. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  880. package/src/rules/kernels/groupnorm.rules.json +0 -10
  881. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  882. package/src/rules/kernels/layernorm.rules.json +0 -6
  883. package/src/rules/kernels/linear-attention.rules.json +0 -6
  884. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  885. package/src/rules/kernels/matmul.rules.json +0 -95
  886. package/src/rules/kernels/modulate.rules.json +0 -6
  887. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  888. package/src/rules/kernels/moe.rules.json +0 -14
  889. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  890. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  891. package/src/rules/kernels/relu.rules.json +0 -6
  892. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  893. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  894. package/src/rules/kernels/residual.rules.json +0 -12
  895. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  896. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  897. package/src/rules/kernels/rope-qk.rules.json +0 -6
  898. package/src/rules/kernels/rope.rules.json +0 -6
  899. package/src/rules/kernels/sample.rules.json +0 -6
  900. package/src/rules/kernels/scale.rules.json +0 -8
  901. package/src/rules/kernels/silu.rules.json +0 -22
  902. package/src/rules/kernels/softmax.rules.json +0 -25
  903. package/src/rules/kernels/split-qg.rules.json +0 -6
  904. package/src/rules/kernels/split-qkv.rules.json +0 -6
  905. package/src/rules/kernels/upsample2d.rules.json +0 -6
  906. package/src/rules/loader/tensor-loader.rules.json +0 -42
  907. package/src/rules/loader/weights.rules.json +0 -42
  908. package/src/rules/tooling/command-runtime.rules.json +0 -65
  909. package/src/tooling/hf-registry-utils.js +0 -395
  910. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  911. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  912. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  913. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  914. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  915. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  916. /package/src/{utils → formats}/array-buffer.js +0 -0
  917. /package/src/{utils → formats}/canonical-hash.js +0 -0
  918. /package/src/{utils → formats}/clone-json.js +0 -0
  919. /package/src/{utils → formats}/encode-utf8.js +0 -0
  920. /package/src/{utils → formats}/load-json.js +0 -0
  921. /package/src/{utils → formats}/plain-object.js +0 -0
  922. /package/src/{utils → formats}/sha256.js +0 -0
  923. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  924. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  925. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  926. /package/src/{utils → storage}/runtime-env.js +0 -0
  927. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -1,90 +1,57 @@
1
-
2
-
3
1
  import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
4
2
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
5
- import { runMatmul, runRMSNorm, runScale, recordScale } from '../../../../gpu/kernel-selector.js';
6
- import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
7
- import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
8
- import { recordLmHeadArgmax } from '../../../../gpu/kernels/lm-head-argmax.js';
3
+ import { runMatmul, runRMSNorm, runScale } from '../../../../gpu/kernel-selector.js';
9
4
  import { createTensor } from '../../../../gpu/tensor.js';
5
+ import { castF16ToF32, castF32ToF16 } from '../../../../gpu/kernels/cast.js';
10
6
  import {
11
- castF16ToF32,
12
- castF32ToF16,
13
- recordCastF16ToF32,
14
- recordCastF32ToF16,
15
- } from '../../../../gpu/kernels/cast.js';
16
- import { createWeightBuffer, createSplitWeightBuffer, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../../gpu/weight-buffer.js';
7
+ createWeightBuffer,
8
+ createSplitWeightBuffer,
9
+ isWeightBuffer,
10
+ isCpuWeightBuffer,
11
+ isGpuBufferInstance,
12
+ isSplitWeightBuffer,
13
+ } from '../../../../gpu/weight-buffer.js';
17
14
  import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
18
15
  import { getRuntimeConfig } from '../../../../config/runtime.js';
19
16
  import { getKernelPathMatmulPrecision, getKernelPathStepPrecision } from '../../../../config/kernel-path-loader.js';
20
17
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
21
18
  import { runProbes } from '../probes.js';
22
19
  import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
23
- import { f16BufferToF32 } from './cpu.js';
24
- import { readBufferWithCleanup, resolveLogitInputScale } from './utils.js';
25
- import { f16ToF32 } from '../../../../loader/dtype-utils.js';
20
+ import { readBufferWithCleanup } from './readback.js';
21
+ import { resolveLogitInputScale } from './scale-policy.js';
22
+ import { finalizeLogitOutputTensor } from './output-transform.js';
23
+ import { runLogitScatter } from '../../../../gpu/kernels/logit-scatter.js';
26
24
  import { shouldForceStableF32Logits, createStableF32LogitsKernelPath } from './precision-policy.js';
25
+ import {
26
+ extractLmHeadChunk,
27
+ isRangeBackedCpuWeightSource,
28
+ normalizeRangeBytes,
29
+ resolveCpuWeightDims,
30
+ shouldMaterializeSplitLmHeadGPU,
31
+ } from './plan.js';
32
+ import {
33
+ coerceTensorDtype,
34
+ recordGreedyLmHeadArgmaxGPU,
35
+ recordLogitsGPU,
36
+ resolveFinalNormGpuBuffer,
37
+ resolveLmHeadMatmulRole,
38
+ resolveMatmulStepDtype,
39
+ resolvePostLayerStepDtype,
40
+ } from './gpu-executor.js';
41
+
42
+ export {
43
+ extractLmHeadChunk,
44
+ resolveCpuWeightDims,
45
+ shouldMaterializeSplitLmHeadGPU,
46
+ writeChunkLogits,
47
+ } from './plan.js';
48
+ export {
49
+ recordGreedyLmHeadArgmaxGPU,
50
+ recordLogitsGPU,
51
+ } from './gpu-executor.js';
27
52
 
28
- function resolvePrecisionFieldDtype(precision, fallback, field) {
29
- const requested = precision?.[field] ?? fallback;
30
- if (requested == null) {
31
- return fallback;
32
- }
33
- return selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: requested });
34
- }
35
-
36
- function resolveMatmulStepDtype(role, phase, kernelPath, fallback, field) {
37
- const precision = getKernelPathMatmulPrecision(role, phase, 0, kernelPath);
38
- return resolvePrecisionFieldDtype(precision, fallback, field);
39
- }
40
-
41
- function resolvePostLayerStepDtype(op, phase, kernelPath, fallback, field) {
42
- const precision = getKernelPathStepPrecision(op, 'postLayer', phase, 0, kernelPath);
43
- return resolvePrecisionFieldDtype(precision, fallback, field);
44
- }
45
-
46
- function resolveLmHeadMatmulRole(phase) {
47
- return phase === 'prefill' ? 'lm_head_prefill' : 'lm_head';
48
- }
49
-
50
- async function coerceTensorDtype(tensor, targetDtype, recorder = null, options = {}) {
51
- if (!targetDtype || tensor.dtype === targetDtype) {
52
- return tensor;
53
- }
54
- assertImplicitDtypeTransitionAllowed({
55
- executionPolicies: options.executionPolicies ?? null,
56
- fromDtype: tensor.dtype,
57
- toDtype: targetDtype,
58
- op: options.op ?? 'logits',
59
- detail: 'The execution graph must declare this cast explicitly.',
60
- transitionDeclaredBy: options.transitionDeclaredBy ?? null,
61
- });
62
- if (tensor.dtype === 'f32' && targetDtype === 'f16') {
63
- return recorder ? await recordCastF32ToF16(recorder, tensor) : await castF32ToF16(tensor);
64
- }
65
- if (tensor.dtype === 'f16' && targetDtype === 'f32') {
66
- return recorder ? await recordCastF16ToF32(recorder, tensor) : await castF16ToF32(tensor);
67
- }
68
- throw new Error(`Unsupported logits matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
69
- }
70
-
71
- const bf16ScratchU32 = new Uint32Array(1);
72
- const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
73
53
  const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
74
54
 
75
- function bf16ToF32(value) {
76
- bf16ScratchU32[0] = (value & 0xffff) << 16;
77
- return bf16ScratchF32[0];
78
- }
79
-
80
- function isRangeBackedCpuWeightSource(value) {
81
- return (
82
- typeof value === 'object'
83
- && value !== null
84
- && value.kind === 'tensor_range_source'
85
- && typeof value.loadRange === 'function'
86
- );
87
- }
88
55
 
89
56
  function alignByteLength(byteLength) {
90
57
  return Math.ceil(byteLength / 4) * 4;
@@ -97,116 +64,6 @@ function writeBufferInChunks(queue, buffer, bytes) {
97
64
  }
98
65
  }
99
66
 
100
- function resolveFinalNormGpuBuffer(finalNorm, queue, label) {
101
- if (isWeightBuffer(finalNorm)) {
102
- return { buffer: finalNorm.buffer, owned: false };
103
- }
104
- if (isGpuBufferInstance(finalNorm)) {
105
- return { buffer: finalNorm, owned: false };
106
- }
107
- if (!ArrayBuffer.isView(finalNorm)) {
108
- throw new Error('[Logits] final_norm must be a GPU buffer, typed array, or WeightBuffer.');
109
- }
110
- const buffer = acquireBuffer(finalNorm.byteLength, undefined, label);
111
- try {
112
- queue.writeBuffer(buffer, 0, finalNorm);
113
- return { buffer, owned: true };
114
- } catch (error) {
115
- releaseBuffer(buffer);
116
- throw error;
117
- }
118
- }
119
-
120
- function normalizeRangeBytes(value, label) {
121
- if (value instanceof Uint8Array) return value;
122
- if (value instanceof ArrayBuffer) return new Uint8Array(value);
123
- if (ArrayBuffer.isView(value)) {
124
- return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
125
- }
126
- throw new Error(
127
- `[Logits] ${label} returned unsupported byte payload type "${value?.constructor?.name ?? typeof value}".`
128
- );
129
- }
130
-
131
- function decodeChunkIntoOutput(bytes, sourceDtype, output, dstOffset, valueCount) {
132
- if (sourceDtype === 'f16') {
133
- const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
134
- for (let index = 0; index < valueCount; index += 1) {
135
- output[dstOffset + index] = f16ToF32(values[index]);
136
- }
137
- return;
138
- }
139
- if (sourceDtype === 'bf16') {
140
- const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
141
- for (let index = 0; index < valueCount; index += 1) {
142
- output[dstOffset + index] = bf16ToF32(values[index]);
143
- }
144
- return;
145
- }
146
- if (((bytes.byteOffset % 4) === 0) && ((bytes.byteLength % 4) === 0)) {
147
- output.set(new Float32Array(bytes.buffer, bytes.byteOffset, valueCount), dstOffset);
148
- return;
149
- }
150
- const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
151
- for (let index = 0; index < valueCount; index += 1) {
152
- output[dstOffset + index] = view.getFloat32(index * 4, true);
153
- }
154
- }
155
-
156
- function readTypedLmHeadChunk(data, layout, hiddenSize, vocabSize, rowOffset, rowCount, sourceDtype) {
157
- if (data instanceof Float32Array) {
158
- if (layout === 'row') {
159
- const start = rowOffset * hiddenSize;
160
- return data.subarray(start, start + rowCount * hiddenSize);
161
- }
162
- const chunk = new Float32Array(hiddenSize * rowCount);
163
- for (let k = 0; k < hiddenSize; k++) {
164
- const srcOffset = k * vocabSize + rowOffset;
165
- const dstOffset = k * rowCount;
166
- chunk.set(data.subarray(srcOffset, srcOffset + rowCount), dstOffset);
167
- }
168
- return chunk;
169
- }
170
-
171
- if (!(data instanceof Uint16Array)) {
172
- throw new Error(
173
- `[Logits] Unsupported CPU LM head chunk source type "${data?.constructor?.name ?? typeof data}".`
174
- );
175
- }
176
-
177
- const chunk = new Float32Array(hiddenSize * rowCount);
178
- if (layout === 'row') {
179
- const start = rowOffset * hiddenSize;
180
- for (let index = 0; index < rowCount * hiddenSize; index += 1) {
181
- const raw = data[start + index];
182
- chunk[index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
183
- }
184
- return chunk;
185
- }
186
-
187
- for (let k = 0; k < hiddenSize; k += 1) {
188
- const srcOffset = k * vocabSize + rowOffset;
189
- const dstOffset = k * rowCount;
190
- for (let index = 0; index < rowCount; index += 1) {
191
- const raw = data[srcOffset + index];
192
- chunk[dstOffset + index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
193
- }
194
- }
195
- return chunk;
196
- }
197
-
198
-
199
- export function resolveCpuWeightDims(lmHead) {
200
- if (lmHead.shape.length !== 2) {
201
- throw new Error(`[Logits] CPU LM head shape must be 2D, got [${lmHead.shape.join(', ')}]`);
202
- }
203
- if (lmHead.layout === 'column') {
204
- return { hiddenSize: lmHead.shape[0], vocabSize: lmHead.shape[1] };
205
- }
206
- return { vocabSize: lmHead.shape[0], hiddenSize: lmHead.shape[1] };
207
- }
208
-
209
-
210
67
  export function resolveLmHeadChunkRows(
211
68
  device,
212
69
  numTokens,
@@ -239,86 +96,6 @@ export function resolveLmHeadChunkRows(
239
96
  }
240
97
 
241
98
 
242
- export async function extractLmHeadChunk(
243
- data,
244
- layout,
245
- hiddenSize,
246
- vocabSize,
247
- rowOffset,
248
- rowCount,
249
- sourceDtype
250
- ) {
251
- if (typeof sourceDtype !== 'string' || sourceDtype.trim() === '') {
252
- throw new Error('[Logits] CPU LM head source dtype is required.');
253
- }
254
- const normalizedSourceDtype = sourceDtype.trim().toLowerCase();
255
- if (normalizedSourceDtype !== 'f32' && normalizedSourceDtype !== 'f16' && normalizedSourceDtype !== 'bf16') {
256
- throw new Error(`[Logits] Unsupported CPU LM head source dtype "${sourceDtype}".`);
257
- }
258
- if (!isRangeBackedCpuWeightSource(data)) {
259
- return readTypedLmHeadChunk(
260
- data,
261
- layout,
262
- hiddenSize,
263
- vocabSize,
264
- rowOffset,
265
- rowCount,
266
- normalizedSourceDtype
267
- );
268
- }
269
-
270
- const bytesPerElement = normalizedSourceDtype === 'f32' ? 4 : 2;
271
- const chunk = new Float32Array(hiddenSize * rowCount);
272
- if (layout === 'row') {
273
- const byteOffset = rowOffset * hiddenSize * bytesPerElement;
274
- const byteLength = rowCount * hiddenSize * bytesPerElement;
275
- const bytes = normalizeRangeBytes(
276
- await data.loadRange(byteOffset, byteLength),
277
- 'CPU LM head range source'
278
- );
279
- if (bytes.byteLength !== byteLength) {
280
- throw new Error(
281
- `[Logits] CPU LM head range source returned ${bytes.byteLength} bytes, expected ${byteLength}.`
282
- );
283
- }
284
- decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, 0, rowCount * hiddenSize);
285
- return chunk;
286
- }
287
-
288
- for (let k = 0; k < hiddenSize; k += 1) {
289
- const byteOffset = (k * vocabSize + rowOffset) * bytesPerElement;
290
- const byteLength = rowCount * bytesPerElement;
291
- const bytes = normalizeRangeBytes(
292
- await data.loadRange(byteOffset, byteLength),
293
- `CPU LM head range source column ${k}`
294
- );
295
- if (bytes.byteLength !== byteLength) {
296
- throw new Error(
297
- `[Logits] CPU LM head range source returned ${bytes.byteLength} bytes for column ${k}, expected ${byteLength}.`
298
- );
299
- }
300
- decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, k * rowCount, rowCount);
301
- }
302
- return chunk;
303
- }
304
-
305
-
306
- export function writeChunkLogits(
307
- target,
308
- chunk,
309
- numTokens,
310
- vocabSize,
311
- rowOffset,
312
- rowCount
313
- ) {
314
- for (let t = 0; t < numTokens; t++) {
315
- const srcOffset = t * rowCount;
316
- const dstOffset = t * vocabSize + rowOffset;
317
- target.set(chunk.subarray(srcOffset, srcOffset + rowCount), dstOffset);
318
- }
319
- }
320
-
321
-
322
99
  function resolveSplitLmHeadRows(device, hiddenSize, largeWeightConfig) {
323
100
  if (largeWeightConfig.safetyRatio == null) {
324
101
  throw new Error('runtime.inference.largeWeights.safetyRatio is required.');
@@ -340,14 +117,6 @@ function resolveSplitLmHeadRows(device, hiddenSize, largeWeightConfig) {
340
117
  return rows;
341
118
  }
342
119
 
343
- export function shouldMaterializeSplitLmHeadGPU(lmHead, largeWeightConfig) {
344
- const overrides = largeWeightConfig?.gpuResidentOverrides;
345
- if (!Array.isArray(overrides) || overrides.length === 0) {
346
- return false;
347
- }
348
- const label = lmHead?.label;
349
- return typeof label === 'string' && overrides.includes(label);
350
- }
351
120
 
352
121
  function destroySplitWeightBuffer(splitWeight) {
353
122
  if (!splitWeight) {
@@ -454,7 +223,7 @@ async function materializeSplitLmHeadGPU(lmHead, hiddenSize, weightVocabSize, la
454
223
  }
455
224
 
456
225
 
457
- export async function computeChunkedLogitsGPU(
226
+ export async function computeChunkedLogitsTensorGPU(
458
227
  normedTensor,
459
228
  lmHead,
460
229
  numTokens,
@@ -477,7 +246,7 @@ export async function computeChunkedLogitsGPU(
477
246
 
478
247
  const splitLmHead = await materializeSplitLmHeadGPU(lmHead, hiddenSize, weightVocabSize, largeWeightConfig);
479
248
  if (splitLmHead) {
480
- return computeSplitLogitsGPU(
249
+ return computeSplitLogitsTensorGPU(
481
250
  normedTensor,
482
251
  splitLmHead,
483
252
  numTokens,
@@ -503,7 +272,11 @@ export async function computeChunkedLogitsGPU(
503
272
  hasF16: caps.hasF16,
504
273
  });
505
274
  const preferF16 = weightDtype === 'f16';
506
- const logits = new Float32Array(numTokens * vocabSize);
275
+ const logitsBuffer = acquireBuffer(
276
+ numTokens * vocabSize * Float32Array.BYTES_PER_ELEMENT,
277
+ undefined,
278
+ 'lm_head_chunked_logits'
279
+ );
507
280
 
508
281
  if (isTraceEnabled('logits')) {
509
282
  trace.logits(`LM_HEAD_CHUNKED: vocab=${vocabSize}, chunkRows=${chunkRows}, layout=${lmHead.layout}, f16=${preferF16}`);
@@ -517,8 +290,9 @@ export async function computeChunkedLogitsGPU(
517
290
  })
518
291
  : normedTensor;
519
292
 
520
- for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
521
- const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
293
+ try {
294
+ for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
295
+ const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
522
296
  const chunkShape = lmHead.layout === 'column'
523
297
  ? [hiddenSize, rowCount]
524
298
  : [rowCount, hiddenSize];
@@ -580,30 +354,32 @@ export async function computeChunkedLogitsGPU(
580
354
  });
581
355
  }
582
356
 
583
- const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsTensor.dtype });
584
- const chunkLogitsData = await readBufferWithCleanup(
585
- logitsTensor.buffer,
586
- numTokens * rowCount * logitsBytes,
587
- () => {
357
+ try {
358
+ await runLogitScatter(logitsTensor, logitsBuffer, {
359
+ rowCount: numTokens,
360
+ chunkColumns: rowCount,
361
+ targetColumns: vocabSize,
362
+ columnOffset: rowOffset,
363
+ });
364
+ } finally {
588
365
  releaseBuffer(logitsTensor.buffer);
589
366
  releaseBuffer(weightBuffer.buffer);
590
367
  }
591
- );
592
- const chunkLogits = logitsTensor.dtype === 'f16'
593
- ? f16BufferToF32(chunkLogitsData)
594
- : new Float32Array(chunkLogitsData);
595
- writeChunkLogits(logits, chunkLogits, numTokens, vocabSize, rowOffset, rowCount);
596
- }
368
+ }
597
369
 
598
- if (matmulInput !== normedTensor) {
599
- releaseBuffer(matmulInput.buffer);
370
+ return createTensor(logitsBuffer, 'f32', [numTokens, vocabSize], 'lm_head_chunked_logits');
371
+ } catch (error) {
372
+ releaseBuffer(logitsBuffer);
373
+ throw error;
374
+ } finally {
375
+ if (matmulInput !== normedTensor) {
376
+ releaseBuffer(matmulInput.buffer);
377
+ }
600
378
  }
601
-
602
- return logits;
603
379
  }
604
380
 
605
381
 
606
- export async function computeSplitLogitsGPU(
382
+ export async function computeSplitLogitsTensorGPU(
607
383
  normedTensor,
608
384
  lmHead,
609
385
  numTokens,
@@ -627,7 +403,11 @@ export async function computeSplitLogitsGPU(
627
403
  const lmHeadRole = resolveLmHeadMatmulRole(phase);
628
404
  const lmHeadInputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'inputDtype');
629
405
  const lmHeadOutputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'outputDtype');
630
- const logits = new Float32Array(numTokens * vocabSize);
406
+ const logitsBuffer = acquireBuffer(
407
+ numTokens * vocabSize * Float32Array.BYTES_PER_ELEMENT,
408
+ undefined,
409
+ 'lm_head_split_logits'
410
+ );
631
411
  let matmulInput = normedTensor;
632
412
  let matmulInputOwned = false;
633
413
 
@@ -678,26 +458,47 @@ export async function computeSplitLogitsGPU(
678
458
  });
679
459
  }
680
460
 
681
- const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsTensor.dtype });
682
- const chunkLogitsData = await readBufferWithCleanup(
683
- logitsTensor.buffer,
684
- numTokens * rowCount * logitsBytes,
685
- () => {
686
- releaseBuffer(logitsTensor.buffer);
687
- }
688
- );
689
- const chunkLogits = logitsTensor.dtype === 'f16'
690
- ? f16BufferToF32(chunkLogitsData)
691
- : new Float32Array(chunkLogitsData);
692
- writeChunkLogits(logits, chunkLogits, numTokens, vocabSize, section.rowStart, rowCount);
461
+ try {
462
+ await runLogitScatter(logitsTensor, logitsBuffer, {
463
+ rowCount: numTokens,
464
+ chunkColumns: rowCount,
465
+ targetColumns: vocabSize,
466
+ columnOffset: section.rowStart,
467
+ });
468
+ } finally {
469
+ releaseBuffer(logitsTensor.buffer);
470
+ }
693
471
  }
472
+ return createTensor(logitsBuffer, 'f32', [numTokens, vocabSize], 'lm_head_split_logits');
473
+ } catch (error) {
474
+ releaseBuffer(logitsBuffer);
475
+ throw error;
694
476
  } finally {
695
477
  if (matmulInputOwned) {
696
478
  releaseBuffer(matmulInput.buffer);
697
479
  }
698
480
  }
699
481
 
700
- return logits;
482
+ }
483
+
484
+ export async function computeChunkedLogitsGPU(...args) {
485
+ const tensor = await computeChunkedLogitsTensorGPU(...args);
486
+ const data = await readBufferWithCleanup(
487
+ tensor.buffer,
488
+ tensor.shape[0] * tensor.shape[1] * Float32Array.BYTES_PER_ELEMENT,
489
+ () => releaseBuffer(tensor.buffer)
490
+ );
491
+ return new Float32Array(data);
492
+ }
493
+
494
+ export async function computeSplitLogitsGPU(...args) {
495
+ const tensor = await computeSplitLogitsTensorGPU(...args);
496
+ const data = await readBufferWithCleanup(
497
+ tensor.buffer,
498
+ tensor.shape[0] * tensor.shape[1] * Float32Array.BYTES_PER_ELEMENT,
499
+ () => releaseBuffer(tensor.buffer)
500
+ );
501
+ return new Float32Array(data);
701
502
  }
702
503
 
703
504
 
@@ -708,6 +509,7 @@ export async function computeLogitsGPU(
708
509
  config,
709
510
  debugFlags,
710
511
  operatorDiagnostics = null,
512
+ options = null,
711
513
  ) {
712
514
  const {
713
515
  hiddenSize,
@@ -717,7 +519,7 @@ export async function computeLogitsGPU(
717
519
  embeddingVocabSize,
718
520
  activationDtype,
719
521
  } = config;
720
- const { finalNorm, lmHead } = weights;
522
+ const { finalNorm, lmHead, lmHeadBias = null } = weights;
721
523
  const device = getDevice();
722
524
 
723
525
  if (!device) {
@@ -882,18 +684,20 @@ export async function computeLogitsGPU(
882
684
  ? embeddingVocabSize
883
685
  : vocabSize;
884
686
 
885
- const logitsTensor = await runMatmul(lmHeadInputTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
687
+ let logitsTensor = await runMatmul(lmHeadInputTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
886
688
  transposeB: 'auto',
887
689
  role: lmHeadRole,
888
690
  kernelPath: stableKernelPath,
889
691
  outputDtype: lmHeadOutputDtype,
890
692
  executionPolicies: config.executionPolicies ?? null,
891
693
  });
892
- await runProbes('logits', logitsTensor.buffer, {
694
+ logitsTensor = await finalizeLogitOutputTensor(logitsTensor, config, {
893
695
  numTokens,
894
- hiddenSize: matmulVocabSize,
696
+ vocabSize: matmulVocabSize,
697
+ targetVocabSize: vocabSize,
698
+ bias: lmHeadBias,
699
+ applySoftcap: options?.applySoftcap === true,
895
700
  operatorDiagnostics,
896
- dtype: logitsTensor.dtype,
897
701
  });
898
702
 
899
703
  // Cleanup intermediate buffers (but keep logitsBuffer)
@@ -908,7 +712,7 @@ export async function computeLogitsGPU(
908
712
  if (normWeightBufferOwned) { releaseBuffer(normWeightBuffer); normWeightBufferOwned = false; }
909
713
  if (lmHeadBufferOwned) { releaseBuffer(isWeightBuffer(lmHeadBuffer) ? lmHeadBuffer.buffer : lmHeadBuffer); lmHeadBufferOwned = false; }
910
714
 
911
- return { logitsBuffer: logitsTensor.buffer, vocabSize: matmulVocabSize, logitsDtype: logitsTensor.dtype };
715
+ return { logitsBuffer: logitsTensor.buffer, vocabSize, logitsDtype: logitsTensor.dtype };
912
716
  } finally {
913
717
  if (inputBufferOwned && inputBuffer) releaseBuffer(inputBuffer);
914
718
  if (normInputOwned && normInputTensor) releaseBuffer(normInputTensor.buffer);
@@ -920,270 +724,3 @@ export async function computeLogitsGPU(
920
724
  if (lmHeadBufferOwned && lmHeadBuffer) releaseBuffer(isWeightBuffer(lmHeadBuffer) ? lmHeadBuffer.buffer : lmHeadBuffer);
921
725
  }
922
726
  }
923
-
924
-
925
- async function recordLogitsTailGPU(
926
- recorder,
927
- hiddenStates,
928
- numTokens,
929
- weights,
930
- config,
931
- operatorDiagnostics = null,
932
- ) {
933
- const {
934
- hiddenSize,
935
- vocabSize,
936
- rmsNormEps,
937
- useTiedEmbeddings,
938
- embeddingVocabSize,
939
- activationDtype = 'f32',
940
- } = config;
941
- const { finalNorm, lmHead } = weights;
942
- const matmulVocabSize = useTiedEmbeddings && embeddingVocabSize ? embeddingVocabSize : vocabSize;
943
-
944
- if (!finalNorm || !lmHead) {
945
- throw new Error('[recordLogitsGPU] Final norm or LM head not loaded');
946
- }
947
- if (isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead)) {
948
- throw new Error('[recordLogitsGPU] CPU-resident or split LM head not supported in recorded path');
949
- }
950
-
951
- // Get norm weight buffer
952
-
953
- let normWeightBuffer;
954
- let normWeightOwned = false;
955
- const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, recorder.device.queue, 'final_norm_w');
956
- normWeightBuffer = resolvedFinalNorm.buffer;
957
- normWeightOwned = resolvedFinalNorm.owned;
958
-
959
-
960
- const inputDtype = activationDtype;
961
- // Wrap input buffer as Tensor for RMSNorm
962
- const inputTensor = createTensor(hiddenStates, inputDtype, [numTokens, hiddenSize], 'logits_input');
963
- const phase = numTokens === 1 ? 'decode' : 'prefill';
964
- const kernelPath = config.kernelPath ?? null;
965
- const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
966
- const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
967
- await runProbes('pre_final_norm', hiddenStates, {
968
- numTokens,
969
- hiddenSize,
970
- recorder,
971
- operatorDiagnostics,
972
- dtype: inputDtype,
973
- });
974
- const forceStableF32Logits = !hasExplicitFinalNormPrecision && shouldForceStableF32Logits(config, inputDtype);
975
- const stableKernelPath = forceStableF32Logits
976
- ? createStableF32LogitsKernelPath(kernelPath)
977
- : kernelPath;
978
- let normInputTensor = inputTensor;
979
- let normInputOwned = false;
980
- if (forceStableF32Logits) {
981
- assertImplicitDtypeTransitionAllowed({
982
- executionPolicies: config.executionPolicies ?? null,
983
- fromDtype: inputTensor.dtype,
984
- toDtype: 'f32',
985
- op: 'logits_final_norm',
986
- detail: 'Stable logits mode would widen activations implicitly before final RMSNorm.',
987
- });
988
- normInputTensor = await recordCastF16ToF32(recorder, inputTensor);
989
- normInputOwned = true;
990
- } else {
991
- const finalNormInputDtype = resolvePostLayerStepDtype('final_norm', phase, stableKernelPath, inputTensor.dtype, 'inputDtype');
992
- normInputTensor = finalNormInputDtype !== inputTensor.dtype
993
- ? await coerceTensorDtype(inputTensor, finalNormInputDtype, recorder, {
994
- executionPolicies: config.executionPolicies ?? null,
995
- op: 'final_norm',
996
- transitionDeclaredBy: 'step_precision',
997
- })
998
- : inputTensor;
999
- normInputOwned = normInputTensor !== inputTensor;
1000
- }
1001
- // Record RMSNorm (no submit)
1002
- const normedTensor = await recordRMSNorm(recorder, normInputTensor, normWeightBuffer, rmsNormEps, {
1003
- batchSize: numTokens,
1004
- hiddenSize,
1005
- rmsNormWeightOffset: config.rmsNormWeightOffset,
1006
- label: 'final_norm',
1007
- });
1008
- let finalNormTensor = normedTensor;
1009
- if (!forceStableF32Logits) {
1010
- const finalNormOutputDtype = resolvePostLayerStepDtype(
1011
- 'final_norm',
1012
- phase,
1013
- stableKernelPath,
1014
- normedTensor.dtype,
1015
- 'outputDtype'
1016
- );
1017
- finalNormTensor = finalNormOutputDtype !== normedTensor.dtype
1018
- ? await coerceTensorDtype(normedTensor, finalNormOutputDtype, recorder, {
1019
- executionPolicies: config.executionPolicies ?? null,
1020
- op: 'final_norm',
1021
- transitionDeclaredBy: 'step_precision',
1022
- })
1023
- : normedTensor;
1024
- }
1025
- await runProbes('final_norm', finalNormTensor.buffer, {
1026
- numTokens,
1027
- hiddenSize,
1028
- recorder,
1029
- operatorDiagnostics,
1030
- dtype: finalNormTensor.dtype,
1031
- });
1032
- const logitInputScale = resolveLogitInputScale(config);
1033
- let logitInputTensor = finalNormTensor;
1034
- let logitInputOwned = false;
1035
- if (logitInputScale !== 1) {
1036
- logitInputTensor = await recordScale(recorder, finalNormTensor, logitInputScale, {
1037
- count: numTokens * hiddenSize,
1038
- });
1039
- logitInputOwned = true;
1040
- }
1041
- const lmHeadRole = resolveLmHeadMatmulRole(phase);
1042
- const lmHeadInputDtype = forceStableF32Logits
1043
- ? logitInputTensor.dtype
1044
- : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'inputDtype');
1045
- const lmHeadOutputDtype = forceStableF32Logits
1046
- ? logitInputTensor.dtype
1047
- : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'outputDtype');
1048
- const lmHeadInputTensor = lmHeadInputDtype !== logitInputTensor.dtype
1049
- ? await coerceTensorDtype(logitInputTensor, lmHeadInputDtype, recorder, {
1050
- executionPolicies: config.executionPolicies ?? null,
1051
- op: 'lm_head',
1052
- transitionDeclaredBy: 'step_precision',
1053
- })
1054
- : logitInputTensor;
1055
-
1056
- // Get LM head buffer
1057
-
1058
- let lmHeadBuffer;
1059
- let lmHeadBufferOwned = false;
1060
- if (isGpuBufferInstance(lmHead)) {
1061
- lmHeadBuffer = lmHead;
1062
- } else if (isWeightBuffer(lmHead)) {
1063
- lmHeadBuffer = lmHead;
1064
- } else {
1065
- const rawBuffer = acquireBuffer( (lmHead).byteLength, undefined, 'lm_head_w');
1066
- recorder.device.queue.writeBuffer(rawBuffer, 0, (lmHead));
1067
- lmHeadBuffer = rawBuffer;
1068
- lmHeadBufferOwned = true;
1069
- }
1070
-
1071
- return {
1072
- hiddenSize,
1073
- matmulVocabSize,
1074
- phase,
1075
- stableKernelPath,
1076
- lmHeadRole,
1077
- lmHeadOutputDtype,
1078
- normedTensor,
1079
- finalNormTensor,
1080
- logitInputTensor,
1081
- logitInputOwned,
1082
- normInputTensor,
1083
- normInputOwned,
1084
- normWeightBuffer,
1085
- normWeightOwned,
1086
- lmHeadInputTensor,
1087
- lmHeadBuffer,
1088
- lmHeadBufferOwned,
1089
- };
1090
- }
1091
-
1092
- function trackRecordedLogitsTail(recorder, tail) {
1093
- const trackedTempBuffers = new Set();
1094
- const trackTempBufferOnce = (buffer) => {
1095
- if (!buffer || trackedTempBuffers.has(buffer)) {
1096
- return;
1097
- }
1098
- trackedTempBuffers.add(buffer);
1099
- recorder.trackTemporaryBuffer(buffer);
1100
- };
1101
- if (tail.finalNormTensor !== tail.normedTensor) {
1102
- trackTempBufferOnce(tail.normedTensor.buffer);
1103
- }
1104
- trackTempBufferOnce(tail.finalNormTensor.buffer);
1105
- if (tail.logitInputOwned) {
1106
- trackTempBufferOnce(tail.logitInputTensor.buffer);
1107
- }
1108
- if (tail.lmHeadInputTensor !== tail.logitInputTensor) {
1109
- trackTempBufferOnce(tail.lmHeadInputTensor.buffer);
1110
- }
1111
- if (tail.normWeightOwned) {
1112
- recorder.trackTemporaryBuffer(tail.normWeightBuffer);
1113
- }
1114
- if (tail.normInputOwned) {
1115
- recorder.trackTemporaryBuffer(tail.normInputTensor.buffer);
1116
- }
1117
- if (tail.lmHeadBufferOwned) {
1118
- recorder.trackTemporaryBuffer(isWeightBuffer(tail.lmHeadBuffer) ? tail.lmHeadBuffer.buffer : tail.lmHeadBuffer);
1119
- }
1120
- }
1121
-
1122
- export async function recordLogitsGPU(
1123
- recorder,
1124
- hiddenStates,
1125
- numTokens,
1126
- weights,
1127
- config,
1128
- operatorDiagnostics = null,
1129
- ) {
1130
- const tail = await recordLogitsTailGPU(
1131
- recorder,
1132
- hiddenStates,
1133
- numTokens,
1134
- weights,
1135
- config,
1136
- operatorDiagnostics
1137
- );
1138
-
1139
- const logitsTensor = await recordMatmul(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, numTokens, tail.matmulVocabSize, tail.hiddenSize, {
1140
- transposeB: 'auto',
1141
- role: tail.lmHeadRole,
1142
- kernelPath: tail.stableKernelPath,
1143
- outputDtype: tail.lmHeadOutputDtype,
1144
- executionPolicies: config.executionPolicies ?? null,
1145
- });
1146
- await runProbes('logits', logitsTensor.buffer, {
1147
- numTokens,
1148
- hiddenSize: tail.matmulVocabSize,
1149
- recorder,
1150
- operatorDiagnostics,
1151
- dtype: logitsTensor.dtype,
1152
- });
1153
-
1154
- trackRecordedLogitsTail(recorder, tail);
1155
-
1156
- return { logitsBuffer: logitsTensor.buffer, vocabSize: tail.matmulVocabSize, logitsDtype: logitsTensor.dtype };
1157
- }
1158
-
1159
- export async function recordGreedyLmHeadArgmaxGPU(
1160
- recorder,
1161
- hiddenStates,
1162
- numTokens,
1163
- weights,
1164
- config,
1165
- options,
1166
- operatorDiagnostics = null,
1167
- ) {
1168
- if (numTokens !== 1) {
1169
- throw new Error(`[recordGreedyLmHeadArgmaxGPU] expected numTokens=1, got ${numTokens}.`);
1170
- }
1171
- const tail = await recordLogitsTailGPU(
1172
- recorder,
1173
- hiddenStates,
1174
- numTokens,
1175
- weights,
1176
- config,
1177
- operatorDiagnostics
1178
- );
1179
- const outputBuffer = await recordLmHeadArgmax(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, {
1180
- vocabSize: tail.matmulVocabSize,
1181
- hiddenSize: tail.hiddenSize,
1182
- padTokenId: options.padTokenId,
1183
- logitSoftcap: options.logitSoftcap,
1184
- outputBuffer: options.outputBuffer,
1185
- outputIndex: options.outputIndex,
1186
- });
1187
- trackRecordedLogitsTail(recorder, tail);
1188
- return outputBuffer;
1189
- }