doppler-gpu 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (927) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +99 -86
  6. package/src/capsule-runtime.d.ts +23 -0
  7. package/src/capsule-runtime.js +23 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +1 -0
  24. package/src/client/capsule-host.d.ts +3 -0
  25. package/src/client/capsule-host.js +1 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +36 -1
  28. package/src/client/doppler-api.js +40 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +322 -0
  45. package/src/client/{runtime → model-host}/model-session.js +190 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +68 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +100 -0
  64. package/src/client/runtime/capsule-program-adapter.js +151 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +85 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +74 -0
  72. package/src/client/runtime/composition-root.js +344 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +30 -0
  92. package/src/client/runtime/session-controller.js +170 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +11 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +58 -0
  109. package/src/config/capsule-operations.json +18 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  128. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
  129. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  130. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
  132. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
  134. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
  135. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
  136. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
  137. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
  140. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
  141. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
  142. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  143. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  145. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  146. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
  147. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  148. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  149. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  150. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
  151. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
  152. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  153. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
  154. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
  155. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  157. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
  158. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  160. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
  161. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  162. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  163. package/src/config/dtype-transition-contract.js +40 -0
  164. package/src/config/embedding-contract.js +26 -0
  165. package/src/config/embedding-reference.js +85 -0
  166. package/src/config/evidence/source-intake-policy.json +1 -1
  167. package/src/config/execution-contract-check.d.ts +16 -73
  168. package/src/config/execution-contract-check.js +7 -429
  169. package/src/config/forecast-manifest.js +56 -0
  170. package/src/config/heterogeneous-attention-contract.js +46 -0
  171. package/src/config/initial-execution-identity.d.ts +58 -0
  172. package/src/config/initial-execution-identity.js +331 -0
  173. package/src/config/kernel-path-loader.d.ts +10 -0
  174. package/src/config/kernel-path-loader.js +23 -8
  175. package/src/config/kernel-registry-contract.d.ts +25 -0
  176. package/src/config/kernel-registry-contract.js +46 -0
  177. package/src/config/kernels/kernel-ref-digests.js +50 -2
  178. package/src/config/kernels/registry.js +1 -1
  179. package/src/config/kernels/registry.json +13592 -11377
  180. package/src/config/lora-layouts.d.ts +10 -0
  181. package/src/config/lora-layouts.js +17 -0
  182. package/src/config/lora-layouts.json +9 -0
  183. package/src/config/manifest-inference-validation.js +575 -0
  184. package/src/config/merge/execution-patches.js +12 -0
  185. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  186. package/src/config/merge/value-selection.js +38 -0
  187. package/src/config/merge.js +43 -11
  188. package/src/config/model-ir-v2.d.ts +57 -0
  189. package/src/config/model-ir-v2.js +258 -0
  190. package/src/config/model-ir.d.ts +49 -0
  191. package/src/config/model-ir.js +180 -0
  192. package/src/config/param-validator.js +1 -1
  193. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  194. package/src/config/platforms/loader.js +2 -1
  195. package/src/config/production-release-evidence.d.ts +23 -0
  196. package/src/config/production-release-evidence.js +423 -0
  197. package/src/config/production-release.js +321 -0
  198. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  199. package/src/config/required-inference-fields-contract-check.js +2 -2
  200. package/src/config/rerank-reference.d.ts +28 -0
  201. package/src/config/rerank-reference.js +89 -0
  202. package/src/config/revocation-policy.d.ts +65 -0
  203. package/src/config/revocation-policy.js +290 -0
  204. package/src/config/revocation-registry.json +11 -0
  205. package/src/config/revocation-updates.d.ts +68 -0
  206. package/src/config/revocation-updates.js +440 -0
  207. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  208. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  209. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  210. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  211. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  212. package/src/config/runtime-merge.js +1 -1
  213. package/src/config/runtime.js +1 -1
  214. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  215. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  216. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  217. package/src/config/schema/conversion-report.schema.js +29 -0
  218. package/src/config/schema/converter.schema.d.ts +1 -0
  219. package/src/config/schema/debug.schema.d.ts +1 -0
  220. package/src/config/schema/doppler.schema.js +5 -5
  221. package/src/config/schema/emulation.schema.js +1 -1
  222. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  223. package/src/config/schema/execution-v1.schema.js +19 -0
  224. package/src/config/schema/harness.schema.d.ts +13 -0
  225. package/src/config/schema/harness.schema.js +6 -0
  226. package/src/config/schema/inference.schema.d.ts +21 -2
  227. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  228. package/src/config/schema/manifest.schema.d.ts +26 -7
  229. package/src/config/schema/manifest.schema.js +10 -1
  230. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  231. package/src/config/schema/program-bundle.schema.js +145 -10
  232. package/src/config/sequence-reference.d.ts +26 -0
  233. package/src/config/sequence-reference.js +100 -0
  234. package/src/config/target-plan.d.ts +108 -0
  235. package/src/config/target-plan.js +411 -0
  236. package/src/config/training-defaults.js +1 -1
  237. package/src/config/transforms/capability-transform-resolver.js +4 -105
  238. package/src/config/transforms/execution-graph/dtype.js +591 -0
  239. package/src/config/transforms/execution-graph/fusion.js +436 -0
  240. package/src/config/transforms/execution-graph/validation.js +414 -0
  241. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  242. package/src/converter/artifact-identity.js +475 -0
  243. package/src/converter/conversion-plan.js +1 -1
  244. package/src/converter/core.js +94 -2096
  245. package/src/converter/forge-candidate-evaluation.js +271 -0
  246. package/src/converter/forge-qualification.js +162 -0
  247. package/src/converter/forge-stages.js +798 -0
  248. package/src/converter/manifest-builder.js +683 -0
  249. package/src/converter/quantization-info.js +4 -1
  250. package/src/converter/quantizer.js +58 -9
  251. package/src/converter/shard-packer.d.ts +1 -1
  252. package/src/converter/shard-packer.js +2 -2
  253. package/src/converter/tensor-transform.js +640 -0
  254. package/src/converter/tokenizer-contract.js +243 -0
  255. package/src/converter/tokenizer-manifest.js +243 -0
  256. package/src/debug/history.js +0 -8
  257. package/src/debug/index.js +0 -10
  258. package/src/debug/tensor.js +1 -1
  259. package/src/debug/trace.js +0 -7
  260. package/src/experimental/adapters/adapter-manager.js +2 -0
  261. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  262. package/src/experimental/adapters/adapter-manifest.js +11 -0
  263. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  264. package/src/experimental/adapters/artifact-contract.js +5 -2
  265. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  266. package/src/experimental/adapters/lora-loader.js +110 -48
  267. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  268. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  269. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  270. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  271. package/src/experimental/browser/browser-converter.js +11 -11
  272. package/src/experimental/diffusion/index.d.ts +1 -1
  273. package/src/experimental/diffusion/index.js +1 -1
  274. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  275. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  276. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  277. package/src/experimental/diffusion/vae.js +596 -0
  278. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  279. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  280. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  281. package/src/experimental/distribution/shard-delivery.js +4 -1709
  282. package/src/experimental/energy/index.d.ts +2 -2
  283. package/src/experimental/energy/index.js +2 -2
  284. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  285. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  286. package/src/experimental/logits/cpu-output.d.ts +27 -0
  287. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  288. package/src/experimental/logits/cpu.d.ts +87 -0
  289. package/src/experimental/logits/cpu.js +95 -0
  290. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  291. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  292. package/src/experimental/orchestration/index.d.ts +1 -1
  293. package/src/experimental/orchestration/index.js +1 -1
  294. package/src/experimental/training/artifacts.js +2 -2
  295. package/src/experimental/training/checkpoint.js +2 -2
  296. package/src/experimental/training/distillation/artifacts.js +1 -1
  297. package/src/experimental/training/distillation/dataset.js +2 -2
  298. package/src/experimental/training/distillation/eval.js +1 -1
  299. package/src/experimental/training/distillation/runtime.js +2 -2
  300. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  301. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  302. package/src/experimental/training/distillation/student-fixture.js +2 -469
  303. package/src/experimental/training/lora/checkpoint.js +746 -0
  304. package/src/experimental/training/lora/execution.js +550 -0
  305. package/src/experimental/training/lora/plan.js +387 -0
  306. package/src/experimental/training/lora/recovery.js +418 -0
  307. package/src/experimental/training/lora-pipeline.js +7 -1949
  308. package/src/experimental/training/operator-artifacts.js +2 -2
  309. package/src/experimental/training/operator-command.js +4 -4
  310. package/src/experimental/training/runner/evaluation.js +62 -0
  311. package/src/experimental/training/runner/lifecycle.js +257 -0
  312. package/src/experimental/training/runner.js +5 -268
  313. package/src/experimental/training/suite/execution.js +879 -0
  314. package/src/experimental/training/suite/plan.js +855 -0
  315. package/src/experimental/training/suite.js +5 -1659
  316. package/src/experimental/training/workloads.d.ts +5 -5
  317. package/src/experimental/training/workloads.js +6 -6
  318. package/src/experimental/turboquant/codebook.js +470 -0
  319. package/src/formats/quantization-constants.d.ts +11 -0
  320. package/src/formats/quantization-constants.js +11 -0
  321. package/src/formats/rdrr/integrity.js +1 -1
  322. package/src/formats/rdrr/lowerings.js +0 -29
  323. package/src/formats/rdrr/merkle.js +1 -1
  324. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  325. package/src/formats/rdrr/validation.js +3 -3
  326. package/src/gpu/command-recorder.d.ts +10 -1
  327. package/src/gpu/command-recorder.js +58 -14
  328. package/src/gpu/constant-buffer.js +29 -0
  329. package/src/gpu/device-state.js +36 -0
  330. package/src/gpu/device.js +6 -41
  331. package/src/gpu/kernel-runtime.js +2 -1
  332. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  333. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  334. package/src/gpu/kernels/attention/executors.js +960 -0
  335. package/src/gpu/kernels/attention/plan.js +472 -0
  336. package/src/gpu/kernels/attention.d.ts +4 -0
  337. package/src/gpu/kernels/attention.js +145 -1545
  338. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  339. package/src/gpu/kernels/backward/adam.js +2 -1
  340. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  341. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  342. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  343. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  344. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  345. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  346. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  347. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  348. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  349. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  350. package/src/gpu/kernels/backward/executor.js +158 -0
  351. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  352. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  353. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  354. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  355. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  356. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  357. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  358. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  359. package/src/gpu/kernels/backward/matmul.js +133 -0
  360. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  361. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  362. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  363. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  364. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  365. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  366. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  367. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  368. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  369. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  370. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  371. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  372. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  374. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  376. package/src/gpu/kernels/backward/utils.js +2 -392
  377. package/src/gpu/kernels/bind-group-validation.js +16 -0
  378. package/src/gpu/kernels/cast.js +2 -1
  379. package/src/gpu/kernels/check-finiteness.js +13 -49
  380. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  381. package/src/gpu/kernels/check-stop.js +14 -33
  382. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  383. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  384. package/src/gpu/kernels/check_stop.wgsl +20 -0
  385. package/src/gpu/kernels/clamp.js +1 -1
  386. package/src/gpu/kernels/conv2d.js +1 -1
  387. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  388. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  389. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  390. package/src/gpu/kernels/dequant.js +7 -2
  391. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  392. package/src/gpu/kernels/dispatch.js +12 -1
  393. package/src/gpu/kernels/embedding-pool.js +72 -0
  394. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  395. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  396. package/src/gpu/kernels/energy.js +2 -1
  397. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  398. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  399. package/src/gpu/kernels/fused_ffn.js +10 -220
  400. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  401. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  402. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  403. package/src/gpu/kernels/gated-short-conv.js +11 -69
  404. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  405. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  406. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  407. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  408. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  409. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  410. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  411. package/src/gpu/kernels/gather.d.ts +1 -1
  412. package/src/gpu/kernels/gather.js +30 -6
  413. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  414. package/src/gpu/kernels/gelu.js +1 -1
  415. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  416. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  417. package/src/gpu/kernels/groupnorm.js +1 -1
  418. package/src/gpu/kernels/index.d.ts +38 -5
  419. package/src/gpu/kernels/index.js +18 -4
  420. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  421. package/src/gpu/kernels/kernel-base.js +1 -1
  422. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  423. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  424. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  425. package/src/gpu/kernels/kernel-configs.js +26 -54
  426. package/src/gpu/kernels/kernel-execution.js +141 -0
  427. package/src/gpu/kernels/kv-cache-write.js +4 -4
  428. package/src/gpu/kernels/kv-quantize.js +4 -1
  429. package/src/gpu/kernels/l2-normalize.js +36 -0
  430. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  431. package/src/gpu/kernels/layernorm.d.ts +4 -4
  432. package/src/gpu/kernels/layernorm.js +44 -7
  433. package/src/gpu/kernels/linear-attention-core.js +29 -449
  434. package/src/gpu/kernels/linear_attention.js +1 -1
  435. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  436. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  437. package/src/gpu/kernels/logit-finalize.js +103 -0
  438. package/src/gpu/kernels/logit-merge.js +13 -125
  439. package/src/gpu/kernels/logit-scatter.js +40 -0
  440. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  441. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  442. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  443. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  444. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  445. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  446. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  447. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  448. package/src/gpu/kernels/matmul-selection.js +3 -2
  449. package/src/gpu/kernels/matmul.js +4 -15
  450. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  451. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  452. package/src/gpu/kernels/modulate.js +1 -1
  453. package/src/gpu/kernels/moe.js +3 -1
  454. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  455. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  456. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  457. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  458. package/src/gpu/kernels/patch_context.wgsl +29 -0
  459. package/src/gpu/kernels/pipeline-cache.js +3 -1
  460. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  461. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  462. package/src/gpu/kernels/relu.js +1 -1
  463. package/src/gpu/kernels/repeat_channels.js +1 -1
  464. package/src/gpu/kernels/residual.js +2 -2
  465. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  466. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  467. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  468. package/src/gpu/kernels/rmsnorm.js +1 -1
  469. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  470. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  471. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  472. package/src/gpu/kernels/rope-precompute.js +160 -0
  473. package/src/gpu/kernels/rope-qk.js +1 -1
  474. package/src/gpu/kernels/rope.js +2 -2
  475. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  476. package/src/gpu/kernels/sample.js +5 -1
  477. package/src/gpu/kernels/scale.js +1 -1
  478. package/src/gpu/kernels/shader-cache.js +8 -4
  479. package/src/gpu/kernels/shader-source-scope.js +68 -0
  480. package/src/gpu/kernels/silu.js +2 -1
  481. package/src/gpu/kernels/soft-embedding.js +3 -1
  482. package/src/gpu/kernels/softmax.js +4 -2
  483. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  484. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  485. package/src/gpu/kernels/split_qg.js +1 -1
  486. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  487. package/src/gpu/kernels/split_qkv.js +18 -3
  488. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  489. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  490. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  491. package/src/gpu/kernels/transpose.js +1 -1
  492. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  493. package/src/gpu/kernels/uniform-utils.js +7 -2
  494. package/src/gpu/kernels/upsample2d.js +1 -1
  495. package/src/gpu/kernels/utils.js +9 -330
  496. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  497. package/src/gpu/kernels/vision-average-pool.js +68 -0
  498. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  499. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  500. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  501. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  502. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  503. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  504. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  505. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  506. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  507. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  508. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  509. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  510. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  511. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  512. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  513. package/src/gpu/readback-buffer.js +20 -0
  514. package/src/gpu/submit-probe.js +2 -26
  515. package/src/gpu/weight-buffer.d.ts +1 -0
  516. package/src/gpu/weight-buffer.js +7 -0
  517. package/src/index.d.ts +10 -0
  518. package/src/index.js +1 -0
  519. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  520. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  521. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  522. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  523. package/src/inference/browser-harness/inference-execution.js +543 -0
  524. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  525. package/src/inference/browser-harness/model-resolution.js +137 -0
  526. package/src/inference/browser-harness/report.js +338 -0
  527. package/src/inference/browser-harness/request.js +263 -0
  528. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  529. package/src/inference/browser-harness/runtime-config.js +255 -0
  530. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  531. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  532. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  533. package/src/inference/browser-harness/runtime-report.js +117 -0
  534. package/src/inference/browser-harness/suite-summary.js +39 -0
  535. package/src/inference/browser-harness/text-evidence.js +694 -0
  536. package/src/inference/browser-harness/text-execution.js +757 -0
  537. package/src/inference/browser-harness/text-input.js +725 -0
  538. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  539. package/src/inference/browser-harness-contract-helpers.js +1 -1
  540. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  541. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  542. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  543. package/src/inference/browser-harness-text-helpers.js +31 -2199
  544. package/src/inference/browser-harness.d.ts +15 -2
  545. package/src/inference/browser-harness.js +38 -1952
  546. package/src/inference/kv-cache/base.js +67 -381
  547. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  548. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  549. package/src/inference/kv-cache/quantized.d.ts +2 -2
  550. package/src/inference/kv-cache/quantized.js +4 -8
  551. package/src/inference/kv-cache/update.js +309 -0
  552. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  553. package/src/inference/model-checkpoint-evidence.js +264 -0
  554. package/src/inference/moe-router.js +36 -216
  555. package/src/inference/multi-model-network.js +5 -341
  556. package/src/inference/pipelines/audio/index.js +1 -20
  557. package/src/inference/pipelines/audio/mel.js +0 -38
  558. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  559. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  560. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  561. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  562. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  563. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  564. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  565. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  566. package/src/inference/pipelines/factory.js +14 -3
  567. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  568. package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
  569. package/src/inference/pipelines/text/abort-contract.js +22 -0
  570. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  571. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  572. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  573. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  574. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  575. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  576. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  577. package/src/inference/pipelines/text/attention/plan.js +17 -0
  578. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  579. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  580. package/src/inference/pipelines/text/attention/projections.js +20 -537
  581. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  582. package/src/inference/pipelines/text/attention/record.js +0 -6
  583. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  584. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  585. package/src/inference/pipelines/text/attention/run.js +0 -6
  586. package/src/inference/pipelines/text/attention/types.js +3 -31
  587. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  588. package/src/inference/pipelines/text/chat-format.js +42 -0
  589. package/src/inference/pipelines/text/config/normalization.js +729 -0
  590. package/src/inference/pipelines/text/config/validation.js +9 -0
  591. package/src/inference/pipelines/text/config.d.ts +12 -1
  592. package/src/inference/pipelines/text/config.js +14 -1228
  593. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  594. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  595. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  596. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  597. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  598. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  599. package/src/inference/pipelines/text/embed.js +178 -241
  600. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  601. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  602. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  603. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  604. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  605. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  606. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  607. package/src/inference/pipelines/text/execution-v1.js +7 -283
  608. package/src/inference/pipelines/text/execution.js +517 -0
  609. package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
  610. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  611. package/src/inference/pipelines/text/ffn/dense.js +4 -968
  612. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  613. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  614. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  615. package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
  616. package/src/inference/pipelines/text/generator/decode.js +952 -0
  617. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  618. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  619. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  620. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  621. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  622. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  623. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  624. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  625. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  626. package/src/inference/pipelines/text/generator/text.js +448 -0
  627. package/src/inference/pipelines/text/generator/training.js +752 -0
  628. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  629. package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
  630. package/src/inference/pipelines/text/generator-runtime.js +5 -135
  631. package/src/inference/pipelines/text/generator-steps.js +30 -972
  632. package/src/inference/pipelines/text/generator.js +165 -3544
  633. package/src/inference/pipelines/text/image-transcription.js +304 -0
  634. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  635. package/src/inference/pipelines/text/init-rope.js +37 -189
  636. package/src/inference/pipelines/text/init.d.ts +8 -3
  637. package/src/inference/pipelines/text/init.js +6 -216
  638. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  639. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  640. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  641. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  642. package/src/inference/pipelines/text/layer-plan.js +45 -3
  643. package/src/inference/pipelines/text/layer.js +52 -522
  644. package/src/inference/pipelines/text/lifecycle.js +839 -0
  645. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  646. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  647. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  648. package/src/inference/pipelines/text/linear-attention.js +7 -537
  649. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  650. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  651. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  652. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  653. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  654. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  655. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  656. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  657. package/src/inference/pipelines/text/logits/index.js +62 -102
  658. package/src/inference/pipelines/text/logits/output-transform.js +47 -0
  659. package/src/inference/pipelines/text/logits/plan.js +184 -0
  660. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  661. package/src/inference/pipelines/text/logits/readback.js +9 -0
  662. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  663. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  664. package/src/inference/pipelines/text/lora-apply.js +12 -4
  665. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  666. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  667. package/src/inference/pipelines/text/model-load.js +0 -20
  668. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  669. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  670. package/src/inference/pipelines/text/moe/plan.js +47 -0
  671. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  672. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  673. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  674. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  675. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  676. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  677. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  678. package/src/inference/pipelines/text/probes.js +20 -0
  679. package/src/inference/pipelines/text/state.d.ts +8 -0
  680. package/src/inference/pipelines/text/state.js +1 -0
  681. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  682. package/src/inference/pipelines/text/types.d.ts +4 -0
  683. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  684. package/src/inference/pipelines/text.d.ts +2 -2
  685. package/src/inference/pipelines/text.js +46 -1633
  686. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  687. package/src/inference/pipelines/video/index.js +0 -21
  688. package/src/inference/pipelines/vision/attention.js +59 -0
  689. package/src/inference/pipelines/vision/encoder.js +254 -369
  690. package/src/inference/pipelines/vision/gemma4.js +204 -318
  691. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  692. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  693. package/src/inference/pipelines/vision/glmocr.js +557 -0
  694. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  695. package/src/inference/pipelines/vision/index.js +8 -27
  696. package/src/inference/pipelines/vision/ops.js +11 -33
  697. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  698. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  699. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  700. package/src/inference/refactor-receipt.js +2 -2
  701. package/src/inference/rerank.js +182 -0
  702. package/src/inference/tokenizer.js +1 -1
  703. package/src/inference/tokenizers/behavior-flags.js +6 -3
  704. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  705. package/src/inference/tokenizers/bundled/registry.js +216 -0
  706. package/src/inference/tokenizers/bundled.js +23 -353
  707. package/src/loader/doppler-loader.d.ts +4 -0
  708. package/src/loader/doppler-loader.js +119 -930
  709. package/src/loader/embedding-limit-preflight.js +11 -4
  710. package/src/loader/embedding-loader.js +1 -3
  711. package/src/loader/gpu-tensor-materialization.js +20 -0
  712. package/src/loader/layer-loader.js +15 -119
  713. package/src/loader/layer-tensor-aliases.js +99 -0
  714. package/src/loader/layer-weight-shape.js +29 -0
  715. package/src/loader/load-timing.js +64 -0
  716. package/src/loader/loader-types.d.ts +3 -0
  717. package/src/loader/model-executor.js +819 -0
  718. package/src/loader/model-load-validation.js +15 -0
  719. package/src/loader/quantization-constants.d.ts +11 -29
  720. package/src/loader/quantization-constants.js +1 -2
  721. package/src/loader/shard-resolver.js +0 -14
  722. package/src/loader/tensor-load-error.js +14 -0
  723. package/src/loader/tensor-shard-indices.js +8 -0
  724. package/src/loader/tensors/bf16-loader.js +125 -0
  725. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  726. package/src/loader/tensors/load-executor.js +577 -0
  727. package/src/loader/tensors/source-transform/executor.js +123 -0
  728. package/src/loader/tensors/source-transform/plan.js +107 -0
  729. package/src/loader/tensors/source-transform.js +15 -264
  730. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  731. package/src/loader/tensors/tensor-loader.js +7 -653
  732. package/src/memory/buffer-pool.js +7 -29
  733. package/src/rules/capability-transform-resolver.js +80 -0
  734. package/src/rules/execution-contract.d.ts +73 -0
  735. package/src/rules/execution-contract.js +429 -0
  736. package/src/rules/execution-rules-contract-check.js +1 -1
  737. package/src/rules/generated/rule-bundle.json +4305 -0
  738. package/src/rules/inference/capability-transforms.rules.json +3 -3
  739. package/src/rules/layer-pattern-contract-check.js +1 -1
  740. package/src/rules/rule-registry.js +70 -57
  741. package/src/storage/artifact-storage-context.js +2 -2
  742. package/src/storage/backend-selection.js +43 -0
  743. package/src/storage/distribution-transport.js +5 -0
  744. package/src/storage/download/integrity.d.ts +33 -0
  745. package/src/storage/download/integrity.js +130 -0
  746. package/src/storage/download/resume-inspection.d.ts +22 -0
  747. package/src/storage/download/resume-inspection.js +134 -0
  748. package/src/storage/download/retry.js +57 -0
  749. package/src/storage/download/state.js +131 -0
  750. package/src/storage/download/transport.js +99 -0
  751. package/src/storage/downloader.js +74 -453
  752. package/src/storage/model-cache.d.ts +1 -0
  753. package/src/storage/model-cache.js +51 -11
  754. package/src/storage/shard-manager.d.ts +5 -0
  755. package/src/storage/shard-manager.js +50 -369
  756. package/src/storage/shards/index.js +60 -0
  757. package/src/storage/shards/integrity.js +225 -0
  758. package/src/storage/shards/lifecycle.js +22 -0
  759. package/src/storage/source-artifact-store.js +14 -4
  760. package/src/tooling/boundary-evidence.d.ts +4 -4
  761. package/src/tooling/boundary-evidence.js +20 -20
  762. package/src/tooling/browser-command-runner.js +13 -6
  763. package/src/tooling/capsule-v2.d.ts +11 -0
  764. package/src/tooling/capsule-v2.js +58 -0
  765. package/src/tooling/capsule.js +20 -0
  766. package/src/tooling/command-api-constants.js +1 -1
  767. package/src/tooling/command-api-family-normalizers.js +163 -5
  768. package/src/tooling/command-api.d.ts +42 -3
  769. package/src/tooling/command-api.js +7 -2
  770. package/src/tooling/command-envelope.js +1 -1
  771. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  772. package/src/tooling/command-runner.html +49 -1
  773. package/src/tooling/command-validation.js +1 -24
  774. package/src/tooling/conversion-run-timing.js +24 -0
  775. package/src/tooling/diagnose-runner.js +11 -9
  776. package/src/tooling/distribution-shard-transport.js +11 -0
  777. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  778. package/src/tooling/execution-cost-ledger.js +197 -15
  779. package/src/tooling/hf-registry/catalog-io.js +63 -0
  780. package/src/tooling/litert/package.js +863 -0
  781. package/src/tooling/litert-package-runtime.js +3 -849
  782. package/src/tooling/model-capsule-forge.js +478 -0
  783. package/src/tooling/node-browser/execution.d.ts +40 -0
  784. package/src/tooling/node-browser/execution.js +308 -0
  785. package/src/tooling/node-browser/static-server.js +188 -0
  786. package/src/tooling/node-browser/transport.d.ts +80 -0
  787. package/src/tooling/node-browser/transport.js +622 -0
  788. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  789. package/src/tooling/node-browser-command-runner.js +10 -1112
  790. package/src/tooling/node-capsule-artifact-store.js +44 -0
  791. package/src/tooling/node-command-execution.js +237 -0
  792. package/src/tooling/node-command-runner.js +26 -205
  793. package/src/tooling/node-converter/input.js +521 -0
  794. package/src/tooling/node-converter/output.js +244 -0
  795. package/src/tooling/node-converter-chunk-batches.js +21 -0
  796. package/src/tooling/node-converter.js +7 -720
  797. package/src/tooling/node-source-runtime.js +16 -276
  798. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  799. package/src/tooling/node-webgpu.js +178 -453
  800. package/src/tooling/opfs-cache.d.ts +5 -0
  801. package/src/tooling/opfs-cache.js +4 -0
  802. package/src/tooling/production-release.d.ts +19 -0
  803. package/src/tooling/production-release.js +647 -0
  804. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  805. package/src/tooling/program-bundle/materialize.js +324 -0
  806. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  807. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  808. package/src/tooling/program-bundle/validation.js +31 -0
  809. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  810. package/src/tooling/program-bundle-parity.d.ts +11 -4
  811. package/src/tooling/program-bundle-parity.js +262 -109
  812. package/src/tooling/program-bundle.d.ts +7 -0
  813. package/src/tooling/program-bundle.js +186 -591
  814. package/src/tooling/provider-v1-contract.d.ts +61 -0
  815. package/src/tooling/provider-v1-contract.js +192 -0
  816. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  817. package/src/tooling/registered-variant-calibration-job.js +210 -0
  818. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  819. package/src/tooling/registered-variant-calibration.js +57 -7
  820. package/src/tooling/runtime-optimization/candidates.js +664 -0
  821. package/src/tooling/runtime-optimization-campaign.js +166 -0
  822. package/src/tooling/runtime-optimization-index.js +16 -1
  823. package/src/tooling/runtime-optimization.d.ts +44 -0
  824. package/src/tooling/runtime-optimization.js +13 -656
  825. package/src/tooling/source-intake.d.ts +1 -1
  826. package/src/tooling/source-intake.js +34 -17
  827. package/src/tooling/source-package-profiles.js +1 -1
  828. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  829. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  830. package/src/tooling/source-runtime/node-loader.js +258 -0
  831. package/src/tooling/source-runtime-bundle.js +26 -411
  832. package/src/tooling/source-runtime-converter-config.js +1 -1
  833. package/src/tooling-experimental-exports.js +0 -5
  834. package/src/tooling-exports/evidence.d.ts +14 -3
  835. package/src/tooling-exports/evidence.js +14 -3
  836. package/src/tooling-exports/storage.d.ts +5 -0
  837. package/src/tooling-exports/storage.js +4 -0
  838. package/src/tooling-exports.d.ts +3 -0
  839. package/src/tooling-exports.js +2 -0
  840. package/src/tooling-exports.shared.d.ts +17 -3
  841. package/src/tooling-exports.shared.js +9 -3
  842. package/src/training.d.ts +6 -6
  843. package/src/training.js +4 -4
  844. package/src/version.js +1 -1
  845. package/tests/kernels/browser/kernel-suite.js +512 -0
  846. package/tests/kernels/browser/test-page.js +169 -14
  847. package/tests/kernels/harness/tolerance.js +1 -0
  848. package/tests/kernels/reference/index.js +13 -1
  849. package/tests/kernels/reference/vision.js +160 -0
  850. package/src/converter/quantizer.d.ts +0 -121
  851. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  852. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  853. package/src/gpu/kernels/utils.d.ts +0 -147
  854. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  855. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  856. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  857. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  858. package/src/rules/converter/execution.rules.json +0 -20
  859. package/src/rules/converter/tensor-roles.rules.json +0 -13
  860. package/src/rules/converter/tokenizer.rules.json +0 -12
  861. package/src/rules/inference/attention.rules.json +0 -69
  862. package/src/rules/inference/config.rules.json +0 -82
  863. package/src/rules/inference/dtype.rules.json +0 -113
  864. package/src/rules/inference/execution.rules.json +0 -195
  865. package/src/rules/inference/ffn.rules.json +0 -123
  866. package/src/rules/inference/layer-pattern.rules.json +0 -16
  867. package/src/rules/inference/layer.rules.json +0 -7
  868. package/src/rules/inference/moe.rules.json +0 -145
  869. package/src/rules/kernels/attention.rules.json +0 -68
  870. package/src/rules/kernels/conv2d.rules.json +0 -6
  871. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  872. package/src/rules/kernels/dequant.rules.json +0 -58
  873. package/src/rules/kernels/energy.rules.json +0 -22
  874. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  875. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  876. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  877. package/src/rules/kernels/gather.rules.json +0 -93
  878. package/src/rules/kernels/gelu.rules.json +0 -11
  879. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  880. package/src/rules/kernels/groupnorm.rules.json +0 -10
  881. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  882. package/src/rules/kernels/layernorm.rules.json +0 -6
  883. package/src/rules/kernels/linear-attention.rules.json +0 -6
  884. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  885. package/src/rules/kernels/matmul.rules.json +0 -95
  886. package/src/rules/kernels/modulate.rules.json +0 -6
  887. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  888. package/src/rules/kernels/moe.rules.json +0 -14
  889. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  890. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  891. package/src/rules/kernels/relu.rules.json +0 -6
  892. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  893. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  894. package/src/rules/kernels/residual.rules.json +0 -12
  895. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  896. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  897. package/src/rules/kernels/rope-qk.rules.json +0 -6
  898. package/src/rules/kernels/rope.rules.json +0 -6
  899. package/src/rules/kernels/sample.rules.json +0 -6
  900. package/src/rules/kernels/scale.rules.json +0 -8
  901. package/src/rules/kernels/silu.rules.json +0 -22
  902. package/src/rules/kernels/softmax.rules.json +0 -25
  903. package/src/rules/kernels/split-qg.rules.json +0 -6
  904. package/src/rules/kernels/split-qkv.rules.json +0 -6
  905. package/src/rules/kernels/upsample2d.rules.json +0 -6
  906. package/src/rules/loader/tensor-loader.rules.json +0 -42
  907. package/src/rules/loader/weights.rules.json +0 -42
  908. package/src/rules/tooling/command-runtime.rules.json +0 -65
  909. package/src/tooling/hf-registry-utils.js +0 -395
  910. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  911. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  912. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  913. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  914. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  915. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  916. /package/src/{utils → formats}/array-buffer.js +0 -0
  917. /package/src/{utils → formats}/canonical-hash.js +0 -0
  918. /package/src/{utils → formats}/clone-json.js +0 -0
  919. /package/src/{utils → formats}/encode-utf8.js +0 -0
  920. /package/src/{utils → formats}/load-json.js +0 -0
  921. /package/src/{utils → formats}/plain-object.js +0 -0
  922. /package/src/{utils → formats}/sha256.js +0 -0
  923. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  924. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  925. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  926. /package/src/{utils → storage}/runtime-env.js +0 -0
  927. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -26,63 +26,10 @@ import { selectRuleValue } from '../../../rules/rule-registry.js';
26
26
  import { getDevice } from '../../../gpu/device.js';
27
27
  import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
28
28
  import { f16ToF32 } from '../../../loader/dtype-utils.js';
29
-
30
- const pleRuntimeCache = new WeakMap();
31
- const pleRangeRowCache = new WeakMap();
32
- let f16ToF32Lookup = null;
33
- const PLE_ACTIVATION_STATIC_QMIN = -128;
34
- const PLE_ACTIVATION_STATIC_QMAX = 127;
35
-
36
- function getF16ToF32Lookup() {
37
- if (f16ToF32Lookup) {
38
- return f16ToF32Lookup;
39
- }
40
- const lookup = new Float32Array(1 << 16);
41
- for (let value = 0; value < lookup.length; value += 1) {
42
- lookup[value] = f16ToF32(value);
43
- }
44
- f16ToF32Lookup = lookup;
45
- return lookup;
46
- }
47
-
48
- function getPerLayerInputWeights(context) {
49
- const weights = context.weights.get('per_layer_inputs');
50
- if (!weights || typeof weights !== 'object') {
51
- throw new Error(
52
- 'Gemma 4 per-layer inputs require global per-layer input weights, ' +
53
- 'but state.weights.get("per_layer_inputs") was missing.'
54
- );
55
- }
56
- return weights;
57
- }
58
-
59
- function normalizePleProjectionNormDtype(dtype) {
60
- if (typeof dtype !== 'string') {
61
- return null;
62
- }
63
- const value = dtype.toLowerCase();
64
- if (value === 'f16' || value === 'f32') {
65
- return value;
66
- }
67
- return null;
68
- }
69
-
70
- function getPleProjectionNormDtype(weight) {
71
- return normalizePleProjectionNormDtype(getWeightDtype(weight))
72
- ?? normalizePleProjectionNormDtype(getBufferDtype(weight))
73
- ?? null;
74
- }
75
-
76
- function isRangeBackedPleProjectionSource(value) {
77
- return isRangeBackedCpuEmbeddingSource(value);
78
- }
79
-
80
- function getPleProjectionWeightDtype(weight) {
81
- if (isCpuWeightBuffer(weight)) {
82
- return weight.dtype ?? null;
83
- }
84
- return getWeightDtype(weight);
85
- }
29
+ import { destroyPleHotVocabularyRuntime, destroyPleRuntimeCacheEntry, ensurePleScaledProjectionNormWeight, getPerLayerInputWeights, getPleHotCachePolicy, getPleRangeRowLoadConfig, getPleSplitTablePolicy, pleRuntimeCache } from './per-layer/materialize.js';
30
+ import { applyPleActivationStaticQdq, getPleProjectionWeightDtype, getPreparedTokenEntry, isRangeBackedPleProjectionSource, loadRangeBackedPleProjectionSliceBytes, loadRangeBackedPleRow, pleRangeRowCache, storePreparedTokenEntry } from './per-layer/plan.js';
31
+ export { loadRangeBackedPleProjectionSliceBytes } from './per-layer/plan.js';
32
+ export { ensurePleGpuHotVocabularyRuntime, ensurePleGpuSplitTablesRuntime, ensurePleScaledProjectionNormWeight, inferPleProjectionNormDtype, resolvePleHotVocabularyCapacity, scalePerLayerProjectionNormWeights } from './per-layer/materialize.js';
86
33
 
87
34
  function isDensePleProjectionDtype(dtype) {
88
35
  return dtype === 'f16' || dtype === 'f32';
@@ -125,69 +72,6 @@ function resolvePleActivationStaticQdqConfig(perLayerInputWeights) {
125
72
  };
126
73
  }
127
74
 
128
- function getTensorElementCount(tensor, label) {
129
- if (!Array.isArray(tensor?.shape) || tensor.shape.length === 0) {
130
- throw new Error(`${label} requires a tensor with an explicit shape.`);
131
- }
132
- const count = tensor.shape.reduce((total, dimension) => total * Number(dimension), 1);
133
- if (!Number.isFinite(count) || count <= 0) {
134
- throw new Error(`${label} resolved an invalid tensor element count (${String(count)}).`);
135
- }
136
- return count;
137
- }
138
-
139
- async function applyPleActivationStaticQdq(tensor, scale, recorder, decodeBuffers, label) {
140
- const count = getTensorElementCount(tensor, label);
141
- if (tensor.dtype === 'f32') {
142
- return recorder
143
- ? await recordActivationStaticQdq(recorder, tensor, scale, {
144
- count,
145
- qmin: PLE_ACTIVATION_STATIC_QMIN,
146
- qmax: PLE_ACTIVATION_STATIC_QMAX,
147
- })
148
- : await runActivationStaticQdq(tensor, scale, {
149
- count,
150
- qmin: PLE_ACTIVATION_STATIC_QMIN,
151
- qmax: PLE_ACTIVATION_STATIC_QMAX,
152
- });
153
- }
154
- if (tensor.dtype !== 'f16') {
155
- throw new Error(
156
- `${label} requires f16/f32 activations for static activation quantize/dequantize. ` +
157
- `Got "${String(tensor.dtype)}".`
158
- );
159
- }
160
-
161
- let widenedTensor = null;
162
- let qdqTensor = null;
163
- try {
164
- widenedTensor = recorder
165
- ? await recordCastF16ToF32(recorder, tensor)
166
- : await castF16ToF32(tensor);
167
- qdqTensor = recorder
168
- ? await recordActivationStaticQdq(recorder, widenedTensor, scale, {
169
- count,
170
- qmin: PLE_ACTIVATION_STATIC_QMIN,
171
- qmax: PLE_ACTIVATION_STATIC_QMAX,
172
- })
173
- : await runActivationStaticQdq(widenedTensor, scale, {
174
- count,
175
- qmin: PLE_ACTIVATION_STATIC_QMIN,
176
- qmax: PLE_ACTIVATION_STATIC_QMAX,
177
- });
178
- return recorder
179
- ? await recordCastF32ToF16(recorder, qdqTensor)
180
- : await castF32ToF16(qdqTensor);
181
- } finally {
182
- if (qdqTensor) {
183
- releaseOrTrack(recorder, qdqTensor.buffer, decodeBuffers);
184
- }
185
- if (widenedTensor) {
186
- releaseOrTrack(recorder, widenedTensor.buffer, decodeBuffers);
187
- }
188
- }
189
- }
190
-
191
75
  export function resolveDensePleProjectionWeight(
192
76
  weight,
193
77
  label = 'per_layer_model_projection'
@@ -225,81 +109,6 @@ export function resolveDensePleProjectionWeight(
225
109
  return weight;
226
110
  }
227
111
 
228
- export async function loadRangeBackedPleProjectionSliceBytes(
229
- weight,
230
- layerIdx,
231
- hiddenSizePerLayerInput,
232
- hiddenSize,
233
- label = 'Range-backed PLE projection slice'
234
- ) {
235
- if (!isCpuWeightBuffer(weight)) {
236
- return null;
237
- }
238
- const cpuData = weight.data;
239
- if (!isRangeBackedPleProjectionSource(cpuData)) {
240
- return null;
241
- }
242
-
243
- const dtype = getPleProjectionWeightDtype(weight);
244
- const layout = getLayout(weight) ?? weight.layout ?? 'row';
245
- const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
246
- dtype,
247
- });
248
- const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
249
- const byteOffset = hiddenOffset * hiddenSize * bytesPerElement;
250
- const byteLength = hiddenSizePerLayerInput * hiddenSize * bytesPerElement;
251
- const bytes = normalizeRangeBytes(
252
- await cpuData.loadRange(byteOffset, byteLength),
253
- label
254
- );
255
- if (bytes.byteLength !== byteLength) {
256
- throw new Error(
257
- `${label} short read for layer ${layerIdx}: expected ${byteLength} bytes, got ${bytes.byteLength}.`
258
- );
259
- }
260
- return {
261
- bytes,
262
- dtype,
263
- layout,
264
- shape: [hiddenSizePerLayerInput, hiddenSize],
265
- };
266
- }
267
-
268
- export function inferPleProjectionNormDtype(weight, hiddenSizePerLayerInput) {
269
- const explicitDtype = getPleProjectionNormDtype(weight);
270
- if (explicitDtype) {
271
- return explicitDtype;
272
- }
273
-
274
- if (!isGpuBufferInstance(weight)) {
275
- return 'f32';
276
- }
277
-
278
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
279
- throw new Error('Gemma 4 per-layer projection norm dtype inference requires hiddenSizePerLayerInput > 0.');
280
- }
281
-
282
- const bytesPerElement = weight.size / hiddenSizePerLayerInput;
283
- if (bytesPerElement !== 2 && bytesPerElement !== 4) {
284
- throw new Error(
285
- 'Gemma 4 per-layer projection norm buffer has unsupported byte size: ' +
286
- `bufferSize=${weight.size}, hiddenSizePerLayerInput=${hiddenSizePerLayerInput}.`
287
- );
288
- }
289
- return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
290
- }
291
-
292
- function createPleProjectionNormTensor(buffer, dtype, hiddenSizePerLayerInput, label) {
293
- return createTensor(buffer, dtype, [hiddenSizePerLayerInput], label);
294
- }
295
-
296
- function destroyPleRuntimeCacheEntry(entry) {
297
- const cachedBuffer = entry?.scaledProjectionNormWeight?.buffer ?? null;
298
- if (cachedBuffer) {
299
- releaseBuffer(cachedBuffer);
300
- }
301
- }
302
-
303
112
  function destroyPleSplitTables(perLayerInputWeights) {
304
113
  const splitTables = Array.isArray(perLayerInputWeights?.embedTokensPerLayerSplit)
305
114
  ? perLayerInputWeights.embedTokensPerLayerSplit
@@ -316,25 +125,6 @@ function destroyPleSplitTables(perLayerInputWeights) {
316
125
  delete perLayerInputWeights.embedTokensPerLayerSplit;
317
126
  }
318
127
 
319
- function destroyPleHotVocabularyRuntime(perLayerInputWeights) {
320
- const runtime = perLayerInputWeights?.embedTokensPerLayerHotRuntime ?? null;
321
- if (!runtime || typeof runtime !== 'object') {
322
- return;
323
- }
324
- const splitTables = Array.isArray(runtime.splitTables) ? runtime.splitTables : [];
325
- for (const table of splitTables) {
326
- const buffer = table?.buffer ?? table ?? null;
327
- if (buffer) {
328
- releaseBuffer(buffer);
329
- }
330
- }
331
- const mapBuffer = runtime.hotTokenIndexMapBuffer ?? null;
332
- if (mapBuffer) {
333
- releaseBuffer(mapBuffer);
334
- }
335
- delete perLayerInputWeights.embedTokensPerLayerHotRuntime;
336
- }
337
-
338
128
  export function destroyPleRuntimeCache(perLayerInputWeights) {
339
129
  if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
340
130
  return;
@@ -351,126 +141,6 @@ export function destroyPleRuntimeCache(perLayerInputWeights) {
351
141
  }
352
142
  }
353
143
 
354
- export function scalePerLayerProjectionNormWeights(weight, combineScale, rmsNormWeightOffset = false) {
355
- if (rmsNormWeightOffset) {
356
- return null;
357
- }
358
- const source = isCpuWeightBuffer(weight) ? weight.data : weight;
359
- const isArrayLikeView = ArrayBuffer.isView(source) && typeof source.length === 'number';
360
- if (!(source instanceof Float32Array) && !isArrayLikeView && !Array.isArray(source)) {
361
- return null;
362
- }
363
- const scaled = Float32Array.from(source);
364
- for (let i = 0; i < scaled.length; i++) {
365
- scaled[i] *= combineScale;
366
- }
367
- return scaled;
368
- }
369
-
370
- export async function ensurePleScaledProjectionNormWeight(context, combineScale = 2 ** -0.5) {
371
- const hiddenSizePerLayerInput = Number(context?.config?.hiddenSizePerLayerInput ?? 0);
372
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
373
- return null;
374
- }
375
- if (context?.weightConfig?.rmsNormWeightOffset) {
376
- return null;
377
- }
378
-
379
- const perLayerInputWeights = getPerLayerInputWeights(context);
380
- const cachedEntry = pleRuntimeCache.get(perLayerInputWeights) ?? null;
381
- if (cachedEntry?.combineScale === combineScale && cachedEntry.scaledProjectionNormWeight) {
382
- return cachedEntry.scaledProjectionNormWeight;
383
- }
384
- if (cachedEntry) {
385
- destroyPleRuntimeCacheEntry(cachedEntry);
386
- pleRuntimeCache.delete(perLayerInputWeights);
387
- }
388
-
389
- const projectionNormWeight = perLayerInputWeights.perLayerProjectionNorm;
390
- if (!projectionNormWeight) {
391
- return null;
392
- }
393
-
394
- let scaledProjectionNormWeight = null;
395
- if (isGpuBufferInstance(projectionNormWeight)) {
396
- const projectionNormDtype = inferPleProjectionNormDtype(
397
- projectionNormWeight,
398
- hiddenSizePerLayerInput
399
- );
400
- const scaledTensor = await runScale(
401
- createPleProjectionNormTensor(
402
- projectionNormWeight,
403
- projectionNormDtype,
404
- hiddenSizePerLayerInput,
405
- 'per_layer_projection_norm'
406
- ),
407
- combineScale,
408
- { count: hiddenSizePerLayerInput }
409
- );
410
- scaledProjectionNormWeight = createPleProjectionNormTensor(
411
- scaledTensor.buffer,
412
- projectionNormDtype,
413
- hiddenSizePerLayerInput,
414
- 'per_layer_projection_norm_scaled'
415
- );
416
- } else {
417
- const scaledValues = scalePerLayerProjectionNormWeights(
418
- projectionNormWeight,
419
- combineScale,
420
- false
421
- );
422
- if (!scaledValues) {
423
- return null;
424
- }
425
- if (scaledValues.length !== hiddenSizePerLayerInput) {
426
- throw new Error(
427
- 'Gemma 4 per-layer projection norm cache shape mismatch: ' +
428
- `expected ${hiddenSizePerLayerInput} values, got ${scaledValues.length}.`
429
- );
430
- }
431
- const device = getDevice();
432
- if (!device) {
433
- throw new Error('No GPU device available for Gemma 4 per-layer projection norm cache.');
434
- }
435
- const scaledBuffer = acquireBuffer(
436
- scaledValues.byteLength,
437
- undefined,
438
- 'per_layer_projection_norm_scaled'
439
- );
440
- try {
441
- device.queue.writeBuffer(scaledBuffer, 0, scaledValues);
442
- } catch (error) {
443
- releaseBuffer(scaledBuffer);
444
- throw error;
445
- }
446
- scaledProjectionNormWeight = createPleProjectionNormTensor(
447
- scaledBuffer,
448
- inferPleProjectionNormDtype(projectionNormWeight, hiddenSizePerLayerInput),
449
- hiddenSizePerLayerInput,
450
- 'per_layer_projection_norm_scaled'
451
- );
452
- }
453
-
454
- if (scaledProjectionNormWeight?.shape?.[0] !== hiddenSizePerLayerInput) {
455
- throw new Error(
456
- 'Gemma 4 per-layer projection norm cache shape mismatch after tensor creation: ' +
457
- `expected ${hiddenSizePerLayerInput}, got ${scaledProjectionNormWeight?.shape?.[0] ?? 'unknown'}.`
458
- );
459
- }
460
- if (!normalizePleProjectionNormDtype(scaledProjectionNormWeight?.dtype)) {
461
- throw new Error(
462
- 'Gemma 4 per-layer projection norm cache produced an invalid dtype: ' +
463
- `"${String(scaledProjectionNormWeight?.dtype ?? 'undefined')}".`
464
- );
465
- }
466
-
467
- pleRuntimeCache.set(perLayerInputWeights, {
468
- combineScale,
469
- scaledProjectionNormWeight,
470
- });
471
- return scaledProjectionNormWeight;
472
- }
473
-
474
144
  function getEmbeddingSource(weight, label) {
475
145
  if (isWeightBuffer(weight)) {
476
146
  return weight.buffer;
@@ -488,17 +158,6 @@ function getEmbeddingDtype(weight) {
488
158
  return getWeightDtype(weight);
489
159
  }
490
160
 
491
- function requirePleSourceDtype(rawDtype, label, allowed = ['f16', 'f32']) {
492
- if (rawDtype == null) {
493
- throw new Error(`${label} requires source dtype metadata.`);
494
- }
495
- const sourceDtype = String(rawDtype).toLowerCase();
496
- if (!allowed.includes(sourceDtype)) {
497
- throw new Error(`${label} requires ${allowed.join('/')} source rows; got "${sourceDtype}".`);
498
- }
499
- return sourceDtype;
500
- }
501
-
502
161
  function getEmbeddingTranspose(weight) {
503
162
  if (isWeightBuffer(weight) || isCpuWeightBuffer(weight)) {
504
163
  return weight.layout === 'column';
@@ -560,107 +219,6 @@ function releasePleSliceBuffer(recorder, buffer, decodeBuffers, cache) {
560
219
  releaseOrTrack(recorder, buffer, decodeBuffers);
561
220
  }
562
221
 
563
- function getPleRowCachePolicy(sessionConfig) {
564
- const rowCache = sessionConfig?.rowCache ?? null;
565
- if (!rowCache || rowCache.mode === 'off') {
566
- return null;
567
- }
568
- if (rowCache.mode !== 'lru') {
569
- throw new Error(
570
- `Gemma 4 per-layer input row cache mode "${String(rowCache.mode)}" is not implemented.`
571
- );
572
- }
573
- const decodedDtype = String(rowCache.decodedDtype ?? '').toLowerCase();
574
- if (decodedDtype !== 'f32') {
575
- throw new Error(
576
- `Gemma 4 range-backed per-layer input row cache requires rowCache.decodedDtype="f32"; ` +
577
- `got "${String(rowCache.decodedDtype)}".`
578
- );
579
- }
580
- const maxRows = Math.trunc(Number(rowCache.maxRows));
581
- const maxBytes = Math.trunc(Number(rowCache.maxBytes));
582
- if (!Number.isFinite(maxRows) || maxRows <= 0) {
583
- throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxRows > 0.');
584
- }
585
- if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
586
- throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxBytes > 0.');
587
- }
588
- return { maxRows, maxBytes };
589
- }
590
-
591
- function getPleHotCachePolicy(sessionConfig) {
592
- const hotCache = sessionConfig?.hotCache ?? null;
593
- if (!hotCache || hotCache.mode === 'off') {
594
- return null;
595
- }
596
- if (hotCache.mode === 'tokenizer_scores') {
597
- const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
598
- if (outputDtype !== 'f16' && outputDtype !== 'f32') {
599
- throw new Error(
600
- `Gemma 4 per-layer input hot vocabulary cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
601
- `got "${String(hotCache.outputDtype)}".`
602
- );
603
- }
604
- const maxTokens = Math.trunc(Number(hotCache.maxTokens));
605
- const maxBytes = Math.trunc(Number(hotCache.maxBytes));
606
- if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
607
- throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxTokens > 0.');
608
- }
609
- if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
610
- throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxBytes > 0.');
611
- }
612
- return {
613
- mode: 'tokenizer_scores',
614
- maxTokens,
615
- maxBytes,
616
- outputDtype,
617
- };
618
- }
619
- if (hotCache.mode !== 'prepared_tokens') {
620
- throw new Error(
621
- `Gemma 4 per-layer input hot cache mode "${String(hotCache.mode)}" is not implemented.`
622
- );
623
- }
624
- const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
625
- if (outputDtype !== 'f16' && outputDtype !== 'f32') {
626
- throw new Error(
627
- `Gemma 4 per-layer input hot cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
628
- `got "${String(hotCache.outputDtype)}".`
629
- );
630
- }
631
- const maxTokens = Math.trunc(Number(hotCache.maxTokens));
632
- const maxBytes = Math.trunc(Number(hotCache.maxBytes));
633
- if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
634
- throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxTokens > 0.');
635
- }
636
- if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
637
- throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxBytes > 0.');
638
- }
639
- return { mode: 'prepared_tokens', maxTokens, maxBytes, outputDtype };
640
- }
641
-
642
- export function resolvePleHotVocabularyCapacity({
643
- maxTokens,
644
- maxBytes,
645
- numLayers,
646
- hiddenSize,
647
- bytesPerElement,
648
- vocabSize,
649
- }) {
650
- const bytesPerHotRow = numLayers * hiddenSize * bytesPerElement;
651
- const tokenIndexMapBytes = vocabSize * Uint32Array.BYTES_PER_ELEMENT;
652
- const tableBudgetBytes = Math.max(0, maxBytes - tokenIndexMapBytes);
653
- const maxTableRows = bytesPerHotRow > 0
654
- ? Math.floor(tableBudgetBytes / bytesPerHotRow)
655
- : 0;
656
- const maxHotTokens = Math.max(0, Math.min(maxTokens, maxTableRows - 1));
657
- return {
658
- maxHotTokens,
659
- bytesPerHotRow,
660
- tokenIndexMapBytes,
661
- };
662
- }
663
-
664
222
  export function getPleHotVocabularyRuntime(context) {
665
223
  const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
666
224
  if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
@@ -670,212 +228,6 @@ export function getPleHotVocabularyRuntime(context) {
670
228
  return runtime && typeof runtime === 'object' ? runtime : null;
671
229
  }
672
230
 
673
- function getPleSplitTablePolicy(sessionConfig) {
674
- if (sessionConfig?.materialization !== 'gpu_split_tables') {
675
- return null;
676
- }
677
- return { mode: 'gpu_split_tables' };
678
- }
679
-
680
- function releasePreparedTokenEntry(cache, tokenId, entry) {
681
- if (!cache || !entry) {
682
- return;
683
- }
684
- cache.preparedTokenEntries?.delete(tokenId);
685
- cache.preparedTokenBytes -= entry?.bytes ?? 0;
686
- for (const buffer of entry.buffers ?? []) {
687
- if (!buffer) continue;
688
- cache.ownedBuffers?.delete(buffer);
689
- releaseBuffer(buffer);
690
- }
691
- }
692
-
693
- function prunePreparedTokenCache(cache, policy) {
694
- while (
695
- cache.preparedTokenEntries.size > policy.maxTokens
696
- || cache.preparedTokenBytes > policy.maxBytes
697
- ) {
698
- const oldest = cache.preparedTokenEntries.keys().next();
699
- if (oldest.done) {
700
- break;
701
- }
702
- const tokenId = oldest.value;
703
- const entry = cache.preparedTokenEntries.get(tokenId);
704
- releasePreparedTokenEntry(cache, tokenId, entry);
705
- }
706
- }
707
-
708
- function getPreparedTokenEntry(cache, tokenId, sessionConfig, activationDtype, stats = null) {
709
- const policy = getPleHotCachePolicy(sessionConfig);
710
- if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
711
- return null;
712
- }
713
- if (policy.outputDtype !== activationDtype) {
714
- throw new Error(
715
- `Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
716
- `got "${String(activationDtype)}".`
717
- );
718
- }
719
- const entry = cache.preparedTokenEntries.get(tokenId) ?? null;
720
- if (!entry) {
721
- if (stats) {
722
- stats.plePreparedTokenCacheMisses = (stats.plePreparedTokenCacheMisses ?? 0) + 1;
723
- stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
724
- stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
725
- }
726
- return null;
727
- }
728
- cache.preparedTokenEntries.delete(tokenId);
729
- cache.preparedTokenEntries.set(tokenId, entry);
730
- if (stats) {
731
- stats.plePreparedTokenCacheHits = (stats.plePreparedTokenCacheHits ?? 0) + 1;
732
- stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
733
- stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
734
- }
735
- return entry.buffers.slice();
736
- }
737
-
738
- function storePreparedTokenEntry(cache, tokenId, buffers, sessionConfig, activationDtype, stats = null) {
739
- const policy = getPleHotCachePolicy(sessionConfig);
740
- if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
741
- return buffers;
742
- }
743
- if (policy.outputDtype !== activationDtype) {
744
- throw new Error(
745
- `Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
746
- `got "${String(activationDtype)}".`
747
- );
748
- }
749
- const existing = cache.preparedTokenEntries.get(tokenId) ?? null;
750
- if (existing) {
751
- releasePreparedTokenEntry(cache, tokenId, existing);
752
- }
753
- const cachedBuffers = buffers.slice();
754
- const bytes = cachedBuffers.reduce((total, buffer) => total + (buffer?.size ?? 0), 0);
755
- for (const buffer of cachedBuffers) {
756
- if (buffer) {
757
- cache.ownedBuffers?.add(buffer);
758
- }
759
- }
760
- cache.preparedTokenEntries.set(tokenId, { buffers: cachedBuffers, bytes });
761
- cache.preparedTokenBytes += bytes;
762
- prunePreparedTokenCache(cache, policy);
763
- if (stats) {
764
- stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
765
- stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
766
- }
767
- return cachedBuffers.slice();
768
- }
769
-
770
- function getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize) {
771
- const sourceDtype = requirePleSourceDtype(
772
- (isCpuWeightBuffer(embedTokensPerLayer) ? embedTokensPerLayer.data?.sourceDtype : null)
773
- ?? embedTokensPerLayer?.dtype,
774
- 'Gemma 4 range-backed per-layer input rows',
775
- ['f16', 'bf16', 'f32']
776
- );
777
- const bytesPerElement = (sourceDtype === 'f16' || sourceDtype === 'bf16') ? 2 : 4;
778
- return {
779
- sourceDtype,
780
- sourceRowBytes: totalPerLayerHiddenSize * bytesPerElement,
781
- };
782
- }
783
-
784
- function getPleRangeRowCache(embedTokensPerLayer, sessionConfig) {
785
- const policy = getPleRowCachePolicy(sessionConfig);
786
- if (!policy) {
787
- return null;
788
- }
789
- const cached = pleRangeRowCache.get(embedTokensPerLayer);
790
- if (cached && cached.maxRows === policy.maxRows && cached.maxBytes === policy.maxBytes) {
791
- return cached;
792
- }
793
- const next = {
794
- maxRows: policy.maxRows,
795
- maxBytes: policy.maxBytes,
796
- totalBytes: 0,
797
- rows: new Map(),
798
- };
799
- pleRangeRowCache.set(embedTokensPerLayer, next);
800
- return next;
801
- }
802
-
803
- function touchPleCachedRow(cache, tokenId) {
804
- const hit = cache?.rows?.get(tokenId) ?? null;
805
- if (!hit) {
806
- return null;
807
- }
808
- cache.rows.delete(tokenId);
809
- cache.rows.set(tokenId, hit);
810
- return hit.row;
811
- }
812
-
813
- function prunePleRangeRowCache(cache) {
814
- while (cache.rows.size > cache.maxRows || cache.totalBytes > cache.maxBytes) {
815
- const oldest = cache.rows.keys().next();
816
- if (oldest.done) {
817
- break;
818
- }
819
- const entry = cache.rows.get(oldest.value);
820
- cache.rows.delete(oldest.value);
821
- cache.totalBytes -= entry?.bytes ?? 0;
822
- }
823
- }
824
-
825
- function cachePleRangeRow(cache, tokenId, row) {
826
- if (!cache) {
827
- return row;
828
- }
829
- const existing = cache.rows.get(tokenId);
830
- if (existing) {
831
- cache.totalBytes -= existing.bytes;
832
- cache.rows.delete(tokenId);
833
- }
834
- cache.rows.set(tokenId, { row, bytes: row.byteLength });
835
- cache.totalBytes += row.byteLength;
836
- prunePleRangeRowCache(cache);
837
- return row;
838
- }
839
-
840
- async function loadRangeBackedPleRow(
841
- tokenId,
842
- embedTokensPerLayer,
843
- totalPerLayerHiddenSize,
844
- sessionConfig,
845
- label,
846
- prefetchedRow = null
847
- ) {
848
- if (!isCpuWeightBuffer(embedTokensPerLayer)) {
849
- return null;
850
- }
851
- const cpuData = embedTokensPerLayer.data;
852
- if (!isRangeBackedCpuEmbeddingSource(cpuData)) {
853
- return null;
854
- }
855
-
856
- const cache = getPleRangeRowCache(embedTokensPerLayer, sessionConfig);
857
- if (prefetchedRow && prefetchedRow.tokenId === tokenId) {
858
- return cachePleRangeRow(cache, tokenId, prefetchedRow.row);
859
- }
860
-
861
- const cached = touchPleCachedRow(cache, tokenId);
862
- if (cached) {
863
- return cached;
864
- }
865
-
866
- const { sourceDtype, sourceRowBytes } = getPleRangeRowLoadConfig(
867
- embedTokensPerLayer,
868
- totalPerLayerHiddenSize
869
- );
870
- const chunk = normalizeRangeBytes(
871
- await cpuData.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
872
- label
873
- );
874
- const row = new Float32Array(totalPerLayerHiddenSize);
875
- decodeRangeChunkIntoOutput(chunk, sourceDtype, row, 0, totalPerLayerHiddenSize);
876
- return cachePleRangeRow(cache, tokenId, row);
877
- }
878
-
879
231
  // Step 5: Prefetch next token's PLE row during current decode step.
880
232
  // Returns a promise resolving to { tokenId, row: Float32Array } or null.
881
233
  // Call after sampling produces the next token; pass result as options.prefetchedRow
@@ -934,391 +286,6 @@ export function hasGpuSplitPerLayerInputEmbeddings(context) {
934
286
  && perLayerInputWeights.embedTokensPerLayerSplit.length > 0;
935
287
  }
936
288
 
937
- export async function ensurePleGpuSplitTablesRuntime(context) {
938
- const policy = getPleSplitTablePolicy(context?.perLayerInputsSession ?? null);
939
- if (!policy) {
940
- return null;
941
- }
942
-
943
- const config = context?.config ?? null;
944
- const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
945
- const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
946
- const numLayers = Number(config?.numLayers ?? 0);
947
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
948
- return null;
949
- }
950
- if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
951
- return null;
952
- }
953
- if (!Number.isFinite(numLayers) || numLayers <= 0) {
954
- return null;
955
- }
956
-
957
- const perLayerInputWeights = getPerLayerInputWeights(context);
958
- if (Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit) && perLayerInputWeights.embedTokensPerLayerSplit.length === numLayers) {
959
- return perLayerInputWeights.embedTokensPerLayerSplit;
960
- }
961
-
962
- const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
963
- if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
964
- throw new Error('Gemma 4 gpu_split_tables materialization requires a range-backed CPU embedTokensPerLayer source.');
965
- }
966
-
967
- const sourceDtype = requirePleSourceDtype(
968
- embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
969
- 'Gemma 4 gpu_split_tables materialization'
970
- );
971
-
972
- const device = getDevice();
973
- if (!device) {
974
- throw new Error('No GPU device available for Gemma 4 gpu_split_tables materialization.');
975
- }
976
-
977
- const bytesPerElement = sourceDtype === 'f16' ? 2 : 4;
978
- const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
979
- const tableBytes = vocabSizePerLayerInput * hiddenSizePerLayerInput * bytesPerElement;
980
- const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
981
- acquireBuffer(tableBytes, undefined, `L${layerIdx}.ple_table_split`),
982
- sourceDtype,
983
- 'row',
984
- [vocabSizePerLayerInput, hiddenSizePerLayerInput],
985
- `L${layerIdx}.embed_tokens_per_layer_split`
986
- ));
987
-
988
- try {
989
- const rowsPerChunk = 128;
990
- for (let rowStart = 0; rowStart < vocabSizePerLayerInput; rowStart += rowsPerChunk) {
991
- const rowCount = Math.min(rowsPerChunk, vocabSizePerLayerInput - rowStart);
992
- const chunkByteOffset = rowStart * totalPerLayerHiddenSize * bytesPerElement;
993
- const chunkByteLength = rowCount * totalPerLayerHiddenSize * bytesPerElement;
994
- const chunk = normalizeRangeBytes(
995
- await embedTokensPerLayer.data.loadRange(chunkByteOffset, chunkByteLength),
996
- 'Gemma 4 split GPU PLE chunk'
997
- );
998
-
999
- if (sourceDtype === 'f16') {
1000
- const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
1001
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1002
- const layerWords = new Uint16Array(rowCount * hiddenSizePerLayerInput);
1003
- for (let row = 0; row < rowCount; row += 1) {
1004
- const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
1005
- layerWords.set(
1006
- sourceWords.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
1007
- row * hiddenSizePerLayerInput
1008
- );
1009
- }
1010
- device.queue.writeBuffer(
1011
- splitTables[layerIdx].buffer,
1012
- rowStart * hiddenSizePerLayerInput * bytesPerElement,
1013
- layerWords.buffer,
1014
- layerWords.byteOffset,
1015
- layerWords.byteLength
1016
- );
1017
- }
1018
- } else {
1019
- const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
1020
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1021
- const layerValues = new Float32Array(rowCount * hiddenSizePerLayerInput);
1022
- for (let row = 0; row < rowCount; row += 1) {
1023
- const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
1024
- layerValues.set(
1025
- sourceValues.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
1026
- row * hiddenSizePerLayerInput
1027
- );
1028
- }
1029
- device.queue.writeBuffer(
1030
- splitTables[layerIdx].buffer,
1031
- rowStart * hiddenSizePerLayerInput * bytesPerElement,
1032
- layerValues.buffer,
1033
- layerValues.byteOffset,
1034
- layerValues.byteLength
1035
- );
1036
- }
1037
- }
1038
- }
1039
- } catch (error) {
1040
- for (const table of splitTables) {
1041
- releaseBuffer(table.buffer);
1042
- }
1043
- throw error;
1044
- }
1045
-
1046
- perLayerInputWeights.embedTokensPerLayerSplit = splitTables;
1047
- return splitTables;
1048
- }
1049
-
1050
- function resolvePleHotVocabularySeedTokenIds(context, maxTokens, vocabSizePerLayerInput) {
1051
- const rawSeedTokenIds = Array.isArray(context?.seedTokenIds) ? context.seedTokenIds : null;
1052
- if (!rawSeedTokenIds || rawSeedTokenIds.length === 0) {
1053
- return [];
1054
- }
1055
-
1056
- const specialTokenIds = new Set();
1057
- const tokenizerSpecialTokens = context?.tokenizer?.getSpecialTokens?.() ?? null;
1058
- if (tokenizerSpecialTokens && typeof tokenizerSpecialTokens === 'object') {
1059
- for (const value of Object.values(tokenizerSpecialTokens)) {
1060
- if (Number.isInteger(value)) {
1061
- specialTokenIds.add(value);
1062
- }
1063
- }
1064
- }
1065
-
1066
- const seeded = [];
1067
- const seen = new Set();
1068
- for (let i = rawSeedTokenIds.length - 1; i >= 0; i -= 1) {
1069
- const tokenId = rawSeedTokenIds[i];
1070
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput) {
1071
- continue;
1072
- }
1073
- if (specialTokenIds.has(tokenId) || seen.has(tokenId)) {
1074
- continue;
1075
- }
1076
- seeded.push(tokenId);
1077
- seen.add(tokenId);
1078
- if (seeded.length >= maxTokens) {
1079
- break;
1080
- }
1081
- }
1082
- return seeded;
1083
- }
1084
-
1085
- function mergePleHotVocabularyTokenIds(seedTokenIds, tokenizerHotTokenIds, maxTokens, vocabSizePerLayerInput) {
1086
- const merged = [];
1087
- const seen = new Set();
1088
-
1089
- for (const tokenId of seedTokenIds ?? []) {
1090
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
1091
- continue;
1092
- }
1093
- merged.push(tokenId);
1094
- seen.add(tokenId);
1095
- if (merged.length >= maxTokens) {
1096
- return merged;
1097
- }
1098
- }
1099
-
1100
- for (const tokenId of tokenizerHotTokenIds ?? []) {
1101
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
1102
- continue;
1103
- }
1104
- merged.push(tokenId);
1105
- seen.add(tokenId);
1106
- if (merged.length >= maxTokens) {
1107
- break;
1108
- }
1109
- }
1110
-
1111
- return merged;
1112
- }
1113
-
1114
- export async function ensurePleGpuHotVocabularyRuntime(context) {
1115
- const policy = getPleHotCachePolicy(context?.perLayerInputsSession ?? null);
1116
- if (!policy || policy.mode !== 'tokenizer_scores') {
1117
- return null;
1118
- }
1119
-
1120
- const config = context?.config ?? null;
1121
- const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
1122
- const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
1123
- const numLayers = Number(config?.numLayers ?? 0);
1124
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
1125
- return null;
1126
- }
1127
- if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
1128
- return null;
1129
- }
1130
- if (!Number.isFinite(numLayers) || numLayers <= 0) {
1131
- return null;
1132
- }
1133
-
1134
- const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
1135
- const capacity = resolvePleHotVocabularyCapacity({
1136
- maxTokens: policy.maxTokens,
1137
- maxBytes: policy.maxBytes,
1138
- numLayers,
1139
- hiddenSize: hiddenSizePerLayerInput,
1140
- bytesPerElement,
1141
- vocabSize: vocabSizePerLayerInput,
1142
- });
1143
- if (capacity.maxHotTokens <= 0) {
1144
- return null;
1145
- }
1146
-
1147
- const tokenizer = context?.tokenizer ?? null;
1148
- const tokenizerHotTokenIds = typeof tokenizer?.getHotTokenIds === 'function'
1149
- ? tokenizer.getHotTokenIds(capacity.maxHotTokens)
1150
- : null;
1151
- const seedTokenIds = resolvePleHotVocabularySeedTokenIds(
1152
- context,
1153
- capacity.maxHotTokens,
1154
- vocabSizePerLayerInput
1155
- );
1156
- const hotTokenIds = mergePleHotVocabularyTokenIds(
1157
- seedTokenIds,
1158
- tokenizerHotTokenIds,
1159
- capacity.maxHotTokens,
1160
- vocabSizePerLayerInput
1161
- );
1162
- if (hotTokenIds.length === 0) {
1163
- return null;
1164
- }
1165
- const hotTokenIdsSignature = hotTokenIds.join(',');
1166
-
1167
- const perLayerInputWeights = getPerLayerInputWeights(context);
1168
- const cached = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
1169
- if (
1170
- cached
1171
- && cached.maxTokens === capacity.maxHotTokens
1172
- && cached.maxBytes === policy.maxBytes
1173
- && cached.outputDtype === policy.outputDtype
1174
- && cached.vocabSize === vocabSizePerLayerInput
1175
- && cached.numLayers === numLayers
1176
- && cached.hotTokenIdsSignature === hotTokenIdsSignature
1177
- ) {
1178
- return cached;
1179
- }
1180
- destroyPleHotVocabularyRuntime(perLayerInputWeights);
1181
-
1182
- const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
1183
- if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
1184
- return null;
1185
- }
1186
-
1187
- const sourceDtype = requirePleSourceDtype(
1188
- embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
1189
- 'Gemma 4 hot vocabulary cache'
1190
- );
1191
- const expandsF16ToF32 = sourceDtype === 'f16' && policy.outputDtype === 'f32';
1192
- if (sourceDtype !== policy.outputDtype && !expandsF16ToF32) {
1193
- throw new Error(
1194
- `Gemma 4 hot vocabulary cache cannot convert source dtype "${sourceDtype}" ` +
1195
- `to output dtype "${policy.outputDtype}".`
1196
- );
1197
- }
1198
-
1199
- const device = getDevice();
1200
- if (!device) {
1201
- throw new Error('No GPU device available for Gemma 4 hot vocabulary cache.');
1202
- }
1203
-
1204
- const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
1205
- const sentinelIndex = hotTokenIds.length;
1206
- const hotRowCount = sentinelIndex + 1;
1207
- const splitTableBytes = hotRowCount * hiddenSizePerLayerInput * bytesPerElement;
1208
- const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
1209
- acquireBuffer(splitTableBytes, undefined, `L${layerIdx}.ple_hot_vocab_table`),
1210
- policy.outputDtype,
1211
- 'row',
1212
- [hotRowCount, hiddenSizePerLayerInput],
1213
- `L${layerIdx}.embed_tokens_per_layer_hot_vocab`
1214
- ));
1215
- const hotTokenIndexMap = new Uint32Array(vocabSizePerLayerInput);
1216
- hotTokenIndexMap.fill(sentinelIndex);
1217
- for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
1218
- const tokenId = hotTokenIds[hotIndex];
1219
- if (Number.isInteger(tokenId) && tokenId >= 0 && tokenId < vocabSizePerLayerInput) {
1220
- hotTokenIndexMap[tokenId] = hotIndex;
1221
- }
1222
- }
1223
- const hotTokenIndexMapBuffer = acquireBuffer(
1224
- hotTokenIndexMap.byteLength,
1225
- undefined,
1226
- 'ple_hot_token_index_map'
1227
- );
1228
-
1229
- try {
1230
- device.queue.writeBuffer(hotTokenIndexMapBuffer, 0, hotTokenIndexMap);
1231
- const zeroRow = policy.outputDtype === 'f16'
1232
- ? new Uint16Array(hiddenSizePerLayerInput)
1233
- : new Float32Array(hiddenSizePerLayerInput);
1234
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1235
- device.queue.writeBuffer(
1236
- splitTables[layerIdx].buffer,
1237
- sentinelIndex * hiddenSizePerLayerInput * bytesPerElement,
1238
- zeroRow.buffer,
1239
- zeroRow.byteOffset,
1240
- zeroRow.byteLength
1241
- );
1242
- }
1243
-
1244
- const { sourceRowBytes } = getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize);
1245
- const expandedF32Row = expandsF16ToF32
1246
- ? new Float32Array(totalPerLayerHiddenSize)
1247
- : null;
1248
- const f16Lookup = expandsF16ToF32 ? getF16ToF32Lookup() : null;
1249
- for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
1250
- const tokenId = hotTokenIds[hotIndex];
1251
- const chunk = normalizeRangeBytes(
1252
- await embedTokensPerLayer.data.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
1253
- 'Gemma 4 hot vocabulary PLE row'
1254
- );
1255
- if (policy.outputDtype === 'f16') {
1256
- const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1257
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1258
- const sourceStart = layerIdx * hiddenSizePerLayerInput;
1259
- device.queue.writeBuffer(
1260
- splitTables[layerIdx].buffer,
1261
- hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1262
- sourceWords.buffer,
1263
- sourceWords.byteOffset + sourceStart * bytesPerElement,
1264
- hiddenSizePerLayerInput * bytesPerElement
1265
- );
1266
- }
1267
- } else if (sourceDtype === 'f16') {
1268
- const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1269
- for (let valueIdx = 0; valueIdx < totalPerLayerHiddenSize; valueIdx += 1) {
1270
- expandedF32Row[valueIdx] = f16Lookup[sourceWords[valueIdx]];
1271
- }
1272
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1273
- const sourceStart = layerIdx * hiddenSizePerLayerInput;
1274
- device.queue.writeBuffer(
1275
- splitTables[layerIdx].buffer,
1276
- hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1277
- expandedF32Row.buffer,
1278
- sourceStart * bytesPerElement,
1279
- hiddenSizePerLayerInput * bytesPerElement
1280
- );
1281
- }
1282
- } else {
1283
- const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1284
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1285
- const sourceStart = layerIdx * hiddenSizePerLayerInput;
1286
- device.queue.writeBuffer(
1287
- splitTables[layerIdx].buffer,
1288
- hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1289
- sourceValues.buffer,
1290
- sourceValues.byteOffset + sourceStart * bytesPerElement,
1291
- hiddenSizePerLayerInput * bytesPerElement
1292
- );
1293
- }
1294
- }
1295
- }
1296
- } catch (error) {
1297
- for (const table of splitTables) {
1298
- releaseBuffer(table.buffer);
1299
- }
1300
- releaseBuffer(hotTokenIndexMapBuffer);
1301
- throw error;
1302
- }
1303
-
1304
- const runtime = {
1305
- mode: 'tokenizer_scores',
1306
- maxTokens: capacity.maxHotTokens,
1307
- maxBytes: policy.maxBytes,
1308
- outputDtype: policy.outputDtype,
1309
- vocabSize: vocabSizePerLayerInput,
1310
- numLayers,
1311
- hotTokenIdsSignature,
1312
- hotTokenIds: Uint32Array.from(hotTokenIds),
1313
- hotTokenIndexMap,
1314
- hotTokenIndexMapBuffer,
1315
- sentinelIndex,
1316
- splitTables,
1317
- };
1318
- perLayerInputWeights.embedTokensPerLayerHotRuntime = runtime;
1319
- return runtime;
1320
- }
1321
-
1322
289
  export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context, options = {}) {
1323
290
  const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
1324
291
  const hiddenSizePerLayerInput = Number(config.hiddenSizePerLayerInput ?? 0);