doppler-gpu 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (953) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +106 -87
  6. package/src/capsule-runtime.d.ts +25 -0
  7. package/src/capsule-runtime.js +24 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +2 -0
  24. package/src/client/capsule-host.d.ts +5 -0
  25. package/src/client/capsule-host.js +2 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +38 -1
  28. package/src/client/doppler-api.js +41 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +323 -0
  45. package/src/client/{runtime → model-host}/model-session.js +202 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +69 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +101 -0
  64. package/src/client/runtime/capsule-program-adapter.js +153 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +89 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +77 -0
  72. package/src/client/runtime/composition-root.js +352 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +19 -0
  92. package/src/client/runtime/session-controller.js +92 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +101 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +12 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +63 -0
  109. package/src/config/capsule-operations.json +17 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  128. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +2 -2
  129. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +62 -4
  130. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  132. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +3 -3
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +3 -3
  134. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +3 -3
  135. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +3 -3
  136. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +3 -3
  137. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +3 -3
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +5 -5
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +4 -4
  140. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +6 -6
  141. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +3 -3
  142. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  143. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +6 -6
  145. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +3 -3
  146. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +3 -3
  147. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  148. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  149. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  150. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  151. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  152. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +3 -3
  153. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  154. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  155. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +6 -6
  157. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +6 -6
  158. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +6 -6
  160. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +7 -7
  161. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +6 -6
  162. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  163. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +3 -3
  164. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +3 -3
  165. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  166. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +60 -8
  167. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  168. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  169. package/src/config/dtype-transition-contract.js +40 -0
  170. package/src/config/embedding-contract.js +26 -0
  171. package/src/config/embedding-reference.js +85 -0
  172. package/src/config/evidence/source-intake-policy.json +1 -1
  173. package/src/config/execution-contract-check.d.ts +16 -73
  174. package/src/config/execution-contract-check.js +7 -429
  175. package/src/config/forecast-manifest.js +56 -0
  176. package/src/config/generation-contract.d.ts +39 -0
  177. package/src/config/generation-contract.js +66 -0
  178. package/src/config/generation-contract.json +32 -0
  179. package/src/config/heterogeneous-attention-contract.js +46 -0
  180. package/src/config/initial-execution-identity.d.ts +58 -0
  181. package/src/config/initial-execution-identity.js +331 -0
  182. package/src/config/kernel-path-loader.d.ts +17 -0
  183. package/src/config/kernel-path-loader.js +42 -8
  184. package/src/config/kernel-registry-contract.d.ts +25 -0
  185. package/src/config/kernel-registry-contract.js +46 -0
  186. package/src/config/kernels/kernel-ref-digests.js +66 -18
  187. package/src/config/kernels/registry.js +1 -1
  188. package/src/config/kernels/registry.json +13617 -11377
  189. package/src/config/lora-layouts.d.ts +10 -0
  190. package/src/config/lora-layouts.js +17 -0
  191. package/src/config/lora-layouts.json +9 -0
  192. package/src/config/manifest-inference-validation.js +575 -0
  193. package/src/config/merge/execution-patches.js +12 -0
  194. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  195. package/src/config/merge/value-selection.js +38 -0
  196. package/src/config/merge.js +43 -11
  197. package/src/config/model-ir-v2.d.ts +57 -0
  198. package/src/config/model-ir-v2.js +258 -0
  199. package/src/config/model-ir.d.ts +49 -0
  200. package/src/config/model-ir.js +180 -0
  201. package/src/config/param-validator.js +1 -1
  202. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  203. package/src/config/platforms/loader.js +2 -1
  204. package/src/config/production-release-evidence.d.ts +23 -0
  205. package/src/config/production-release-evidence.js +423 -0
  206. package/src/config/production-release.js +321 -0
  207. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  208. package/src/config/required-inference-fields-contract-check.js +2 -2
  209. package/src/config/rerank-reference.d.ts +28 -0
  210. package/src/config/rerank-reference.js +89 -0
  211. package/src/config/revocation-policy.d.ts +65 -0
  212. package/src/config/revocation-policy.js +290 -0
  213. package/src/config/revocation-registry.json +11 -0
  214. package/src/config/revocation-updates.d.ts +68 -0
  215. package/src/config/revocation-updates.js +440 -0
  216. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  217. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  218. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  219. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  220. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  221. package/src/config/runtime-merge.js +1 -1
  222. package/src/config/runtime.js +1 -1
  223. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  224. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  225. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  226. package/src/config/schema/conversion-report.schema.js +29 -0
  227. package/src/config/schema/converter.schema.d.ts +1 -0
  228. package/src/config/schema/debug.schema.d.ts +1 -0
  229. package/src/config/schema/doppler.schema.js +6 -5
  230. package/src/config/schema/emulation.schema.js +1 -1
  231. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  232. package/src/config/schema/execution-v1.schema.js +19 -0
  233. package/src/config/schema/harness.schema.d.ts +13 -0
  234. package/src/config/schema/harness.schema.js +6 -0
  235. package/src/config/schema/inference.schema.d.ts +21 -2
  236. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  237. package/src/config/schema/manifest.schema.d.ts +26 -7
  238. package/src/config/schema/manifest.schema.js +10 -1
  239. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  240. package/src/config/schema/program-bundle.schema.js +145 -10
  241. package/src/config/sequence-reference.d.ts +26 -0
  242. package/src/config/sequence-reference.js +100 -0
  243. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  244. package/src/config/target-plan.d.ts +108 -0
  245. package/src/config/target-plan.js +411 -0
  246. package/src/config/training-defaults.js +1 -1
  247. package/src/config/transforms/capability-transform-resolver.js +4 -105
  248. package/src/config/transforms/execution-graph/dtype.js +591 -0
  249. package/src/config/transforms/execution-graph/fusion.js +436 -0
  250. package/src/config/transforms/execution-graph/validation.js +414 -0
  251. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  252. package/src/converter/artifact-identity.js +475 -0
  253. package/src/converter/conversion-plan.js +1 -1
  254. package/src/converter/core.js +94 -2096
  255. package/src/converter/forge-candidate-evaluation.js +271 -0
  256. package/src/converter/forge-qualification.js +163 -0
  257. package/src/converter/forge-stages.js +800 -0
  258. package/src/converter/manifest-builder.js +683 -0
  259. package/src/converter/quantization-info.js +4 -1
  260. package/src/converter/quantizer.js +76 -13
  261. package/src/converter/shard-packer.d.ts +1 -1
  262. package/src/converter/shard-packer.js +2 -2
  263. package/src/converter/tensor-transform.js +640 -0
  264. package/src/converter/tokenizer-contract.js +243 -0
  265. package/src/converter/tokenizer-manifest.js +243 -0
  266. package/src/debug/history.js +0 -8
  267. package/src/debug/index.js +0 -10
  268. package/src/debug/tensor.js +1 -1
  269. package/src/debug/trace.js +0 -7
  270. package/src/experimental/adapters/adapter-manager.js +2 -0
  271. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  272. package/src/experimental/adapters/adapter-manifest.js +11 -0
  273. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  274. package/src/experimental/adapters/artifact-contract.js +5 -2
  275. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  276. package/src/experimental/adapters/lora-loader.js +110 -48
  277. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  278. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  279. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  280. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  281. package/src/experimental/browser/browser-converter.js +11 -11
  282. package/src/experimental/diffusion/index.d.ts +1 -1
  283. package/src/experimental/diffusion/index.js +1 -1
  284. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  285. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  286. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  287. package/src/experimental/diffusion/vae.js +596 -0
  288. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  289. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  290. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  291. package/src/experimental/distribution/shard-delivery.js +4 -1709
  292. package/src/experimental/energy/index.d.ts +2 -2
  293. package/src/experimental/energy/index.js +2 -2
  294. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  295. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  296. package/src/experimental/logits/cpu-output.d.ts +27 -0
  297. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  298. package/src/experimental/logits/cpu.d.ts +87 -0
  299. package/src/experimental/logits/cpu.js +95 -0
  300. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  301. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  302. package/src/experimental/orchestration/index.d.ts +1 -1
  303. package/src/experimental/orchestration/index.js +1 -1
  304. package/src/experimental/training/artifacts.js +2 -2
  305. package/src/experimental/training/checkpoint.js +2 -2
  306. package/src/experimental/training/distillation/artifacts.js +1 -1
  307. package/src/experimental/training/distillation/dataset.js +2 -2
  308. package/src/experimental/training/distillation/eval.js +1 -1
  309. package/src/experimental/training/distillation/runtime.js +2 -2
  310. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  311. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  312. package/src/experimental/training/distillation/student-fixture.js +2 -469
  313. package/src/experimental/training/lora/checkpoint.js +746 -0
  314. package/src/experimental/training/lora/execution.js +550 -0
  315. package/src/experimental/training/lora/plan.js +387 -0
  316. package/src/experimental/training/lora/recovery.js +418 -0
  317. package/src/experimental/training/lora-pipeline.js +7 -1949
  318. package/src/experimental/training/operator-artifacts.js +2 -2
  319. package/src/experimental/training/operator-command.js +4 -4
  320. package/src/experimental/training/runner/evaluation.js +62 -0
  321. package/src/experimental/training/runner/lifecycle.js +257 -0
  322. package/src/experimental/training/runner.js +5 -268
  323. package/src/experimental/training/suite/execution.js +879 -0
  324. package/src/experimental/training/suite/plan.js +855 -0
  325. package/src/experimental/training/suite.js +5 -1659
  326. package/src/experimental/training/workloads.d.ts +5 -5
  327. package/src/experimental/training/workloads.js +6 -6
  328. package/src/experimental/turboquant/codebook.js +470 -0
  329. package/src/formats/quantization-constants.d.ts +11 -0
  330. package/src/formats/quantization-constants.js +11 -0
  331. package/src/formats/rdrr/integrity.js +1 -1
  332. package/src/formats/rdrr/lowerings.js +0 -29
  333. package/src/formats/rdrr/merkle.js +1 -1
  334. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  335. package/src/formats/rdrr/validation.js +3 -3
  336. package/src/gpu/command-recorder.d.ts +10 -1
  337. package/src/gpu/command-recorder.js +58 -14
  338. package/src/gpu/constant-buffer.js +29 -0
  339. package/src/gpu/device-state.js +44 -0
  340. package/src/gpu/device.js +11 -41
  341. package/src/gpu/kernel-runtime.js +2 -1
  342. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  343. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  344. package/src/gpu/kernels/attention/executors.js +960 -0
  345. package/src/gpu/kernels/attention/plan.js +472 -0
  346. package/src/gpu/kernels/attention.d.ts +4 -0
  347. package/src/gpu/kernels/attention.js +145 -1545
  348. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  349. package/src/gpu/kernels/backward/adam.js +2 -1
  350. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  351. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  352. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  353. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  354. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  355. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  356. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  357. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  358. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  359. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  360. package/src/gpu/kernels/backward/executor.js +158 -0
  361. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  362. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  363. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  364. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  365. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  366. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  367. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  368. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  369. package/src/gpu/kernels/backward/matmul.js +133 -0
  370. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  371. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  372. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  374. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  376. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  377. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  378. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  379. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  380. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  381. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  382. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  383. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  384. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  385. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  386. package/src/gpu/kernels/backward/utils.js +2 -392
  387. package/src/gpu/kernels/bind-group-validation.js +16 -0
  388. package/src/gpu/kernels/cast.js +2 -1
  389. package/src/gpu/kernels/check-finiteness.js +13 -49
  390. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  391. package/src/gpu/kernels/check-stop.js +14 -33
  392. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  393. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  394. package/src/gpu/kernels/check_stop.wgsl +20 -0
  395. package/src/gpu/kernels/clamp.js +1 -1
  396. package/src/gpu/kernels/conv2d.js +1 -1
  397. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  398. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  399. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  400. package/src/gpu/kernels/dequant.js +7 -2
  401. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  402. package/src/gpu/kernels/dispatch.js +12 -1
  403. package/src/gpu/kernels/embedding-pool.js +72 -0
  404. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  405. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  406. package/src/gpu/kernels/energy.js +2 -1
  407. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  408. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  409. package/src/gpu/kernels/fused_ffn.js +10 -220
  410. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  411. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  412. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  413. package/src/gpu/kernels/gated-short-conv.js +11 -69
  414. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  415. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  416. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  417. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  418. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  419. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  420. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  421. package/src/gpu/kernels/gather.d.ts +1 -1
  422. package/src/gpu/kernels/gather.js +30 -6
  423. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  424. package/src/gpu/kernels/gelu.d.ts +4 -0
  425. package/src/gpu/kernels/gelu.js +12 -2
  426. package/src/gpu/kernels/gelu.wgsl +16 -0
  427. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  428. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  429. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  430. package/src/gpu/kernels/groupnorm.js +1 -1
  431. package/src/gpu/kernels/index.d.ts +38 -5
  432. package/src/gpu/kernels/index.js +18 -4
  433. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  434. package/src/gpu/kernels/kernel-base.js +1 -1
  435. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  436. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  437. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  438. package/src/gpu/kernels/kernel-configs.js +26 -54
  439. package/src/gpu/kernels/kernel-execution.js +141 -0
  440. package/src/gpu/kernels/kv-cache-write.js +4 -4
  441. package/src/gpu/kernels/kv-quantize.js +4 -1
  442. package/src/gpu/kernels/l2-normalize.js +36 -0
  443. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  444. package/src/gpu/kernels/layernorm.d.ts +4 -4
  445. package/src/gpu/kernels/layernorm.js +44 -7
  446. package/src/gpu/kernels/linear-attention-core.js +29 -449
  447. package/src/gpu/kernels/linear_attention.js +1 -1
  448. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  449. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  450. package/src/gpu/kernels/logit-finalize.js +103 -0
  451. package/src/gpu/kernels/logit-merge.js +13 -125
  452. package/src/gpu/kernels/logit-scatter.js +40 -0
  453. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  454. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  455. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  456. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  457. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  458. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  459. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  460. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  461. package/src/gpu/kernels/matmul-selection.js +3 -2
  462. package/src/gpu/kernels/matmul.js +4 -15
  463. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  464. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  465. package/src/gpu/kernels/modulate.js +1 -1
  466. package/src/gpu/kernels/moe.js +3 -1
  467. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  468. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  469. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  470. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  471. package/src/gpu/kernels/patch_context.wgsl +29 -0
  472. package/src/gpu/kernels/pipeline-cache.js +3 -1
  473. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  474. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  475. package/src/gpu/kernels/relu.js +1 -1
  476. package/src/gpu/kernels/rep-penalty.js +30 -5
  477. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  478. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  479. package/src/gpu/kernels/repeat_channels.js +1 -1
  480. package/src/gpu/kernels/residual.js +2 -2
  481. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  482. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  483. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  484. package/src/gpu/kernels/rmsnorm.js +1 -1
  485. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  486. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  487. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  488. package/src/gpu/kernels/rope-precompute.js +160 -0
  489. package/src/gpu/kernels/rope-qk.js +1 -1
  490. package/src/gpu/kernels/rope.js +2 -2
  491. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  492. package/src/gpu/kernels/sample.d.ts +1 -0
  493. package/src/gpu/kernels/sample.js +36 -17
  494. package/src/gpu/kernels/sample.wgsl +112 -157
  495. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  496. package/src/gpu/kernels/scale.js +1 -1
  497. package/src/gpu/kernels/shader-cache.js +8 -4
  498. package/src/gpu/kernels/shader-source-scope.js +68 -0
  499. package/src/gpu/kernels/silu.js +2 -1
  500. package/src/gpu/kernels/soft-embedding.js +3 -1
  501. package/src/gpu/kernels/softmax.js +4 -2
  502. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  503. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  504. package/src/gpu/kernels/split_qg.js +1 -1
  505. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  506. package/src/gpu/kernels/split_qkv.js +18 -3
  507. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  508. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  509. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  510. package/src/gpu/kernels/transpose.js +1 -1
  511. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  512. package/src/gpu/kernels/uniform-utils.js +7 -2
  513. package/src/gpu/kernels/upsample2d.js +1 -1
  514. package/src/gpu/kernels/utils.js +9 -330
  515. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  516. package/src/gpu/kernels/vision-average-pool.js +68 -0
  517. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  518. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  519. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  520. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  521. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  522. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  523. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  524. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  525. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  526. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  527. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  528. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  529. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  530. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  531. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  532. package/src/gpu/readback-buffer.js +20 -0
  533. package/src/gpu/submit-probe.js +2 -26
  534. package/src/gpu/weight-buffer.d.ts +1 -0
  535. package/src/gpu/weight-buffer.js +7 -0
  536. package/src/index.d.ts +12 -0
  537. package/src/index.js +2 -0
  538. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  539. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  540. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  541. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  542. package/src/inference/browser-harness/inference-execution.js +543 -0
  543. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  544. package/src/inference/browser-harness/model-resolution.js +137 -0
  545. package/src/inference/browser-harness/report.js +338 -0
  546. package/src/inference/browser-harness/request.js +263 -0
  547. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  548. package/src/inference/browser-harness/runtime-config.js +255 -0
  549. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  550. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  551. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  552. package/src/inference/browser-harness/runtime-report.js +117 -0
  553. package/src/inference/browser-harness/suite-summary.js +39 -0
  554. package/src/inference/browser-harness/text-evidence.js +694 -0
  555. package/src/inference/browser-harness/text-execution.js +757 -0
  556. package/src/inference/browser-harness/text-input.js +725 -0
  557. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  558. package/src/inference/browser-harness-contract-helpers.js +1 -1
  559. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  560. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  561. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  562. package/src/inference/browser-harness-text-helpers.js +31 -2199
  563. package/src/inference/browser-harness.d.ts +15 -2
  564. package/src/inference/browser-harness.js +38 -1952
  565. package/src/inference/kv-cache/base.js +67 -381
  566. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  567. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  568. package/src/inference/kv-cache/quantized.d.ts +2 -2
  569. package/src/inference/kv-cache/quantized.js +4 -8
  570. package/src/inference/kv-cache/update.js +309 -0
  571. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  572. package/src/inference/model-checkpoint-evidence.js +264 -0
  573. package/src/inference/moe-router.js +36 -216
  574. package/src/inference/multi-model-network.js +5 -341
  575. package/src/inference/pipelines/audio/index.js +1 -20
  576. package/src/inference/pipelines/audio/mel.js +0 -38
  577. package/src/inference/pipelines/context.js +6 -2
  578. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  579. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  580. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  581. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  582. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  583. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  584. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  585. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  586. package/src/inference/pipelines/factory.js +14 -3
  587. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  588. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  589. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -124
  590. package/src/inference/pipelines/text/abort-contract.js +22 -0
  591. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  592. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  593. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  594. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  595. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  596. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  597. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  598. package/src/inference/pipelines/text/attention/plan.js +17 -0
  599. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  600. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  601. package/src/inference/pipelines/text/attention/projections.js +20 -537
  602. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  603. package/src/inference/pipelines/text/attention/record.js +0 -6
  604. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  605. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  606. package/src/inference/pipelines/text/attention/run.js +0 -6
  607. package/src/inference/pipelines/text/attention/types.js +3 -31
  608. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  609. package/src/inference/pipelines/text/chat-format.js +42 -0
  610. package/src/inference/pipelines/text/config/normalization.js +729 -0
  611. package/src/inference/pipelines/text/config/validation.js +9 -0
  612. package/src/inference/pipelines/text/config.d.ts +12 -1
  613. package/src/inference/pipelines/text/config.js +14 -1228
  614. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  615. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  616. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  617. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  618. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  619. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  620. package/src/inference/pipelines/text/embed.js +178 -241
  621. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  622. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  623. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  624. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  625. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  626. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  627. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  628. package/src/inference/pipelines/text/execution-v1.js +7 -283
  629. package/src/inference/pipelines/text/execution.js +517 -0
  630. package/src/inference/pipelines/text/ffn/dense-executor.js +723 -0
  631. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  632. package/src/inference/pipelines/text/ffn/dense.js +6 -970
  633. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  634. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  635. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  636. package/src/inference/pipelines/text/generator/decode-runtime.js +985 -0
  637. package/src/inference/pipelines/text/generator/decode.js +964 -0
  638. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  639. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  640. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  641. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  642. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  643. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  644. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  645. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  646. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  647. package/src/inference/pipelines/text/generator/text.js +448 -0
  648. package/src/inference/pipelines/text/generator/training.js +753 -0
  649. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  650. package/src/inference/pipelines/text/generator-logits-observation.js +48 -0
  651. package/src/inference/pipelines/text/generator-runtime.js +23 -154
  652. package/src/inference/pipelines/text/generator-steps.js +45 -980
  653. package/src/inference/pipelines/text/generator.js +179 -3552
  654. package/src/inference/pipelines/text/image-transcription.js +304 -0
  655. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  656. package/src/inference/pipelines/text/init-rope.js +37 -189
  657. package/src/inference/pipelines/text/init.d.ts +8 -3
  658. package/src/inference/pipelines/text/init.js +6 -216
  659. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  660. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  661. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  662. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  663. package/src/inference/pipelines/text/layer-plan.js +45 -3
  664. package/src/inference/pipelines/text/layer.js +52 -522
  665. package/src/inference/pipelines/text/lifecycle.js +839 -0
  666. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  667. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  668. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  669. package/src/inference/pipelines/text/linear-attention.js +28 -534
  670. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  671. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  672. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  673. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  674. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  675. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  676. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  677. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  678. package/src/inference/pipelines/text/logits/index.js +62 -102
  679. package/src/inference/pipelines/text/logits/output-transform.js +54 -0
  680. package/src/inference/pipelines/text/logits/plan.js +184 -0
  681. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  682. package/src/inference/pipelines/text/logits/readback.js +9 -0
  683. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  684. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  685. package/src/inference/pipelines/text/lora-apply.js +12 -4
  686. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  687. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  688. package/src/inference/pipelines/text/model-load.js +0 -20
  689. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  690. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  691. package/src/inference/pipelines/text/moe/plan.js +47 -0
  692. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  693. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  694. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  695. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  696. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  697. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  698. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  699. package/src/inference/pipelines/text/probes.js +20 -0
  700. package/src/inference/pipelines/text/sampling-config.js +22 -113
  701. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  702. package/src/inference/pipelines/text/sampling.js +18 -375
  703. package/src/inference/pipelines/text/state.d.ts +8 -0
  704. package/src/inference/pipelines/text/state.js +1 -0
  705. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  706. package/src/inference/pipelines/text/types.d.ts +7 -3
  707. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  708. package/src/inference/pipelines/text.d.ts +2 -2
  709. package/src/inference/pipelines/text.js +46 -1633
  710. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  711. package/src/inference/pipelines/video/index.js +0 -21
  712. package/src/inference/pipelines/vision/attention.js +59 -0
  713. package/src/inference/pipelines/vision/encoder.js +254 -369
  714. package/src/inference/pipelines/vision/gemma4.js +204 -318
  715. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  716. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  717. package/src/inference/pipelines/vision/glmocr.js +557 -0
  718. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  719. package/src/inference/pipelines/vision/index.js +8 -27
  720. package/src/inference/pipelines/vision/ops.js +11 -33
  721. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  722. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  723. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  724. package/src/inference/refactor-receipt.js +2 -2
  725. package/src/inference/rerank.js +182 -0
  726. package/src/inference/token-sampling.js +319 -0
  727. package/src/inference/tokenizer.js +1 -1
  728. package/src/inference/tokenizers/behavior-flags.js +6 -3
  729. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  730. package/src/inference/tokenizers/bundled/registry.js +216 -0
  731. package/src/inference/tokenizers/bundled.js +23 -353
  732. package/src/loader/doppler-loader.d.ts +4 -0
  733. package/src/loader/doppler-loader.js +119 -930
  734. package/src/loader/embedding-limit-preflight.js +11 -4
  735. package/src/loader/embedding-loader.js +1 -3
  736. package/src/loader/gpu-tensor-materialization.js +20 -0
  737. package/src/loader/layer-loader.js +15 -119
  738. package/src/loader/layer-tensor-aliases.js +99 -0
  739. package/src/loader/layer-weight-shape.js +29 -0
  740. package/src/loader/load-timing.js +64 -0
  741. package/src/loader/loader-types.d.ts +3 -0
  742. package/src/loader/model-executor.js +819 -0
  743. package/src/loader/model-load-validation.js +15 -0
  744. package/src/loader/quantization-constants.d.ts +11 -29
  745. package/src/loader/quantization-constants.js +1 -2
  746. package/src/loader/shard-resolver.js +0 -14
  747. package/src/loader/tensor-load-error.js +14 -0
  748. package/src/loader/tensor-shard-indices.js +8 -0
  749. package/src/loader/tensors/bf16-loader.js +125 -0
  750. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  751. package/src/loader/tensors/load-executor.js +577 -0
  752. package/src/loader/tensors/source-transform/executor.js +123 -0
  753. package/src/loader/tensors/source-transform/plan.js +107 -0
  754. package/src/loader/tensors/source-transform.js +15 -264
  755. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  756. package/src/loader/tensors/tensor-loader.js +7 -653
  757. package/src/memory/buffer-pool.js +7 -29
  758. package/src/rules/capability-transform-resolver.js +80 -0
  759. package/src/rules/execution-contract.d.ts +73 -0
  760. package/src/rules/execution-contract.js +429 -0
  761. package/src/rules/execution-rules-contract-check.js +1 -1
  762. package/src/rules/generated/rule-bundle.json +4305 -0
  763. package/src/rules/inference/capability-transforms.rules.json +3 -3
  764. package/src/rules/layer-pattern-contract-check.js +1 -1
  765. package/src/rules/rule-registry.js +70 -57
  766. package/src/storage/artifact-storage-context.js +2 -2
  767. package/src/storage/backend-selection.js +43 -0
  768. package/src/storage/backends/opfs-store.d.ts +41 -0
  769. package/src/storage/distribution-transport.js +5 -0
  770. package/src/storage/download/integrity.d.ts +33 -0
  771. package/src/storage/download/integrity.js +130 -0
  772. package/src/storage/download/resume-inspection.d.ts +22 -0
  773. package/src/storage/download/resume-inspection.js +134 -0
  774. package/src/storage/download/retry.js +57 -0
  775. package/src/storage/download/state.js +131 -0
  776. package/src/storage/download/transport.js +99 -0
  777. package/src/storage/downloader.js +74 -453
  778. package/src/storage/model-cache.d.ts +1 -0
  779. package/src/storage/model-cache.js +51 -11
  780. package/src/storage/shard-manager.d.ts +5 -0
  781. package/src/storage/shard-manager.js +50 -369
  782. package/src/storage/shards/index.js +60 -0
  783. package/src/storage/shards/integrity.js +225 -0
  784. package/src/storage/shards/lifecycle.js +22 -0
  785. package/src/storage/source-artifact-store.js +14 -4
  786. package/src/tooling/boundary-evidence.d.ts +4 -4
  787. package/src/tooling/boundary-evidence.js +20 -20
  788. package/src/tooling/browser-command-runner.js +13 -6
  789. package/src/tooling/capsule-v2.d.ts +11 -0
  790. package/src/tooling/capsule-v2.js +58 -0
  791. package/src/tooling/capsule.js +20 -0
  792. package/src/tooling/command-api-constants.js +1 -1
  793. package/src/tooling/command-api-family-normalizers.js +163 -5
  794. package/src/tooling/command-api.d.ts +42 -3
  795. package/src/tooling/command-api.js +7 -2
  796. package/src/tooling/command-envelope.js +1 -1
  797. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  798. package/src/tooling/command-runner.html +49 -1
  799. package/src/tooling/command-validation.js +1 -24
  800. package/src/tooling/conversion-run-timing.js +24 -0
  801. package/src/tooling/diagnose-runner.js +11 -9
  802. package/src/tooling/distribution-shard-transport.js +11 -0
  803. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  804. package/src/tooling/execution-cost-ledger.js +197 -15
  805. package/src/tooling/hf-registry/catalog-io.js +63 -0
  806. package/src/tooling/litert/package.js +863 -0
  807. package/src/tooling/litert-package-runtime.js +3 -849
  808. package/src/tooling/model-capsule-forge.js +478 -0
  809. package/src/tooling/node-browser/execution.d.ts +40 -0
  810. package/src/tooling/node-browser/execution.js +308 -0
  811. package/src/tooling/node-browser/static-server.js +189 -0
  812. package/src/tooling/node-browser/transport.d.ts +80 -0
  813. package/src/tooling/node-browser/transport.js +622 -0
  814. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  815. package/src/tooling/node-browser-command-runner.js +10 -1112
  816. package/src/tooling/node-capsule-artifact-store.js +44 -0
  817. package/src/tooling/node-command-execution.js +237 -0
  818. package/src/tooling/node-command-runner.js +26 -205
  819. package/src/tooling/node-converter/input.js +521 -0
  820. package/src/tooling/node-converter/output.js +244 -0
  821. package/src/tooling/node-converter-chunk-batches.js +21 -0
  822. package/src/tooling/node-converter.js +7 -720
  823. package/src/tooling/node-source-runtime.js +16 -276
  824. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  825. package/src/tooling/node-webgpu.js +178 -453
  826. package/src/tooling/opfs-cache.d.ts +5 -0
  827. package/src/tooling/opfs-cache.js +4 -0
  828. package/src/tooling/production-release.d.ts +19 -0
  829. package/src/tooling/production-release.js +647 -0
  830. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  831. package/src/tooling/program-bundle/materialize.js +324 -0
  832. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  833. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  834. package/src/tooling/program-bundle/validation.js +31 -0
  835. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  836. package/src/tooling/program-bundle-parity.d.ts +11 -4
  837. package/src/tooling/program-bundle-parity.js +262 -109
  838. package/src/tooling/program-bundle.d.ts +7 -0
  839. package/src/tooling/program-bundle.js +186 -591
  840. package/src/tooling/provider-v1-contract.d.ts +61 -0
  841. package/src/tooling/provider-v1-contract.js +192 -0
  842. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  843. package/src/tooling/registered-variant-calibration-job.js +210 -0
  844. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  845. package/src/tooling/registered-variant-calibration.js +57 -7
  846. package/src/tooling/runtime-optimization/candidates.js +664 -0
  847. package/src/tooling/runtime-optimization-campaign.js +166 -0
  848. package/src/tooling/runtime-optimization-index.js +16 -1
  849. package/src/tooling/runtime-optimization.d.ts +44 -0
  850. package/src/tooling/runtime-optimization.js +13 -656
  851. package/src/tooling/source-intake.d.ts +1 -1
  852. package/src/tooling/source-intake.js +34 -17
  853. package/src/tooling/source-package-profiles.js +1 -1
  854. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  855. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  856. package/src/tooling/source-runtime/node-loader.js +258 -0
  857. package/src/tooling/source-runtime-bundle.js +26 -411
  858. package/src/tooling/source-runtime-converter-config.js +1 -1
  859. package/src/tooling-experimental-exports.js +0 -5
  860. package/src/tooling-exports/evidence.d.ts +14 -3
  861. package/src/tooling-exports/evidence.js +14 -3
  862. package/src/tooling-exports/storage.d.ts +7 -0
  863. package/src/tooling-exports/storage.js +5 -0
  864. package/src/tooling-exports.d.ts +3 -0
  865. package/src/tooling-exports.js +2 -0
  866. package/src/tooling-exports.shared.d.ts +17 -3
  867. package/src/tooling-exports.shared.js +9 -3
  868. package/src/training.d.ts +6 -6
  869. package/src/training.js +4 -4
  870. package/src/version.js +1 -1
  871. package/tests/kernels/browser/kernel-suite.js +512 -0
  872. package/tests/kernels/browser/test-page.js +170 -14
  873. package/tests/kernels/harness/tolerance.js +1 -0
  874. package/tests/kernels/reference/index.js +13 -1
  875. package/tests/kernels/reference/vision.js +160 -0
  876. package/src/converter/quantizer.d.ts +0 -121
  877. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  878. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  879. package/src/gpu/kernels/utils.d.ts +0 -147
  880. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  881. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  882. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  883. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  884. package/src/rules/converter/execution.rules.json +0 -20
  885. package/src/rules/converter/tensor-roles.rules.json +0 -13
  886. package/src/rules/converter/tokenizer.rules.json +0 -12
  887. package/src/rules/inference/attention.rules.json +0 -69
  888. package/src/rules/inference/config.rules.json +0 -82
  889. package/src/rules/inference/dtype.rules.json +0 -113
  890. package/src/rules/inference/execution.rules.json +0 -195
  891. package/src/rules/inference/ffn.rules.json +0 -123
  892. package/src/rules/inference/layer-pattern.rules.json +0 -16
  893. package/src/rules/inference/layer.rules.json +0 -7
  894. package/src/rules/inference/moe.rules.json +0 -145
  895. package/src/rules/kernels/attention.rules.json +0 -68
  896. package/src/rules/kernels/conv2d.rules.json +0 -6
  897. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  898. package/src/rules/kernels/dequant.rules.json +0 -58
  899. package/src/rules/kernels/energy.rules.json +0 -22
  900. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  901. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  902. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  903. package/src/rules/kernels/gather.rules.json +0 -93
  904. package/src/rules/kernels/gelu.rules.json +0 -11
  905. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  906. package/src/rules/kernels/groupnorm.rules.json +0 -10
  907. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  908. package/src/rules/kernels/layernorm.rules.json +0 -6
  909. package/src/rules/kernels/linear-attention.rules.json +0 -6
  910. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  911. package/src/rules/kernels/matmul.rules.json +0 -95
  912. package/src/rules/kernels/modulate.rules.json +0 -6
  913. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  914. package/src/rules/kernels/moe.rules.json +0 -14
  915. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  916. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  917. package/src/rules/kernels/relu.rules.json +0 -6
  918. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  919. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  920. package/src/rules/kernels/residual.rules.json +0 -12
  921. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  922. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  923. package/src/rules/kernels/rope-qk.rules.json +0 -6
  924. package/src/rules/kernels/rope.rules.json +0 -6
  925. package/src/rules/kernels/sample.rules.json +0 -6
  926. package/src/rules/kernels/scale.rules.json +0 -8
  927. package/src/rules/kernels/silu.rules.json +0 -22
  928. package/src/rules/kernels/softmax.rules.json +0 -25
  929. package/src/rules/kernels/split-qg.rules.json +0 -6
  930. package/src/rules/kernels/split-qkv.rules.json +0 -6
  931. package/src/rules/kernels/upsample2d.rules.json +0 -6
  932. package/src/rules/loader/tensor-loader.rules.json +0 -42
  933. package/src/rules/loader/weights.rules.json +0 -42
  934. package/src/rules/tooling/command-runtime.rules.json +0 -65
  935. package/src/tooling/hf-registry-utils.js +0 -395
  936. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  937. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  938. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  939. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  940. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  941. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  942. /package/src/{utils → formats}/array-buffer.js +0 -0
  943. /package/src/{utils → formats}/canonical-hash.js +0 -0
  944. /package/src/{utils → formats}/clone-json.js +0 -0
  945. /package/src/{utils → formats}/encode-utf8.js +0 -0
  946. /package/src/{utils → formats}/load-json.js +0 -0
  947. /package/src/{utils → formats}/plain-object.js +0 -0
  948. /package/src/{utils → formats}/sha256.js +0 -0
  949. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  950. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  951. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  952. /package/src/{utils → storage}/runtime-env.js +0 -0
  953. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -26,63 +26,10 @@ import { selectRuleValue } from '../../../rules/rule-registry.js';
26
26
  import { getDevice } from '../../../gpu/device.js';
27
27
  import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
28
28
  import { f16ToF32 } from '../../../loader/dtype-utils.js';
29
-
30
- const pleRuntimeCache = new WeakMap();
31
- const pleRangeRowCache = new WeakMap();
32
- let f16ToF32Lookup = null;
33
- const PLE_ACTIVATION_STATIC_QMIN = -128;
34
- const PLE_ACTIVATION_STATIC_QMAX = 127;
35
-
36
- function getF16ToF32Lookup() {
37
- if (f16ToF32Lookup) {
38
- return f16ToF32Lookup;
39
- }
40
- const lookup = new Float32Array(1 << 16);
41
- for (let value = 0; value < lookup.length; value += 1) {
42
- lookup[value] = f16ToF32(value);
43
- }
44
- f16ToF32Lookup = lookup;
45
- return lookup;
46
- }
47
-
48
- function getPerLayerInputWeights(context) {
49
- const weights = context.weights.get('per_layer_inputs');
50
- if (!weights || typeof weights !== 'object') {
51
- throw new Error(
52
- 'Gemma 4 per-layer inputs require global per-layer input weights, ' +
53
- 'but state.weights.get("per_layer_inputs") was missing.'
54
- );
55
- }
56
- return weights;
57
- }
58
-
59
- function normalizePleProjectionNormDtype(dtype) {
60
- if (typeof dtype !== 'string') {
61
- return null;
62
- }
63
- const value = dtype.toLowerCase();
64
- if (value === 'f16' || value === 'f32') {
65
- return value;
66
- }
67
- return null;
68
- }
69
-
70
- function getPleProjectionNormDtype(weight) {
71
- return normalizePleProjectionNormDtype(getWeightDtype(weight))
72
- ?? normalizePleProjectionNormDtype(getBufferDtype(weight))
73
- ?? null;
74
- }
75
-
76
- function isRangeBackedPleProjectionSource(value) {
77
- return isRangeBackedCpuEmbeddingSource(value);
78
- }
79
-
80
- function getPleProjectionWeightDtype(weight) {
81
- if (isCpuWeightBuffer(weight)) {
82
- return weight.dtype ?? null;
83
- }
84
- return getWeightDtype(weight);
85
- }
29
+ import { destroyPleHotVocabularyRuntime, destroyPleRuntimeCacheEntry, ensurePleScaledProjectionNormWeight, getPerLayerInputWeights, getPleHotCachePolicy, getPleRangeRowLoadConfig, getPleSplitTablePolicy, pleRuntimeCache } from './per-layer/materialize.js';
30
+ import { applyPleActivationStaticQdq, getPleProjectionWeightDtype, getPreparedTokenEntry, isRangeBackedPleProjectionSource, loadRangeBackedPleProjectionSliceBytes, loadRangeBackedPleRow, pleRangeRowCache, storePreparedTokenEntry } from './per-layer/plan.js';
31
+ export { loadRangeBackedPleProjectionSliceBytes } from './per-layer/plan.js';
32
+ export { ensurePleGpuHotVocabularyRuntime, ensurePleGpuSplitTablesRuntime, ensurePleScaledProjectionNormWeight, inferPleProjectionNormDtype, resolvePleHotVocabularyCapacity, scalePerLayerProjectionNormWeights } from './per-layer/materialize.js';
86
33
 
87
34
  function isDensePleProjectionDtype(dtype) {
88
35
  return dtype === 'f16' || dtype === 'f32';
@@ -125,69 +72,6 @@ function resolvePleActivationStaticQdqConfig(perLayerInputWeights) {
125
72
  };
126
73
  }
127
74
 
128
- function getTensorElementCount(tensor, label) {
129
- if (!Array.isArray(tensor?.shape) || tensor.shape.length === 0) {
130
- throw new Error(`${label} requires a tensor with an explicit shape.`);
131
- }
132
- const count = tensor.shape.reduce((total, dimension) => total * Number(dimension), 1);
133
- if (!Number.isFinite(count) || count <= 0) {
134
- throw new Error(`${label} resolved an invalid tensor element count (${String(count)}).`);
135
- }
136
- return count;
137
- }
138
-
139
- async function applyPleActivationStaticQdq(tensor, scale, recorder, decodeBuffers, label) {
140
- const count = getTensorElementCount(tensor, label);
141
- if (tensor.dtype === 'f32') {
142
- return recorder
143
- ? await recordActivationStaticQdq(recorder, tensor, scale, {
144
- count,
145
- qmin: PLE_ACTIVATION_STATIC_QMIN,
146
- qmax: PLE_ACTIVATION_STATIC_QMAX,
147
- })
148
- : await runActivationStaticQdq(tensor, scale, {
149
- count,
150
- qmin: PLE_ACTIVATION_STATIC_QMIN,
151
- qmax: PLE_ACTIVATION_STATIC_QMAX,
152
- });
153
- }
154
- if (tensor.dtype !== 'f16') {
155
- throw new Error(
156
- `${label} requires f16/f32 activations for static activation quantize/dequantize. ` +
157
- `Got "${String(tensor.dtype)}".`
158
- );
159
- }
160
-
161
- let widenedTensor = null;
162
- let qdqTensor = null;
163
- try {
164
- widenedTensor = recorder
165
- ? await recordCastF16ToF32(recorder, tensor)
166
- : await castF16ToF32(tensor);
167
- qdqTensor = recorder
168
- ? await recordActivationStaticQdq(recorder, widenedTensor, scale, {
169
- count,
170
- qmin: PLE_ACTIVATION_STATIC_QMIN,
171
- qmax: PLE_ACTIVATION_STATIC_QMAX,
172
- })
173
- : await runActivationStaticQdq(widenedTensor, scale, {
174
- count,
175
- qmin: PLE_ACTIVATION_STATIC_QMIN,
176
- qmax: PLE_ACTIVATION_STATIC_QMAX,
177
- });
178
- return recorder
179
- ? await recordCastF32ToF16(recorder, qdqTensor)
180
- : await castF32ToF16(qdqTensor);
181
- } finally {
182
- if (qdqTensor) {
183
- releaseOrTrack(recorder, qdqTensor.buffer, decodeBuffers);
184
- }
185
- if (widenedTensor) {
186
- releaseOrTrack(recorder, widenedTensor.buffer, decodeBuffers);
187
- }
188
- }
189
- }
190
-
191
75
  export function resolveDensePleProjectionWeight(
192
76
  weight,
193
77
  label = 'per_layer_model_projection'
@@ -225,81 +109,6 @@ export function resolveDensePleProjectionWeight(
225
109
  return weight;
226
110
  }
227
111
 
228
- export async function loadRangeBackedPleProjectionSliceBytes(
229
- weight,
230
- layerIdx,
231
- hiddenSizePerLayerInput,
232
- hiddenSize,
233
- label = 'Range-backed PLE projection slice'
234
- ) {
235
- if (!isCpuWeightBuffer(weight)) {
236
- return null;
237
- }
238
- const cpuData = weight.data;
239
- if (!isRangeBackedPleProjectionSource(cpuData)) {
240
- return null;
241
- }
242
-
243
- const dtype = getPleProjectionWeightDtype(weight);
244
- const layout = getLayout(weight) ?? weight.layout ?? 'row';
245
- const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
246
- dtype,
247
- });
248
- const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
249
- const byteOffset = hiddenOffset * hiddenSize * bytesPerElement;
250
- const byteLength = hiddenSizePerLayerInput * hiddenSize * bytesPerElement;
251
- const bytes = normalizeRangeBytes(
252
- await cpuData.loadRange(byteOffset, byteLength),
253
- label
254
- );
255
- if (bytes.byteLength !== byteLength) {
256
- throw new Error(
257
- `${label} short read for layer ${layerIdx}: expected ${byteLength} bytes, got ${bytes.byteLength}.`
258
- );
259
- }
260
- return {
261
- bytes,
262
- dtype,
263
- layout,
264
- shape: [hiddenSizePerLayerInput, hiddenSize],
265
- };
266
- }
267
-
268
- export function inferPleProjectionNormDtype(weight, hiddenSizePerLayerInput) {
269
- const explicitDtype = getPleProjectionNormDtype(weight);
270
- if (explicitDtype) {
271
- return explicitDtype;
272
- }
273
-
274
- if (!isGpuBufferInstance(weight)) {
275
- return 'f32';
276
- }
277
-
278
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
279
- throw new Error('Gemma 4 per-layer projection norm dtype inference requires hiddenSizePerLayerInput > 0.');
280
- }
281
-
282
- const bytesPerElement = weight.size / hiddenSizePerLayerInput;
283
- if (bytesPerElement !== 2 && bytesPerElement !== 4) {
284
- throw new Error(
285
- 'Gemma 4 per-layer projection norm buffer has unsupported byte size: ' +
286
- `bufferSize=${weight.size}, hiddenSizePerLayerInput=${hiddenSizePerLayerInput}.`
287
- );
288
- }
289
- return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
290
- }
291
-
292
- function createPleProjectionNormTensor(buffer, dtype, hiddenSizePerLayerInput, label) {
293
- return createTensor(buffer, dtype, [hiddenSizePerLayerInput], label);
294
- }
295
-
296
- function destroyPleRuntimeCacheEntry(entry) {
297
- const cachedBuffer = entry?.scaledProjectionNormWeight?.buffer ?? null;
298
- if (cachedBuffer) {
299
- releaseBuffer(cachedBuffer);
300
- }
301
- }
302
-
303
112
  function destroyPleSplitTables(perLayerInputWeights) {
304
113
  const splitTables = Array.isArray(perLayerInputWeights?.embedTokensPerLayerSplit)
305
114
  ? perLayerInputWeights.embedTokensPerLayerSplit
@@ -316,25 +125,6 @@ function destroyPleSplitTables(perLayerInputWeights) {
316
125
  delete perLayerInputWeights.embedTokensPerLayerSplit;
317
126
  }
318
127
 
319
- function destroyPleHotVocabularyRuntime(perLayerInputWeights) {
320
- const runtime = perLayerInputWeights?.embedTokensPerLayerHotRuntime ?? null;
321
- if (!runtime || typeof runtime !== 'object') {
322
- return;
323
- }
324
- const splitTables = Array.isArray(runtime.splitTables) ? runtime.splitTables : [];
325
- for (const table of splitTables) {
326
- const buffer = table?.buffer ?? table ?? null;
327
- if (buffer) {
328
- releaseBuffer(buffer);
329
- }
330
- }
331
- const mapBuffer = runtime.hotTokenIndexMapBuffer ?? null;
332
- if (mapBuffer) {
333
- releaseBuffer(mapBuffer);
334
- }
335
- delete perLayerInputWeights.embedTokensPerLayerHotRuntime;
336
- }
337
-
338
128
  export function destroyPleRuntimeCache(perLayerInputWeights) {
339
129
  if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
340
130
  return;
@@ -351,126 +141,6 @@ export function destroyPleRuntimeCache(perLayerInputWeights) {
351
141
  }
352
142
  }
353
143
 
354
- export function scalePerLayerProjectionNormWeights(weight, combineScale, rmsNormWeightOffset = false) {
355
- if (rmsNormWeightOffset) {
356
- return null;
357
- }
358
- const source = isCpuWeightBuffer(weight) ? weight.data : weight;
359
- const isArrayLikeView = ArrayBuffer.isView(source) && typeof source.length === 'number';
360
- if (!(source instanceof Float32Array) && !isArrayLikeView && !Array.isArray(source)) {
361
- return null;
362
- }
363
- const scaled = Float32Array.from(source);
364
- for (let i = 0; i < scaled.length; i++) {
365
- scaled[i] *= combineScale;
366
- }
367
- return scaled;
368
- }
369
-
370
- export async function ensurePleScaledProjectionNormWeight(context, combineScale = 2 ** -0.5) {
371
- const hiddenSizePerLayerInput = Number(context?.config?.hiddenSizePerLayerInput ?? 0);
372
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
373
- return null;
374
- }
375
- if (context?.weightConfig?.rmsNormWeightOffset) {
376
- return null;
377
- }
378
-
379
- const perLayerInputWeights = getPerLayerInputWeights(context);
380
- const cachedEntry = pleRuntimeCache.get(perLayerInputWeights) ?? null;
381
- if (cachedEntry?.combineScale === combineScale && cachedEntry.scaledProjectionNormWeight) {
382
- return cachedEntry.scaledProjectionNormWeight;
383
- }
384
- if (cachedEntry) {
385
- destroyPleRuntimeCacheEntry(cachedEntry);
386
- pleRuntimeCache.delete(perLayerInputWeights);
387
- }
388
-
389
- const projectionNormWeight = perLayerInputWeights.perLayerProjectionNorm;
390
- if (!projectionNormWeight) {
391
- return null;
392
- }
393
-
394
- let scaledProjectionNormWeight = null;
395
- if (isGpuBufferInstance(projectionNormWeight)) {
396
- const projectionNormDtype = inferPleProjectionNormDtype(
397
- projectionNormWeight,
398
- hiddenSizePerLayerInput
399
- );
400
- const scaledTensor = await runScale(
401
- createPleProjectionNormTensor(
402
- projectionNormWeight,
403
- projectionNormDtype,
404
- hiddenSizePerLayerInput,
405
- 'per_layer_projection_norm'
406
- ),
407
- combineScale,
408
- { count: hiddenSizePerLayerInput }
409
- );
410
- scaledProjectionNormWeight = createPleProjectionNormTensor(
411
- scaledTensor.buffer,
412
- projectionNormDtype,
413
- hiddenSizePerLayerInput,
414
- 'per_layer_projection_norm_scaled'
415
- );
416
- } else {
417
- const scaledValues = scalePerLayerProjectionNormWeights(
418
- projectionNormWeight,
419
- combineScale,
420
- false
421
- );
422
- if (!scaledValues) {
423
- return null;
424
- }
425
- if (scaledValues.length !== hiddenSizePerLayerInput) {
426
- throw new Error(
427
- 'Gemma 4 per-layer projection norm cache shape mismatch: ' +
428
- `expected ${hiddenSizePerLayerInput} values, got ${scaledValues.length}.`
429
- );
430
- }
431
- const device = getDevice();
432
- if (!device) {
433
- throw new Error('No GPU device available for Gemma 4 per-layer projection norm cache.');
434
- }
435
- const scaledBuffer = acquireBuffer(
436
- scaledValues.byteLength,
437
- undefined,
438
- 'per_layer_projection_norm_scaled'
439
- );
440
- try {
441
- device.queue.writeBuffer(scaledBuffer, 0, scaledValues);
442
- } catch (error) {
443
- releaseBuffer(scaledBuffer);
444
- throw error;
445
- }
446
- scaledProjectionNormWeight = createPleProjectionNormTensor(
447
- scaledBuffer,
448
- inferPleProjectionNormDtype(projectionNormWeight, hiddenSizePerLayerInput),
449
- hiddenSizePerLayerInput,
450
- 'per_layer_projection_norm_scaled'
451
- );
452
- }
453
-
454
- if (scaledProjectionNormWeight?.shape?.[0] !== hiddenSizePerLayerInput) {
455
- throw new Error(
456
- 'Gemma 4 per-layer projection norm cache shape mismatch after tensor creation: ' +
457
- `expected ${hiddenSizePerLayerInput}, got ${scaledProjectionNormWeight?.shape?.[0] ?? 'unknown'}.`
458
- );
459
- }
460
- if (!normalizePleProjectionNormDtype(scaledProjectionNormWeight?.dtype)) {
461
- throw new Error(
462
- 'Gemma 4 per-layer projection norm cache produced an invalid dtype: ' +
463
- `"${String(scaledProjectionNormWeight?.dtype ?? 'undefined')}".`
464
- );
465
- }
466
-
467
- pleRuntimeCache.set(perLayerInputWeights, {
468
- combineScale,
469
- scaledProjectionNormWeight,
470
- });
471
- return scaledProjectionNormWeight;
472
- }
473
-
474
144
  function getEmbeddingSource(weight, label) {
475
145
  if (isWeightBuffer(weight)) {
476
146
  return weight.buffer;
@@ -488,17 +158,6 @@ function getEmbeddingDtype(weight) {
488
158
  return getWeightDtype(weight);
489
159
  }
490
160
 
491
- function requirePleSourceDtype(rawDtype, label, allowed = ['f16', 'f32']) {
492
- if (rawDtype == null) {
493
- throw new Error(`${label} requires source dtype metadata.`);
494
- }
495
- const sourceDtype = String(rawDtype).toLowerCase();
496
- if (!allowed.includes(sourceDtype)) {
497
- throw new Error(`${label} requires ${allowed.join('/')} source rows; got "${sourceDtype}".`);
498
- }
499
- return sourceDtype;
500
- }
501
-
502
161
  function getEmbeddingTranspose(weight) {
503
162
  if (isWeightBuffer(weight) || isCpuWeightBuffer(weight)) {
504
163
  return weight.layout === 'column';
@@ -560,107 +219,6 @@ function releasePleSliceBuffer(recorder, buffer, decodeBuffers, cache) {
560
219
  releaseOrTrack(recorder, buffer, decodeBuffers);
561
220
  }
562
221
 
563
- function getPleRowCachePolicy(sessionConfig) {
564
- const rowCache = sessionConfig?.rowCache ?? null;
565
- if (!rowCache || rowCache.mode === 'off') {
566
- return null;
567
- }
568
- if (rowCache.mode !== 'lru') {
569
- throw new Error(
570
- `Gemma 4 per-layer input row cache mode "${String(rowCache.mode)}" is not implemented.`
571
- );
572
- }
573
- const decodedDtype = String(rowCache.decodedDtype ?? '').toLowerCase();
574
- if (decodedDtype !== 'f32') {
575
- throw new Error(
576
- `Gemma 4 range-backed per-layer input row cache requires rowCache.decodedDtype="f32"; ` +
577
- `got "${String(rowCache.decodedDtype)}".`
578
- );
579
- }
580
- const maxRows = Math.trunc(Number(rowCache.maxRows));
581
- const maxBytes = Math.trunc(Number(rowCache.maxBytes));
582
- if (!Number.isFinite(maxRows) || maxRows <= 0) {
583
- throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxRows > 0.');
584
- }
585
- if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
586
- throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxBytes > 0.');
587
- }
588
- return { maxRows, maxBytes };
589
- }
590
-
591
- function getPleHotCachePolicy(sessionConfig) {
592
- const hotCache = sessionConfig?.hotCache ?? null;
593
- if (!hotCache || hotCache.mode === 'off') {
594
- return null;
595
- }
596
- if (hotCache.mode === 'tokenizer_scores') {
597
- const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
598
- if (outputDtype !== 'f16' && outputDtype !== 'f32') {
599
- throw new Error(
600
- `Gemma 4 per-layer input hot vocabulary cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
601
- `got "${String(hotCache.outputDtype)}".`
602
- );
603
- }
604
- const maxTokens = Math.trunc(Number(hotCache.maxTokens));
605
- const maxBytes = Math.trunc(Number(hotCache.maxBytes));
606
- if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
607
- throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxTokens > 0.');
608
- }
609
- if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
610
- throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxBytes > 0.');
611
- }
612
- return {
613
- mode: 'tokenizer_scores',
614
- maxTokens,
615
- maxBytes,
616
- outputDtype,
617
- };
618
- }
619
- if (hotCache.mode !== 'prepared_tokens') {
620
- throw new Error(
621
- `Gemma 4 per-layer input hot cache mode "${String(hotCache.mode)}" is not implemented.`
622
- );
623
- }
624
- const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
625
- if (outputDtype !== 'f16' && outputDtype !== 'f32') {
626
- throw new Error(
627
- `Gemma 4 per-layer input hot cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
628
- `got "${String(hotCache.outputDtype)}".`
629
- );
630
- }
631
- const maxTokens = Math.trunc(Number(hotCache.maxTokens));
632
- const maxBytes = Math.trunc(Number(hotCache.maxBytes));
633
- if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
634
- throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxTokens > 0.');
635
- }
636
- if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
637
- throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxBytes > 0.');
638
- }
639
- return { mode: 'prepared_tokens', maxTokens, maxBytes, outputDtype };
640
- }
641
-
642
- export function resolvePleHotVocabularyCapacity({
643
- maxTokens,
644
- maxBytes,
645
- numLayers,
646
- hiddenSize,
647
- bytesPerElement,
648
- vocabSize,
649
- }) {
650
- const bytesPerHotRow = numLayers * hiddenSize * bytesPerElement;
651
- const tokenIndexMapBytes = vocabSize * Uint32Array.BYTES_PER_ELEMENT;
652
- const tableBudgetBytes = Math.max(0, maxBytes - tokenIndexMapBytes);
653
- const maxTableRows = bytesPerHotRow > 0
654
- ? Math.floor(tableBudgetBytes / bytesPerHotRow)
655
- : 0;
656
- const maxHotTokens = Math.max(0, Math.min(maxTokens, maxTableRows - 1));
657
- return {
658
- maxHotTokens,
659
- bytesPerHotRow,
660
- tokenIndexMapBytes,
661
- };
662
- }
663
-
664
222
  export function getPleHotVocabularyRuntime(context) {
665
223
  const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
666
224
  if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
@@ -670,212 +228,6 @@ export function getPleHotVocabularyRuntime(context) {
670
228
  return runtime && typeof runtime === 'object' ? runtime : null;
671
229
  }
672
230
 
673
- function getPleSplitTablePolicy(sessionConfig) {
674
- if (sessionConfig?.materialization !== 'gpu_split_tables') {
675
- return null;
676
- }
677
- return { mode: 'gpu_split_tables' };
678
- }
679
-
680
- function releasePreparedTokenEntry(cache, tokenId, entry) {
681
- if (!cache || !entry) {
682
- return;
683
- }
684
- cache.preparedTokenEntries?.delete(tokenId);
685
- cache.preparedTokenBytes -= entry?.bytes ?? 0;
686
- for (const buffer of entry.buffers ?? []) {
687
- if (!buffer) continue;
688
- cache.ownedBuffers?.delete(buffer);
689
- releaseBuffer(buffer);
690
- }
691
- }
692
-
693
- function prunePreparedTokenCache(cache, policy) {
694
- while (
695
- cache.preparedTokenEntries.size > policy.maxTokens
696
- || cache.preparedTokenBytes > policy.maxBytes
697
- ) {
698
- const oldest = cache.preparedTokenEntries.keys().next();
699
- if (oldest.done) {
700
- break;
701
- }
702
- const tokenId = oldest.value;
703
- const entry = cache.preparedTokenEntries.get(tokenId);
704
- releasePreparedTokenEntry(cache, tokenId, entry);
705
- }
706
- }
707
-
708
- function getPreparedTokenEntry(cache, tokenId, sessionConfig, activationDtype, stats = null) {
709
- const policy = getPleHotCachePolicy(sessionConfig);
710
- if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
711
- return null;
712
- }
713
- if (policy.outputDtype !== activationDtype) {
714
- throw new Error(
715
- `Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
716
- `got "${String(activationDtype)}".`
717
- );
718
- }
719
- const entry = cache.preparedTokenEntries.get(tokenId) ?? null;
720
- if (!entry) {
721
- if (stats) {
722
- stats.plePreparedTokenCacheMisses = (stats.plePreparedTokenCacheMisses ?? 0) + 1;
723
- stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
724
- stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
725
- }
726
- return null;
727
- }
728
- cache.preparedTokenEntries.delete(tokenId);
729
- cache.preparedTokenEntries.set(tokenId, entry);
730
- if (stats) {
731
- stats.plePreparedTokenCacheHits = (stats.plePreparedTokenCacheHits ?? 0) + 1;
732
- stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
733
- stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
734
- }
735
- return entry.buffers.slice();
736
- }
737
-
738
- function storePreparedTokenEntry(cache, tokenId, buffers, sessionConfig, activationDtype, stats = null) {
739
- const policy = getPleHotCachePolicy(sessionConfig);
740
- if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
741
- return buffers;
742
- }
743
- if (policy.outputDtype !== activationDtype) {
744
- throw new Error(
745
- `Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
746
- `got "${String(activationDtype)}".`
747
- );
748
- }
749
- const existing = cache.preparedTokenEntries.get(tokenId) ?? null;
750
- if (existing) {
751
- releasePreparedTokenEntry(cache, tokenId, existing);
752
- }
753
- const cachedBuffers = buffers.slice();
754
- const bytes = cachedBuffers.reduce((total, buffer) => total + (buffer?.size ?? 0), 0);
755
- for (const buffer of cachedBuffers) {
756
- if (buffer) {
757
- cache.ownedBuffers?.add(buffer);
758
- }
759
- }
760
- cache.preparedTokenEntries.set(tokenId, { buffers: cachedBuffers, bytes });
761
- cache.preparedTokenBytes += bytes;
762
- prunePreparedTokenCache(cache, policy);
763
- if (stats) {
764
- stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
765
- stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
766
- }
767
- return cachedBuffers.slice();
768
- }
769
-
770
- function getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize) {
771
- const sourceDtype = requirePleSourceDtype(
772
- (isCpuWeightBuffer(embedTokensPerLayer) ? embedTokensPerLayer.data?.sourceDtype : null)
773
- ?? embedTokensPerLayer?.dtype,
774
- 'Gemma 4 range-backed per-layer input rows',
775
- ['f16', 'bf16', 'f32']
776
- );
777
- const bytesPerElement = (sourceDtype === 'f16' || sourceDtype === 'bf16') ? 2 : 4;
778
- return {
779
- sourceDtype,
780
- sourceRowBytes: totalPerLayerHiddenSize * bytesPerElement,
781
- };
782
- }
783
-
784
- function getPleRangeRowCache(embedTokensPerLayer, sessionConfig) {
785
- const policy = getPleRowCachePolicy(sessionConfig);
786
- if (!policy) {
787
- return null;
788
- }
789
- const cached = pleRangeRowCache.get(embedTokensPerLayer);
790
- if (cached && cached.maxRows === policy.maxRows && cached.maxBytes === policy.maxBytes) {
791
- return cached;
792
- }
793
- const next = {
794
- maxRows: policy.maxRows,
795
- maxBytes: policy.maxBytes,
796
- totalBytes: 0,
797
- rows: new Map(),
798
- };
799
- pleRangeRowCache.set(embedTokensPerLayer, next);
800
- return next;
801
- }
802
-
803
- function touchPleCachedRow(cache, tokenId) {
804
- const hit = cache?.rows?.get(tokenId) ?? null;
805
- if (!hit) {
806
- return null;
807
- }
808
- cache.rows.delete(tokenId);
809
- cache.rows.set(tokenId, hit);
810
- return hit.row;
811
- }
812
-
813
- function prunePleRangeRowCache(cache) {
814
- while (cache.rows.size > cache.maxRows || cache.totalBytes > cache.maxBytes) {
815
- const oldest = cache.rows.keys().next();
816
- if (oldest.done) {
817
- break;
818
- }
819
- const entry = cache.rows.get(oldest.value);
820
- cache.rows.delete(oldest.value);
821
- cache.totalBytes -= entry?.bytes ?? 0;
822
- }
823
- }
824
-
825
- function cachePleRangeRow(cache, tokenId, row) {
826
- if (!cache) {
827
- return row;
828
- }
829
- const existing = cache.rows.get(tokenId);
830
- if (existing) {
831
- cache.totalBytes -= existing.bytes;
832
- cache.rows.delete(tokenId);
833
- }
834
- cache.rows.set(tokenId, { row, bytes: row.byteLength });
835
- cache.totalBytes += row.byteLength;
836
- prunePleRangeRowCache(cache);
837
- return row;
838
- }
839
-
840
- async function loadRangeBackedPleRow(
841
- tokenId,
842
- embedTokensPerLayer,
843
- totalPerLayerHiddenSize,
844
- sessionConfig,
845
- label,
846
- prefetchedRow = null
847
- ) {
848
- if (!isCpuWeightBuffer(embedTokensPerLayer)) {
849
- return null;
850
- }
851
- const cpuData = embedTokensPerLayer.data;
852
- if (!isRangeBackedCpuEmbeddingSource(cpuData)) {
853
- return null;
854
- }
855
-
856
- const cache = getPleRangeRowCache(embedTokensPerLayer, sessionConfig);
857
- if (prefetchedRow && prefetchedRow.tokenId === tokenId) {
858
- return cachePleRangeRow(cache, tokenId, prefetchedRow.row);
859
- }
860
-
861
- const cached = touchPleCachedRow(cache, tokenId);
862
- if (cached) {
863
- return cached;
864
- }
865
-
866
- const { sourceDtype, sourceRowBytes } = getPleRangeRowLoadConfig(
867
- embedTokensPerLayer,
868
- totalPerLayerHiddenSize
869
- );
870
- const chunk = normalizeRangeBytes(
871
- await cpuData.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
872
- label
873
- );
874
- const row = new Float32Array(totalPerLayerHiddenSize);
875
- decodeRangeChunkIntoOutput(chunk, sourceDtype, row, 0, totalPerLayerHiddenSize);
876
- return cachePleRangeRow(cache, tokenId, row);
877
- }
878
-
879
231
  // Step 5: Prefetch next token's PLE row during current decode step.
880
232
  // Returns a promise resolving to { tokenId, row: Float32Array } or null.
881
233
  // Call after sampling produces the next token; pass result as options.prefetchedRow
@@ -934,391 +286,6 @@ export function hasGpuSplitPerLayerInputEmbeddings(context) {
934
286
  && perLayerInputWeights.embedTokensPerLayerSplit.length > 0;
935
287
  }
936
288
 
937
- export async function ensurePleGpuSplitTablesRuntime(context) {
938
- const policy = getPleSplitTablePolicy(context?.perLayerInputsSession ?? null);
939
- if (!policy) {
940
- return null;
941
- }
942
-
943
- const config = context?.config ?? null;
944
- const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
945
- const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
946
- const numLayers = Number(config?.numLayers ?? 0);
947
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
948
- return null;
949
- }
950
- if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
951
- return null;
952
- }
953
- if (!Number.isFinite(numLayers) || numLayers <= 0) {
954
- return null;
955
- }
956
-
957
- const perLayerInputWeights = getPerLayerInputWeights(context);
958
- if (Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit) && perLayerInputWeights.embedTokensPerLayerSplit.length === numLayers) {
959
- return perLayerInputWeights.embedTokensPerLayerSplit;
960
- }
961
-
962
- const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
963
- if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
964
- throw new Error('Gemma 4 gpu_split_tables materialization requires a range-backed CPU embedTokensPerLayer source.');
965
- }
966
-
967
- const sourceDtype = requirePleSourceDtype(
968
- embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
969
- 'Gemma 4 gpu_split_tables materialization'
970
- );
971
-
972
- const device = getDevice();
973
- if (!device) {
974
- throw new Error('No GPU device available for Gemma 4 gpu_split_tables materialization.');
975
- }
976
-
977
- const bytesPerElement = sourceDtype === 'f16' ? 2 : 4;
978
- const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
979
- const tableBytes = vocabSizePerLayerInput * hiddenSizePerLayerInput * bytesPerElement;
980
- const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
981
- acquireBuffer(tableBytes, undefined, `L${layerIdx}.ple_table_split`),
982
- sourceDtype,
983
- 'row',
984
- [vocabSizePerLayerInput, hiddenSizePerLayerInput],
985
- `L${layerIdx}.embed_tokens_per_layer_split`
986
- ));
987
-
988
- try {
989
- const rowsPerChunk = 128;
990
- for (let rowStart = 0; rowStart < vocabSizePerLayerInput; rowStart += rowsPerChunk) {
991
- const rowCount = Math.min(rowsPerChunk, vocabSizePerLayerInput - rowStart);
992
- const chunkByteOffset = rowStart * totalPerLayerHiddenSize * bytesPerElement;
993
- const chunkByteLength = rowCount * totalPerLayerHiddenSize * bytesPerElement;
994
- const chunk = normalizeRangeBytes(
995
- await embedTokensPerLayer.data.loadRange(chunkByteOffset, chunkByteLength),
996
- 'Gemma 4 split GPU PLE chunk'
997
- );
998
-
999
- if (sourceDtype === 'f16') {
1000
- const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
1001
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1002
- const layerWords = new Uint16Array(rowCount * hiddenSizePerLayerInput);
1003
- for (let row = 0; row < rowCount; row += 1) {
1004
- const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
1005
- layerWords.set(
1006
- sourceWords.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
1007
- row * hiddenSizePerLayerInput
1008
- );
1009
- }
1010
- device.queue.writeBuffer(
1011
- splitTables[layerIdx].buffer,
1012
- rowStart * hiddenSizePerLayerInput * bytesPerElement,
1013
- layerWords.buffer,
1014
- layerWords.byteOffset,
1015
- layerWords.byteLength
1016
- );
1017
- }
1018
- } else {
1019
- const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
1020
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1021
- const layerValues = new Float32Array(rowCount * hiddenSizePerLayerInput);
1022
- for (let row = 0; row < rowCount; row += 1) {
1023
- const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
1024
- layerValues.set(
1025
- sourceValues.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
1026
- row * hiddenSizePerLayerInput
1027
- );
1028
- }
1029
- device.queue.writeBuffer(
1030
- splitTables[layerIdx].buffer,
1031
- rowStart * hiddenSizePerLayerInput * bytesPerElement,
1032
- layerValues.buffer,
1033
- layerValues.byteOffset,
1034
- layerValues.byteLength
1035
- );
1036
- }
1037
- }
1038
- }
1039
- } catch (error) {
1040
- for (const table of splitTables) {
1041
- releaseBuffer(table.buffer);
1042
- }
1043
- throw error;
1044
- }
1045
-
1046
- perLayerInputWeights.embedTokensPerLayerSplit = splitTables;
1047
- return splitTables;
1048
- }
1049
-
1050
- function resolvePleHotVocabularySeedTokenIds(context, maxTokens, vocabSizePerLayerInput) {
1051
- const rawSeedTokenIds = Array.isArray(context?.seedTokenIds) ? context.seedTokenIds : null;
1052
- if (!rawSeedTokenIds || rawSeedTokenIds.length === 0) {
1053
- return [];
1054
- }
1055
-
1056
- const specialTokenIds = new Set();
1057
- const tokenizerSpecialTokens = context?.tokenizer?.getSpecialTokens?.() ?? null;
1058
- if (tokenizerSpecialTokens && typeof tokenizerSpecialTokens === 'object') {
1059
- for (const value of Object.values(tokenizerSpecialTokens)) {
1060
- if (Number.isInteger(value)) {
1061
- specialTokenIds.add(value);
1062
- }
1063
- }
1064
- }
1065
-
1066
- const seeded = [];
1067
- const seen = new Set();
1068
- for (let i = rawSeedTokenIds.length - 1; i >= 0; i -= 1) {
1069
- const tokenId = rawSeedTokenIds[i];
1070
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput) {
1071
- continue;
1072
- }
1073
- if (specialTokenIds.has(tokenId) || seen.has(tokenId)) {
1074
- continue;
1075
- }
1076
- seeded.push(tokenId);
1077
- seen.add(tokenId);
1078
- if (seeded.length >= maxTokens) {
1079
- break;
1080
- }
1081
- }
1082
- return seeded;
1083
- }
1084
-
1085
- function mergePleHotVocabularyTokenIds(seedTokenIds, tokenizerHotTokenIds, maxTokens, vocabSizePerLayerInput) {
1086
- const merged = [];
1087
- const seen = new Set();
1088
-
1089
- for (const tokenId of seedTokenIds ?? []) {
1090
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
1091
- continue;
1092
- }
1093
- merged.push(tokenId);
1094
- seen.add(tokenId);
1095
- if (merged.length >= maxTokens) {
1096
- return merged;
1097
- }
1098
- }
1099
-
1100
- for (const tokenId of tokenizerHotTokenIds ?? []) {
1101
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
1102
- continue;
1103
- }
1104
- merged.push(tokenId);
1105
- seen.add(tokenId);
1106
- if (merged.length >= maxTokens) {
1107
- break;
1108
- }
1109
- }
1110
-
1111
- return merged;
1112
- }
1113
-
1114
- export async function ensurePleGpuHotVocabularyRuntime(context) {
1115
- const policy = getPleHotCachePolicy(context?.perLayerInputsSession ?? null);
1116
- if (!policy || policy.mode !== 'tokenizer_scores') {
1117
- return null;
1118
- }
1119
-
1120
- const config = context?.config ?? null;
1121
- const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
1122
- const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
1123
- const numLayers = Number(config?.numLayers ?? 0);
1124
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
1125
- return null;
1126
- }
1127
- if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
1128
- return null;
1129
- }
1130
- if (!Number.isFinite(numLayers) || numLayers <= 0) {
1131
- return null;
1132
- }
1133
-
1134
- const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
1135
- const capacity = resolvePleHotVocabularyCapacity({
1136
- maxTokens: policy.maxTokens,
1137
- maxBytes: policy.maxBytes,
1138
- numLayers,
1139
- hiddenSize: hiddenSizePerLayerInput,
1140
- bytesPerElement,
1141
- vocabSize: vocabSizePerLayerInput,
1142
- });
1143
- if (capacity.maxHotTokens <= 0) {
1144
- return null;
1145
- }
1146
-
1147
- const tokenizer = context?.tokenizer ?? null;
1148
- const tokenizerHotTokenIds = typeof tokenizer?.getHotTokenIds === 'function'
1149
- ? tokenizer.getHotTokenIds(capacity.maxHotTokens)
1150
- : null;
1151
- const seedTokenIds = resolvePleHotVocabularySeedTokenIds(
1152
- context,
1153
- capacity.maxHotTokens,
1154
- vocabSizePerLayerInput
1155
- );
1156
- const hotTokenIds = mergePleHotVocabularyTokenIds(
1157
- seedTokenIds,
1158
- tokenizerHotTokenIds,
1159
- capacity.maxHotTokens,
1160
- vocabSizePerLayerInput
1161
- );
1162
- if (hotTokenIds.length === 0) {
1163
- return null;
1164
- }
1165
- const hotTokenIdsSignature = hotTokenIds.join(',');
1166
-
1167
- const perLayerInputWeights = getPerLayerInputWeights(context);
1168
- const cached = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
1169
- if (
1170
- cached
1171
- && cached.maxTokens === capacity.maxHotTokens
1172
- && cached.maxBytes === policy.maxBytes
1173
- && cached.outputDtype === policy.outputDtype
1174
- && cached.vocabSize === vocabSizePerLayerInput
1175
- && cached.numLayers === numLayers
1176
- && cached.hotTokenIdsSignature === hotTokenIdsSignature
1177
- ) {
1178
- return cached;
1179
- }
1180
- destroyPleHotVocabularyRuntime(perLayerInputWeights);
1181
-
1182
- const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
1183
- if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
1184
- return null;
1185
- }
1186
-
1187
- const sourceDtype = requirePleSourceDtype(
1188
- embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
1189
- 'Gemma 4 hot vocabulary cache'
1190
- );
1191
- const expandsF16ToF32 = sourceDtype === 'f16' && policy.outputDtype === 'f32';
1192
- if (sourceDtype !== policy.outputDtype && !expandsF16ToF32) {
1193
- throw new Error(
1194
- `Gemma 4 hot vocabulary cache cannot convert source dtype "${sourceDtype}" ` +
1195
- `to output dtype "${policy.outputDtype}".`
1196
- );
1197
- }
1198
-
1199
- const device = getDevice();
1200
- if (!device) {
1201
- throw new Error('No GPU device available for Gemma 4 hot vocabulary cache.');
1202
- }
1203
-
1204
- const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
1205
- const sentinelIndex = hotTokenIds.length;
1206
- const hotRowCount = sentinelIndex + 1;
1207
- const splitTableBytes = hotRowCount * hiddenSizePerLayerInput * bytesPerElement;
1208
- const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
1209
- acquireBuffer(splitTableBytes, undefined, `L${layerIdx}.ple_hot_vocab_table`),
1210
- policy.outputDtype,
1211
- 'row',
1212
- [hotRowCount, hiddenSizePerLayerInput],
1213
- `L${layerIdx}.embed_tokens_per_layer_hot_vocab`
1214
- ));
1215
- const hotTokenIndexMap = new Uint32Array(vocabSizePerLayerInput);
1216
- hotTokenIndexMap.fill(sentinelIndex);
1217
- for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
1218
- const tokenId = hotTokenIds[hotIndex];
1219
- if (Number.isInteger(tokenId) && tokenId >= 0 && tokenId < vocabSizePerLayerInput) {
1220
- hotTokenIndexMap[tokenId] = hotIndex;
1221
- }
1222
- }
1223
- const hotTokenIndexMapBuffer = acquireBuffer(
1224
- hotTokenIndexMap.byteLength,
1225
- undefined,
1226
- 'ple_hot_token_index_map'
1227
- );
1228
-
1229
- try {
1230
- device.queue.writeBuffer(hotTokenIndexMapBuffer, 0, hotTokenIndexMap);
1231
- const zeroRow = policy.outputDtype === 'f16'
1232
- ? new Uint16Array(hiddenSizePerLayerInput)
1233
- : new Float32Array(hiddenSizePerLayerInput);
1234
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1235
- device.queue.writeBuffer(
1236
- splitTables[layerIdx].buffer,
1237
- sentinelIndex * hiddenSizePerLayerInput * bytesPerElement,
1238
- zeroRow.buffer,
1239
- zeroRow.byteOffset,
1240
- zeroRow.byteLength
1241
- );
1242
- }
1243
-
1244
- const { sourceRowBytes } = getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize);
1245
- const expandedF32Row = expandsF16ToF32
1246
- ? new Float32Array(totalPerLayerHiddenSize)
1247
- : null;
1248
- const f16Lookup = expandsF16ToF32 ? getF16ToF32Lookup() : null;
1249
- for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
1250
- const tokenId = hotTokenIds[hotIndex];
1251
- const chunk = normalizeRangeBytes(
1252
- await embedTokensPerLayer.data.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
1253
- 'Gemma 4 hot vocabulary PLE row'
1254
- );
1255
- if (policy.outputDtype === 'f16') {
1256
- const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1257
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1258
- const sourceStart = layerIdx * hiddenSizePerLayerInput;
1259
- device.queue.writeBuffer(
1260
- splitTables[layerIdx].buffer,
1261
- hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1262
- sourceWords.buffer,
1263
- sourceWords.byteOffset + sourceStart * bytesPerElement,
1264
- hiddenSizePerLayerInput * bytesPerElement
1265
- );
1266
- }
1267
- } else if (sourceDtype === 'f16') {
1268
- const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1269
- for (let valueIdx = 0; valueIdx < totalPerLayerHiddenSize; valueIdx += 1) {
1270
- expandedF32Row[valueIdx] = f16Lookup[sourceWords[valueIdx]];
1271
- }
1272
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1273
- const sourceStart = layerIdx * hiddenSizePerLayerInput;
1274
- device.queue.writeBuffer(
1275
- splitTables[layerIdx].buffer,
1276
- hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1277
- expandedF32Row.buffer,
1278
- sourceStart * bytesPerElement,
1279
- hiddenSizePerLayerInput * bytesPerElement
1280
- );
1281
- }
1282
- } else {
1283
- const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1284
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1285
- const sourceStart = layerIdx * hiddenSizePerLayerInput;
1286
- device.queue.writeBuffer(
1287
- splitTables[layerIdx].buffer,
1288
- hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1289
- sourceValues.buffer,
1290
- sourceValues.byteOffset + sourceStart * bytesPerElement,
1291
- hiddenSizePerLayerInput * bytesPerElement
1292
- );
1293
- }
1294
- }
1295
- }
1296
- } catch (error) {
1297
- for (const table of splitTables) {
1298
- releaseBuffer(table.buffer);
1299
- }
1300
- releaseBuffer(hotTokenIndexMapBuffer);
1301
- throw error;
1302
- }
1303
-
1304
- const runtime = {
1305
- mode: 'tokenizer_scores',
1306
- maxTokens: capacity.maxHotTokens,
1307
- maxBytes: policy.maxBytes,
1308
- outputDtype: policy.outputDtype,
1309
- vocabSize: vocabSizePerLayerInput,
1310
- numLayers,
1311
- hotTokenIdsSignature,
1312
- hotTokenIds: Uint32Array.from(hotTokenIds),
1313
- hotTokenIndexMap,
1314
- hotTokenIndexMapBuffer,
1315
- sentinelIndex,
1316
- splitTables,
1317
- };
1318
- perLayerInputWeights.embedTokensPerLayerHotRuntime = runtime;
1319
- return runtime;
1320
- }
1321
-
1322
289
  export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context, options = {}) {
1323
290
  const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
1324
291
  const hiddenSizePerLayerInput = Number(config.hiddenSizePerLayerInput ?? 0);