doppler-gpu 0.4.15 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1063) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -213
  4. package/models/catalog.json +241 -33
  5. package/package.json +122 -86
  6. package/src/capsule-runtime.d.ts +23 -0
  7. package/src/capsule-runtime.js +23 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +101 -1349
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +4 -8
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +1 -0
  24. package/src/client/capsule-host.d.ts +3 -0
  25. package/src/client/capsule-host.js +1 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +64 -1
  28. package/src/client/doppler-api.js +42 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.d.ts +122 -0
  41. package/src/client/inspection.js +343 -0
  42. package/src/client/model-host/index.d.ts +140 -0
  43. package/src/client/model-host/index.js +548 -0
  44. package/src/client/model-host/model-evidence.js +255 -0
  45. package/src/client/model-host/model-session.d.ts +322 -0
  46. package/src/client/model-host/model-session.js +593 -0
  47. package/src/client/provider.d.ts +18 -2
  48. package/src/client/provider.js +26 -3
  49. package/src/client/receipt.js +30 -72
  50. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  51. package/src/client/runtime/capsule-acquisition.js +103 -0
  52. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  53. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  54. package/src/client/runtime/capsule-artifact-source.js +62 -0
  55. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  56. package/src/client/runtime/capsule-embedding.js +76 -0
  57. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  58. package/src/client/runtime/capsule-forecast-program.js +85 -0
  59. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  60. package/src/client/runtime/capsule-forecast.js +31 -0
  61. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  62. package/src/client/runtime/capsule-operation-adapters.js +68 -0
  63. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  64. package/src/client/runtime/capsule-operation-executor.js +100 -0
  65. package/src/client/runtime/capsule-program-adapter.js +151 -0
  66. package/src/client/runtime/capsule-release-authorization.js +25 -0
  67. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  68. package/src/client/runtime/capsule-rerank.js +105 -0
  69. package/src/client/runtime/capsule-session-execution.js +85 -0
  70. package/src/client/runtime/command-executor.d.ts +15 -0
  71. package/src/client/runtime/command-executor.js +128 -0
  72. package/src/client/runtime/composition-root.d.ts +74 -0
  73. package/src/client/runtime/composition-root.js +344 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  75. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  76. package/src/client/runtime/index.d.ts +1 -63
  77. package/src/client/runtime/index.js +1 -261
  78. package/src/client/runtime/lora.d.ts +14 -0
  79. package/src/client/runtime/lora.js +19 -1
  80. package/src/client/runtime/model-manager.d.ts +0 -4
  81. package/src/client/runtime/model-manager.js +12 -28
  82. package/src/client/runtime/model-rerank.js +30 -0
  83. package/src/client/runtime/model-session.d.ts +1 -174
  84. package/src/client/runtime/model-source.d.ts +19 -1
  85. package/src/client/runtime/model-source.js +46 -17
  86. package/src/client/runtime/node-quickstart-cache.js +1 -1
  87. package/src/client/runtime/resolution-policy.d.ts +20 -0
  88. package/src/client/runtime/resolution-policy.js +84 -0
  89. package/src/client/runtime/resource-binder.d.ts +24 -0
  90. package/src/client/runtime/resource-binder.js +135 -0
  91. package/src/client/runtime/scoped-session.d.ts +126 -0
  92. package/src/client/runtime/scoped-session.js +276 -0
  93. package/src/client/runtime/session-controller.d.ts +30 -0
  94. package/src/client/runtime/session-controller.js +170 -0
  95. package/src/client/runtime/source-runtime.js +1 -1
  96. package/src/client/runtime/target-selector.d.ts +9 -0
  97. package/src/client/runtime/target-selector.js +2 -0
  98. package/src/client/runtime/types.d.ts +1 -1
  99. package/src/client/runtime/types.js +0 -1
  100. package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
  101. package/src/client/wrap-pipeline-handle.js +3 -9
  102. package/src/config/backward-registry-loader.js +1 -1
  103. package/src/config/capsule-adapter-policy.js +30 -0
  104. package/src/config/capsule-adapters.d.ts +21 -0
  105. package/src/config/capsule-adapters.js +40 -0
  106. package/src/config/capsule-adapters.json +9 -0
  107. package/src/config/capsule-loading.d.ts +2 -0
  108. package/src/config/capsule-loading.js +14 -0
  109. package/src/config/capsule-loading.schema.json +11 -0
  110. package/src/config/capsule-operation.d.ts +19 -0
  111. package/src/config/capsule-operation.js +58 -0
  112. package/src/config/capsule-operations.json +18 -0
  113. package/src/config/capsule-release-contract.d.ts +66 -0
  114. package/src/config/capsule-release-contract.js +197 -0
  115. package/src/config/capsule-release-events.d.ts +53 -0
  116. package/src/config/capsule-release-events.js +172 -0
  117. package/src/config/capsule-sequence-receipt.js +19 -0
  118. package/src/config/capsule-serve.d.ts +9 -0
  119. package/src/config/capsule-serve.js +28 -0
  120. package/src/config/capsule-serve.schema.json +14 -0
  121. package/src/config/capsule-signature.d.ts +6 -0
  122. package/src/config/capsule-signature.js +50 -0
  123. package/src/config/capsule-v2.d.ts +70 -0
  124. package/src/config/capsule-v2.js +431 -0
  125. package/src/config/capsule-v3.d.ts +19 -0
  126. package/src/config/capsule-v3.js +72 -0
  127. package/src/config/capsule.d.ts +22 -0
  128. package/src/config/capsule.js +52 -0
  129. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  130. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  131. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
  132. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  133. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
  134. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
  135. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
  136. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
  137. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
  138. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
  140. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
  141. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
  142. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
  143. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
  144. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
  145. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  146. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
  147. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  148. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  149. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
  150. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  151. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  152. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  153. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
  154. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
  155. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
  157. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
  158. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  160. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
  161. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
  162. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  163. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
  164. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  165. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  166. package/src/config/dtype-transition-contract.js +40 -0
  167. package/src/config/embedding-contract.js +26 -0
  168. package/src/config/embedding-reference.js +85 -0
  169. package/src/config/evidence/boundary-evidence-policy.json +54 -0
  170. package/src/config/evidence/source-intake-policy.json +122 -0
  171. package/src/config/execution-contract-check.d.ts +16 -73
  172. package/src/config/execution-contract-check.js +7 -429
  173. package/src/config/forecast-manifest.js +56 -0
  174. package/src/config/heterogeneous-attention-contract.js +46 -0
  175. package/src/config/initial-execution-identity.d.ts +58 -0
  176. package/src/config/initial-execution-identity.js +331 -0
  177. package/src/config/inspection/observation-policies.json +80 -0
  178. package/src/config/kernel-path-loader.d.ts +10 -0
  179. package/src/config/kernel-path-loader.js +23 -8
  180. package/src/config/kernel-registry-contract.d.ts +25 -0
  181. package/src/config/kernel-registry-contract.js +46 -0
  182. package/src/config/kernels/kernel-ref-digests.js +50 -2
  183. package/src/config/kernels/registry.js +1 -1
  184. package/src/config/kernels/registry.json +13592 -11377
  185. package/src/config/lora-layouts.d.ts +10 -0
  186. package/src/config/lora-layouts.js +17 -0
  187. package/src/config/lora-layouts.json +9 -0
  188. package/src/config/manifest-inference-validation.js +575 -0
  189. package/src/config/merge/execution-patches.js +12 -0
  190. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  191. package/src/config/merge/value-selection.js +38 -0
  192. package/src/config/merge.js +43 -11
  193. package/src/config/model-ir-v2.d.ts +57 -0
  194. package/src/config/model-ir-v2.js +258 -0
  195. package/src/config/model-ir.d.ts +49 -0
  196. package/src/config/model-ir.js +180 -0
  197. package/src/config/param-validator.js +22 -37
  198. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  199. package/src/config/platforms/loader.js +2 -1
  200. package/src/config/production-release-evidence.d.ts +23 -0
  201. package/src/config/production-release-evidence.js +423 -0
  202. package/src/config/production-release.js +321 -0
  203. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  204. package/src/config/required-inference-fields-contract-check.js +2 -2
  205. package/src/config/rerank-reference.d.ts +28 -0
  206. package/src/config/rerank-reference.js +89 -0
  207. package/src/config/revocation-policy.d.ts +65 -0
  208. package/src/config/revocation-policy.js +290 -0
  209. package/src/config/revocation-registry.json +11 -0
  210. package/src/config/revocation-updates.d.ts +68 -0
  211. package/src/config/revocation-updates.js +440 -0
  212. package/src/config/runtime/compute/f16-activations.json +1 -0
  213. package/src/config/runtime/compute/f16-batched.json +1 -0
  214. package/src/config/runtime/default.json +1 -0
  215. package/src/config/runtime/diagnostics/debug-logits.json +1 -0
  216. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
  217. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
  218. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
  219. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
  220. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
  221. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
  222. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
  223. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
  224. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
  225. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
  226. package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
  227. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
  228. package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
  229. package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
  230. package/src/config/runtime/model/gemma2-debug.json +1 -0
  231. package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
  232. package/src/config/runtime/model/gemma2-pipeline.json +1 -0
  233. package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
  234. package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
  235. package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
  236. package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
  237. package/src/config/runtime/optimization-candidates/registry.json +5 -0
  238. package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
  239. package/src/config/runtime/profiles/default.json +1 -0
  240. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
  241. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
  242. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
  243. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
  244. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
  245. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +2 -6
  246. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
  247. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
  248. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
  249. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
  250. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
  251. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
  252. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
  253. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
  254. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
  255. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
  256. package/src/config/runtime/profiles/low-memory.json +1 -0
  257. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  258. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  259. package/src/config/runtime/profiles/production.json +1 -3
  260. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
  261. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
  262. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
  263. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
  264. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
  265. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
  266. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
  267. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
  268. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
  269. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
  270. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
  271. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
  272. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
  273. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
  274. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
  275. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
  276. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
  277. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +2 -6
  278. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  279. package/src/config/runtime/profiles/rerank-stability.json +1 -3
  280. package/src/config/runtime/profiles/rerank-throughput.json +1 -3
  281. package/src/config/runtime/profiles/simulation.json +1 -3
  282. package/src/config/runtime/profiles/throughput.json +1 -3
  283. package/src/config/runtime/profiles/trace-layers.json +1 -3
  284. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
  285. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
  286. package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
  287. package/src/config/runtime/profiles/turboquant.json +1 -0
  288. package/src/config/runtime/profiles/vector-stability.json +1 -3
  289. package/src/config/runtime/profiles/vector-throughput.json +1 -0
  290. package/src/config/runtime/profiles/verbose-trace.json +1 -3
  291. package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
  292. package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
  293. package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
  294. package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
  295. package/src/config/runtime-merge.js +1 -1
  296. package/src/config/runtime.js +1 -1
  297. package/src/config/schema/benchmark.schema.d.ts +7 -0
  298. package/src/config/schema/benchmark.schema.js +5 -0
  299. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  300. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  301. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  302. package/src/config/schema/conversion-report.schema.js +29 -0
  303. package/src/config/schema/converter.schema.d.ts +1 -0
  304. package/src/config/schema/debug.schema.d.ts +1 -0
  305. package/src/config/schema/doppler.schema.js +5 -5
  306. package/src/config/schema/emulation.schema.js +1 -1
  307. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  308. package/src/config/schema/execution-v1.schema.js +19 -0
  309. package/src/config/schema/harness.schema.d.ts +13 -0
  310. package/src/config/schema/harness.schema.js +6 -0
  311. package/src/config/schema/index.d.ts +8 -16
  312. package/src/config/schema/index.js +5 -8
  313. package/src/config/schema/inference.schema.d.ts +21 -2
  314. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
  315. package/src/config/schema/kernel-thresholds.schema.js +1 -13
  316. package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
  317. package/src/config/schema/kernel-warmup.schema.js +0 -1
  318. package/src/config/schema/manifest.schema.d.ts +26 -7
  319. package/src/config/schema/manifest.schema.js +10 -1
  320. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  321. package/src/config/schema/program-bundle.schema.js +145 -10
  322. package/src/config/schema/runtime-profile.schema.d.ts +24 -0
  323. package/src/config/schema/runtime-profile.schema.js +92 -0
  324. package/src/config/schema/shared-runtime.schema.d.ts +1 -4
  325. package/src/config/schema/shared-runtime.schema.js +0 -2
  326. package/src/config/schema/tooling.schema.d.ts +8 -5
  327. package/src/config/schema/tooling.schema.js +3 -2
  328. package/src/config/sequence-reference.d.ts +26 -0
  329. package/src/config/sequence-reference.js +100 -0
  330. package/src/config/target-plan.d.ts +108 -0
  331. package/src/config/target-plan.js +411 -0
  332. package/src/config/training-defaults.js +1 -1
  333. package/src/config/transforms/capability-transform-resolver.js +4 -105
  334. package/src/config/transforms/execution-graph/dtype.js +591 -0
  335. package/src/config/transforms/execution-graph/fusion.js +436 -0
  336. package/src/config/transforms/execution-graph/validation.js +414 -0
  337. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  338. package/src/converter/artifact-identity.js +475 -0
  339. package/src/converter/conversion-plan.js +1 -1
  340. package/src/converter/core.js +94 -2096
  341. package/src/converter/forge-candidate-evaluation.js +271 -0
  342. package/src/converter/forge-qualification.js +162 -0
  343. package/src/converter/forge-stages.js +798 -0
  344. package/src/converter/manifest-builder.js +683 -0
  345. package/src/converter/quantization-info.js +4 -1
  346. package/src/converter/quantizer.js +58 -9
  347. package/src/converter/shard-packer.d.ts +1 -1
  348. package/src/converter/shard-packer.js +2 -2
  349. package/src/converter/tensor-transform.js +640 -0
  350. package/src/converter/tokenizer-contract.js +243 -0
  351. package/src/converter/tokenizer-manifest.js +243 -0
  352. package/src/debug/history.js +0 -8
  353. package/src/debug/index.js +0 -10
  354. package/src/debug/tensor.d.ts +2 -0
  355. package/src/debug/tensor.js +21 -2
  356. package/src/debug/trace.js +0 -7
  357. package/src/experimental/adapters/adapter-manager.js +2 -0
  358. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  359. package/src/experimental/adapters/adapter-manifest.js +11 -0
  360. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  361. package/src/experimental/adapters/artifact-contract.js +5 -2
  362. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  363. package/src/experimental/adapters/lora-loader.js +110 -48
  364. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  365. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  366. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  367. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  368. package/src/experimental/browser/browser-converter.js +11 -11
  369. package/src/experimental/diffusion/index.d.ts +1 -1
  370. package/src/experimental/diffusion/index.js +1 -1
  371. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  372. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  373. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  374. package/src/experimental/diffusion/vae.js +596 -0
  375. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  376. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  377. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  378. package/src/experimental/distribution/shard-delivery.js +4 -1709
  379. package/src/experimental/energy/index.d.ts +2 -2
  380. package/src/experimental/energy/index.js +2 -2
  381. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  382. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  383. package/src/experimental/logits/cpu-output.d.ts +27 -0
  384. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  385. package/src/experimental/logits/cpu.d.ts +87 -0
  386. package/src/experimental/logits/cpu.js +95 -0
  387. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  388. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  389. package/src/experimental/orchestration/index.d.ts +1 -1
  390. package/src/experimental/orchestration/index.js +1 -1
  391. package/src/experimental/training/artifacts.js +2 -2
  392. package/src/experimental/training/checkpoint.js +2 -2
  393. package/src/experimental/training/distillation/artifacts.js +1 -1
  394. package/src/experimental/training/distillation/dataset.js +2 -2
  395. package/src/experimental/training/distillation/eval.js +1 -1
  396. package/src/experimental/training/distillation/runtime.js +2 -2
  397. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  398. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  399. package/src/experimental/training/distillation/student-fixture.js +2 -469
  400. package/src/experimental/training/lora/checkpoint.js +746 -0
  401. package/src/experimental/training/lora/execution.js +550 -0
  402. package/src/experimental/training/lora/plan.js +387 -0
  403. package/src/experimental/training/lora/recovery.js +418 -0
  404. package/src/experimental/training/lora-pipeline.js +7 -1949
  405. package/src/experimental/training/operator-artifacts.js +2 -2
  406. package/src/experimental/training/operator-command.js +4 -4
  407. package/src/experimental/training/runner/evaluation.js +62 -0
  408. package/src/experimental/training/runner/lifecycle.js +257 -0
  409. package/src/experimental/training/runner.js +5 -268
  410. package/src/experimental/training/suite/execution.js +879 -0
  411. package/src/experimental/training/suite/plan.js +855 -0
  412. package/src/experimental/training/suite.js +5 -1659
  413. package/src/experimental/training/workloads.d.ts +5 -5
  414. package/src/experimental/training/workloads.js +6 -6
  415. package/src/experimental/turboquant/codebook.js +470 -0
  416. package/src/formats/quantization-constants.d.ts +11 -0
  417. package/src/formats/quantization-constants.js +11 -0
  418. package/src/formats/rdrr/integrity.js +1 -1
  419. package/src/formats/rdrr/lowerings.js +0 -29
  420. package/src/formats/rdrr/merkle.js +1 -1
  421. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  422. package/src/formats/rdrr/validation.js +3 -3
  423. package/src/gpu/command-recorder.d.ts +15 -1
  424. package/src/gpu/command-recorder.js +63 -2
  425. package/src/gpu/constant-buffer.js +29 -0
  426. package/src/gpu/device-state.js +36 -0
  427. package/src/gpu/device.js +6 -41
  428. package/src/gpu/kernel-runtime.js +3 -10
  429. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  430. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  431. package/src/gpu/kernels/attention/executors.js +960 -0
  432. package/src/gpu/kernels/attention/plan.js +472 -0
  433. package/src/gpu/kernels/attention.d.ts +4 -0
  434. package/src/gpu/kernels/attention.js +145 -1545
  435. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  436. package/src/gpu/kernels/backward/adam.js +2 -1
  437. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  438. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  439. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  440. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  441. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  442. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  443. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  444. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  445. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  446. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  447. package/src/gpu/kernels/backward/executor.js +158 -0
  448. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  449. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  450. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  451. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  452. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  453. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  454. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  455. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  456. package/src/gpu/kernels/backward/matmul.js +133 -0
  457. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  458. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  459. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  460. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  461. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  462. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  463. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  464. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  465. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  466. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  467. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  468. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  469. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  470. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  471. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  472. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  473. package/src/gpu/kernels/backward/utils.js +2 -392
  474. package/src/gpu/kernels/bind-group-validation.js +16 -0
  475. package/src/gpu/kernels/cast.js +2 -1
  476. package/src/gpu/kernels/check-finiteness.js +13 -49
  477. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  478. package/src/gpu/kernels/check-stop.js +14 -33
  479. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  480. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  481. package/src/gpu/kernels/check_stop.wgsl +20 -0
  482. package/src/gpu/kernels/clamp.js +1 -1
  483. package/src/gpu/kernels/conv2d.js +1 -1
  484. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  485. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  486. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  487. package/src/gpu/kernels/dequant.js +7 -2
  488. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  489. package/src/gpu/kernels/dispatch.js +12 -1
  490. package/src/gpu/kernels/embedding-pool.js +72 -0
  491. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  492. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  493. package/src/gpu/kernels/energy.js +2 -1
  494. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  495. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  496. package/src/gpu/kernels/fused_ffn.js +10 -220
  497. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  498. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  499. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  500. package/src/gpu/kernels/gated-short-conv.js +11 -69
  501. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  502. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  503. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  504. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  505. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  506. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  507. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  508. package/src/gpu/kernels/gather.d.ts +1 -1
  509. package/src/gpu/kernels/gather.js +30 -6
  510. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  511. package/src/gpu/kernels/gelu.js +1 -1
  512. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  513. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  514. package/src/gpu/kernels/groupnorm.js +1 -1
  515. package/src/gpu/kernels/index.d.ts +38 -7
  516. package/src/gpu/kernels/index.js +18 -6
  517. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  518. package/src/gpu/kernels/kernel-base.js +1 -1
  519. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  520. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  521. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  522. package/src/gpu/kernels/kernel-configs.js +26 -54
  523. package/src/gpu/kernels/kernel-execution.js +141 -0
  524. package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
  525. package/src/gpu/kernels/kernel-prewarm.js +60 -0
  526. package/src/gpu/kernels/kv-cache-write.js +4 -4
  527. package/src/gpu/kernels/kv-quantize.js +4 -1
  528. package/src/gpu/kernels/l2-normalize.js +36 -0
  529. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  530. package/src/gpu/kernels/layernorm.d.ts +4 -4
  531. package/src/gpu/kernels/layernorm.js +44 -7
  532. package/src/gpu/kernels/linear-attention-core.js +29 -449
  533. package/src/gpu/kernels/linear_attention.js +1 -1
  534. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  535. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  536. package/src/gpu/kernels/logit-finalize.js +103 -0
  537. package/src/gpu/kernels/logit-merge.js +13 -125
  538. package/src/gpu/kernels/logit-scatter.js +40 -0
  539. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  540. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  541. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  542. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  543. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  544. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  545. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  546. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  547. package/src/gpu/kernels/matmul-selection.js +3 -2
  548. package/src/gpu/kernels/matmul.js +4 -15
  549. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  550. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  551. package/src/gpu/kernels/modulate.js +1 -1
  552. package/src/gpu/kernels/moe.js +3 -1
  553. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  554. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  555. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  556. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  557. package/src/gpu/kernels/patch_context.wgsl +29 -0
  558. package/src/gpu/kernels/pipeline-cache.js +3 -1
  559. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  560. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  561. package/src/gpu/kernels/relu.js +1 -1
  562. package/src/gpu/kernels/repeat_channels.js +1 -1
  563. package/src/gpu/kernels/residual.js +2 -2
  564. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  565. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  566. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  567. package/src/gpu/kernels/rmsnorm.js +1 -1
  568. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  569. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  570. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  571. package/src/gpu/kernels/rope-precompute.js +160 -0
  572. package/src/gpu/kernels/rope-qk.js +1 -1
  573. package/src/gpu/kernels/rope.js +2 -2
  574. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  575. package/src/gpu/kernels/sample.js +5 -1
  576. package/src/gpu/kernels/scale.js +1 -1
  577. package/src/gpu/kernels/shader-cache.js +8 -4
  578. package/src/gpu/kernels/shader-source-scope.js +68 -0
  579. package/src/gpu/kernels/silu.js +2 -1
  580. package/src/gpu/kernels/soft-embedding.js +3 -1
  581. package/src/gpu/kernels/softmax.js +4 -2
  582. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  583. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  584. package/src/gpu/kernels/split_qg.js +1 -1
  585. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  586. package/src/gpu/kernels/split_qkv.js +18 -3
  587. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  588. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  589. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  590. package/src/gpu/kernels/transpose.js +1 -1
  591. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  592. package/src/gpu/kernels/uniform-utils.js +7 -2
  593. package/src/gpu/kernels/upsample2d.js +1 -1
  594. package/src/gpu/kernels/utils.js +9 -332
  595. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  596. package/src/gpu/kernels/vision-average-pool.js +68 -0
  597. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  598. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  599. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  600. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  601. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  602. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  603. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  604. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  605. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  606. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  607. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  608. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  609. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  610. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  611. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  612. package/src/gpu/readback-buffer.js +20 -0
  613. package/src/gpu/submit-probe.js +2 -26
  614. package/src/gpu/weight-buffer.d.ts +1 -0
  615. package/src/gpu/weight-buffer.js +7 -0
  616. package/src/index.d.ts +23 -1
  617. package/src/index.js +8 -1
  618. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  619. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  620. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  621. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  622. package/src/inference/browser-harness/inference-execution.js +543 -0
  623. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +18 -145
  624. package/src/inference/browser-harness/model-resolution.js +137 -0
  625. package/src/inference/browser-harness/report.js +338 -0
  626. package/src/inference/browser-harness/request.js +263 -0
  627. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  628. package/src/inference/browser-harness/runtime-config.js +255 -0
  629. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  630. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  631. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  632. package/src/inference/browser-harness/runtime-report.js +117 -0
  633. package/src/inference/browser-harness/suite-summary.js +39 -0
  634. package/src/inference/browser-harness/text-evidence.js +694 -0
  635. package/src/inference/browser-harness/text-execution.js +757 -0
  636. package/src/inference/browser-harness/text-input.js +725 -0
  637. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  638. package/src/inference/browser-harness-contract-helpers.js +1 -1
  639. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  640. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  641. package/src/inference/browser-harness-runtime-helpers.js +3 -408
  642. package/src/inference/browser-harness-text-helpers.js +31 -2196
  643. package/src/inference/browser-harness.d.ts +19 -2
  644. package/src/inference/browser-harness.js +46 -1873
  645. package/src/inference/kv-cache/base.js +67 -381
  646. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  647. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  648. package/src/inference/kv-cache/quantized.d.ts +2 -2
  649. package/src/inference/kv-cache/quantized.js +4 -8
  650. package/src/inference/kv-cache/update.js +309 -0
  651. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  652. package/src/inference/model-checkpoint-evidence.js +264 -0
  653. package/src/inference/moe-router.js +36 -216
  654. package/src/inference/multi-model-network.js +5 -341
  655. package/src/inference/observation-context.d.ts +22 -0
  656. package/src/inference/observation-context.js +41 -0
  657. package/src/inference/pipelines/audio/index.js +1 -20
  658. package/src/inference/pipelines/audio/mel.js +0 -38
  659. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  660. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  661. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  662. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  663. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  664. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  665. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  666. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  667. package/src/inference/pipelines/factory.js +14 -3
  668. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  669. package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
  670. package/src/inference/pipelines/text/abort-contract.js +22 -0
  671. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  672. package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
  673. package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
  674. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  675. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  676. package/src/inference/pipelines/text/attention/index.js +7 -0
  677. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  678. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  679. package/src/inference/pipelines/text/attention/interpreter.js +984 -0
  680. package/src/inference/pipelines/text/attention/plan.d.ts +37 -0
  681. package/src/inference/pipelines/text/attention/plan.js +345 -0
  682. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  683. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  684. package/src/inference/pipelines/text/attention/projections.js +20 -537
  685. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  686. package/src/inference/pipelines/text/attention/receipt.js +60 -0
  687. package/src/inference/pipelines/text/attention/record.js +1 -1277
  688. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  689. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  690. package/src/inference/pipelines/text/attention/run.js +1 -1385
  691. package/src/inference/pipelines/text/attention/types.js +3 -31
  692. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  693. package/src/inference/pipelines/text/chat-format.js +42 -0
  694. package/src/inference/pipelines/text/config/normalization.js +729 -0
  695. package/src/inference/pipelines/text/config/validation.js +9 -0
  696. package/src/inference/pipelines/text/config.d.ts +12 -1
  697. package/src/inference/pipelines/text/config.js +14 -1228
  698. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  699. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  700. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  701. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  702. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  703. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  704. package/src/inference/pipelines/text/embed.js +178 -241
  705. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  706. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  707. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  708. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  709. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  710. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  711. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  712. package/src/inference/pipelines/text/execution-v1.js +7 -283
  713. package/src/inference/pipelines/text/execution.js +517 -0
  714. package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
  715. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  716. package/src/inference/pipelines/text/ffn/dense.js +4 -968
  717. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  718. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  719. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  720. package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
  721. package/src/inference/pipelines/text/generator/decode.js +952 -0
  722. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  723. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  724. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  725. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  726. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  727. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  728. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  729. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  730. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +44 -110
  731. package/src/inference/pipelines/text/generator/text.js +448 -0
  732. package/src/inference/pipelines/text/generator/training.js +752 -0
  733. package/src/inference/pipelines/text/generator-decode-policy.js +5 -2
  734. package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
  735. package/src/inference/pipelines/text/generator-runtime.js +5 -135
  736. package/src/inference/pipelines/text/generator-steps.js +37 -962
  737. package/src/inference/pipelines/text/generator.js +165 -3539
  738. package/src/inference/pipelines/text/image-transcription.js +304 -0
  739. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  740. package/src/inference/pipelines/text/init-rope.js +37 -189
  741. package/src/inference/pipelines/text/init.d.ts +11 -3
  742. package/src/inference/pipelines/text/init.js +6 -216
  743. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  744. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  745. package/src/inference/pipelines/text/layer-plan-gpu.js +15 -6
  746. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  747. package/src/inference/pipelines/text/layer-plan.js +45 -3
  748. package/src/inference/pipelines/text/layer.js +54 -522
  749. package/src/inference/pipelines/text/lifecycle.js +839 -0
  750. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  751. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  752. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  753. package/src/inference/pipelines/text/linear-attention.js +7 -537
  754. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  755. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  756. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  757. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  758. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  759. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  760. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  761. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  762. package/src/inference/pipelines/text/logits/index.js +62 -102
  763. package/src/inference/pipelines/text/logits/output-transform.js +47 -0
  764. package/src/inference/pipelines/text/logits/plan.js +184 -0
  765. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  766. package/src/inference/pipelines/text/logits/readback.js +9 -0
  767. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  768. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  769. package/src/inference/pipelines/text/lora-apply.js +12 -4
  770. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  771. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  772. package/src/inference/pipelines/text/model-load.d.ts +0 -1
  773. package/src/inference/pipelines/text/model-load.js +2 -31
  774. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  775. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  776. package/src/inference/pipelines/text/moe/plan.js +47 -0
  777. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  778. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  779. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  780. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  781. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  782. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  783. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  784. package/src/inference/pipelines/text/probes.js +30 -0
  785. package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
  786. package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
  787. package/src/inference/pipelines/text/state.d.ts +14 -0
  788. package/src/inference/pipelines/text/state.js +4 -0
  789. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  790. package/src/inference/pipelines/text/types.d.ts +10 -0
  791. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  792. package/src/inference/pipelines/text.d.ts +2 -2
  793. package/src/inference/pipelines/text.js +48 -1620
  794. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  795. package/src/inference/pipelines/video/index.js +0 -21
  796. package/src/inference/pipelines/vision/attention.js +59 -0
  797. package/src/inference/pipelines/vision/encoder.js +254 -369
  798. package/src/inference/pipelines/vision/gemma4.js +204 -318
  799. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  800. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  801. package/src/inference/pipelines/vision/glmocr.js +557 -0
  802. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  803. package/src/inference/pipelines/vision/index.js +8 -27
  804. package/src/inference/pipelines/vision/ops.js +11 -33
  805. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  806. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  807. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  808. package/src/inference/refactor-receipt.d.ts +49 -0
  809. package/src/inference/refactor-receipt.js +188 -0
  810. package/src/inference/rerank.js +182 -0
  811. package/src/inference/resource-scope.d.ts +33 -0
  812. package/src/inference/resource-scope.js +168 -0
  813. package/src/inference/test-harness.d.ts +3 -0
  814. package/src/inference/test-harness.js +1 -0
  815. package/src/inference/tokenizer.js +1 -1
  816. package/src/inference/tokenizers/behavior-flags.js +6 -3
  817. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  818. package/src/inference/tokenizers/bundled/registry.js +216 -0
  819. package/src/inference/tokenizers/bundled.js +23 -353
  820. package/src/loader/doppler-loader.d.ts +4 -0
  821. package/src/loader/doppler-loader.js +119 -930
  822. package/src/loader/embedding-limit-preflight.js +11 -4
  823. package/src/loader/embedding-loader.js +1 -3
  824. package/src/loader/gpu-tensor-materialization.js +20 -0
  825. package/src/loader/layer-loader.js +15 -119
  826. package/src/loader/layer-tensor-aliases.js +99 -0
  827. package/src/loader/layer-weight-shape.js +29 -0
  828. package/src/loader/load-timing.js +64 -0
  829. package/src/loader/loader-types.d.ts +3 -0
  830. package/src/loader/model-executor.js +819 -0
  831. package/src/loader/model-load-validation.js +15 -0
  832. package/src/loader/quantization-constants.d.ts +11 -29
  833. package/src/loader/quantization-constants.js +1 -2
  834. package/src/loader/shard-resolver.js +0 -14
  835. package/src/loader/tensor-load-error.js +14 -0
  836. package/src/loader/tensor-shard-indices.js +8 -0
  837. package/src/loader/tensors/bf16-loader.js +125 -0
  838. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  839. package/src/loader/tensors/load-executor.js +577 -0
  840. package/src/loader/tensors/source-transform/executor.js +123 -0
  841. package/src/loader/tensors/source-transform/plan.js +107 -0
  842. package/src/loader/tensors/source-transform.js +15 -264
  843. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  844. package/src/loader/tensors/tensor-loader.js +7 -653
  845. package/src/memory/buffer-pool.js +7 -29
  846. package/src/rules/capability-transform-resolver.js +80 -0
  847. package/src/rules/execution-contract.d.ts +73 -0
  848. package/src/rules/execution-contract.js +429 -0
  849. package/src/rules/execution-rules-contract-check.js +1 -1
  850. package/src/rules/generated/rule-bundle.json +4305 -0
  851. package/src/rules/inference/capability-transforms.rules.json +3 -3
  852. package/src/rules/layer-pattern-contract-check.js +1 -1
  853. package/src/rules/rule-registry.js +70 -57
  854. package/src/storage/artifact-storage-context.js +2 -2
  855. package/src/storage/backend-selection.js +43 -0
  856. package/src/storage/distribution-transport.js +5 -0
  857. package/src/storage/download/integrity.d.ts +33 -0
  858. package/src/storage/download/integrity.js +130 -0
  859. package/src/storage/download/resume-inspection.d.ts +22 -0
  860. package/src/storage/download/resume-inspection.js +134 -0
  861. package/src/storage/download/retry.js +57 -0
  862. package/src/storage/download/state.js +131 -0
  863. package/src/storage/download/transport.js +99 -0
  864. package/src/storage/downloader.js +74 -453
  865. package/src/storage/model-cache.d.ts +54 -0
  866. package/src/storage/model-cache.js +590 -0
  867. package/src/storage/shard-manager.d.ts +5 -0
  868. package/src/storage/shard-manager.js +50 -369
  869. package/src/storage/shards/index.js +60 -0
  870. package/src/storage/shards/integrity.js +225 -0
  871. package/src/storage/shards/lifecycle.js +22 -0
  872. package/src/storage/source-artifact-store.js +14 -4
  873. package/src/tooling/boundary-evidence.d.ts +51 -0
  874. package/src/tooling/boundary-evidence.js +408 -0
  875. package/src/tooling/browser-command-runner.js +17 -10
  876. package/src/tooling/capsule-v2.d.ts +11 -0
  877. package/src/tooling/capsule-v2.js +58 -0
  878. package/src/tooling/capsule.js +20 -0
  879. package/src/tooling/command-api-constants.js +1 -1
  880. package/src/tooling/command-api-family-normalizers.js +163 -5
  881. package/src/tooling/command-api.d.ts +42 -3
  882. package/src/tooling/command-api.js +7 -2
  883. package/src/tooling/command-context.d.ts +25 -0
  884. package/src/tooling/command-context.js +53 -0
  885. package/src/tooling/command-envelope.js +1 -1
  886. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  887. package/src/tooling/command-runner.html +49 -1
  888. package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +22 -5
  889. package/src/tooling/command-validation.js +1 -24
  890. package/src/tooling/conversion-run-timing.js +24 -0
  891. package/src/tooling/demo-receipts.d.ts +56 -0
  892. package/src/tooling/demo-receipts.js +113 -0
  893. package/src/tooling/diagnose-runner.js +13 -11
  894. package/src/tooling/distribution-shard-transport.js +11 -0
  895. package/src/tooling/execution-cost-ledger.d.ts +95 -0
  896. package/src/tooling/execution-cost-ledger.js +381 -0
  897. package/src/tooling/hf-registry/catalog-io.js +63 -0
  898. package/src/tooling/litert/package.js +863 -0
  899. package/src/tooling/litert-package-runtime.js +3 -849
  900. package/src/tooling/model-capsule-forge.js +478 -0
  901. package/src/tooling/node-browser/execution.d.ts +40 -0
  902. package/src/tooling/node-browser/execution.js +308 -0
  903. package/src/tooling/node-browser/static-server.js +188 -0
  904. package/src/tooling/node-browser/transport.d.ts +80 -0
  905. package/src/tooling/node-browser/transport.js +622 -0
  906. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  907. package/src/tooling/node-browser-command-runner.js +10 -1112
  908. package/src/tooling/node-capsule-artifact-store.js +44 -0
  909. package/src/tooling/node-command-execution.js +237 -0
  910. package/src/tooling/node-command-runner.js +26 -205
  911. package/src/tooling/node-converter/input.js +521 -0
  912. package/src/tooling/node-converter/output.js +244 -0
  913. package/src/tooling/node-converter-chunk-batches.js +21 -0
  914. package/src/tooling/node-converter.js +7 -720
  915. package/src/tooling/node-source-runtime.js +16 -276
  916. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  917. package/src/tooling/node-webgpu.js +178 -453
  918. package/src/tooling/opfs-cache.d.ts +15 -48
  919. package/src/tooling/opfs-cache.js +8 -510
  920. package/src/tooling/precision-replay-math.d.ts +42 -0
  921. package/src/tooling/precision-replay-math.js +174 -0
  922. package/src/tooling/production-release.d.ts +19 -0
  923. package/src/tooling/production-release.js +647 -0
  924. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  925. package/src/tooling/program-bundle/materialize.js +324 -0
  926. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  927. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  928. package/src/tooling/program-bundle/validation.js +31 -0
  929. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  930. package/src/tooling/program-bundle-parity.d.ts +14 -4
  931. package/src/tooling/program-bundle-parity.js +267 -106
  932. package/src/tooling/program-bundle.d.ts +8 -52
  933. package/src/tooling/program-bundle.js +186 -730
  934. package/src/tooling/provider-v1-contract.d.ts +61 -0
  935. package/src/tooling/provider-v1-contract.js +192 -0
  936. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  937. package/src/tooling/registered-variant-calibration-job.js +210 -0
  938. package/src/tooling/registered-variant-calibration.d.ts +184 -0
  939. package/src/tooling/registered-variant-calibration.js +336 -0
  940. package/src/tooling/runtime-input-composition.js +105 -53
  941. package/src/tooling/runtime-intent.js +41 -0
  942. package/src/tooling/runtime-optimization/candidates.js +664 -0
  943. package/src/tooling/runtime-optimization-campaign.js +166 -0
  944. package/src/tooling/runtime-optimization-index.d.ts +6 -0
  945. package/src/tooling/runtime-optimization-index.js +87 -0
  946. package/src/tooling/runtime-optimization.d.ts +91 -6
  947. package/src/tooling/runtime-optimization.js +290 -414
  948. package/src/tooling/source-intake.d.ts +49 -0
  949. package/src/tooling/source-intake.js +431 -0
  950. package/src/tooling/source-package-profiles.js +1 -1
  951. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  952. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  953. package/src/tooling/source-runtime/node-loader.js +258 -0
  954. package/src/tooling/source-runtime-bundle.js +26 -411
  955. package/src/tooling/source-runtime-converter-config.js +1 -1
  956. package/src/tooling-experimental-exports.js +0 -5
  957. package/src/tooling-exports/evidence.d.ts +45 -0
  958. package/src/tooling-exports/evidence.js +45 -0
  959. package/src/tooling-exports/runtime.d.ts +7 -0
  960. package/src/tooling-exports/runtime.js +7 -0
  961. package/src/tooling-exports/storage.d.ts +5 -0
  962. package/src/tooling-exports/storage.js +4 -0
  963. package/src/tooling-exports.d.ts +14 -0
  964. package/src/tooling-exports.js +8 -3
  965. package/src/tooling-exports.shared.d.ts +46 -0
  966. package/src/tooling-exports.shared.js +37 -0
  967. package/src/training.d.ts +6 -6
  968. package/src/training.js +4 -4
  969. package/src/version.js +1 -1
  970. package/tests/kernels/browser/kernel-suite.js +512 -0
  971. package/tests/kernels/browser/test-page.js +169 -14
  972. package/tests/kernels/harness/tolerance.js +1 -0
  973. package/tests/kernels/reference/index.js +13 -1
  974. package/tests/kernels/reference/vision.js +160 -0
  975. package/src/client/runtime/model-session.js +0 -432
  976. package/src/config/schema/tuner.schema.d.ts +0 -39
  977. package/src/config/schema/tuner.schema.js +0 -13
  978. package/src/converter/quantizer.d.ts +0 -121
  979. package/src/gpu/kernel-tuner/benchmarks.js +0 -902
  980. package/src/gpu/kernel-tuner/cache.js +0 -145
  981. package/src/gpu/kernel-tuner/index.js +0 -38
  982. package/src/gpu/kernel-tuner/tuner.js +0 -247
  983. package/src/gpu/kernel-tuner.js +0 -12
  984. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  985. package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
  986. package/src/gpu/kernels/kernel-tuning.js +0 -158
  987. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  988. package/src/gpu/kernels/utils.d.ts +0 -149
  989. package/src/gpu/profiler.js +0 -382
  990. package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
  991. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  992. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  993. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  994. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  995. package/src/rules/converter/execution.rules.json +0 -20
  996. package/src/rules/converter/tensor-roles.rules.json +0 -13
  997. package/src/rules/converter/tokenizer.rules.json +0 -12
  998. package/src/rules/inference/attention.rules.json +0 -69
  999. package/src/rules/inference/config.rules.json +0 -82
  1000. package/src/rules/inference/dtype.rules.json +0 -113
  1001. package/src/rules/inference/execution.rules.json +0 -195
  1002. package/src/rules/inference/ffn.rules.json +0 -123
  1003. package/src/rules/inference/layer-pattern.rules.json +0 -16
  1004. package/src/rules/inference/layer.rules.json +0 -7
  1005. package/src/rules/inference/moe.rules.json +0 -145
  1006. package/src/rules/kernels/attention.rules.json +0 -68
  1007. package/src/rules/kernels/conv2d.rules.json +0 -6
  1008. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  1009. package/src/rules/kernels/dequant.rules.json +0 -58
  1010. package/src/rules/kernels/energy.rules.json +0 -22
  1011. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  1012. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  1013. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  1014. package/src/rules/kernels/gather.rules.json +0 -93
  1015. package/src/rules/kernels/gelu.rules.json +0 -11
  1016. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  1017. package/src/rules/kernels/groupnorm.rules.json +0 -10
  1018. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  1019. package/src/rules/kernels/layernorm.rules.json +0 -6
  1020. package/src/rules/kernels/linear-attention.rules.json +0 -6
  1021. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  1022. package/src/rules/kernels/matmul.rules.json +0 -95
  1023. package/src/rules/kernels/modulate.rules.json +0 -6
  1024. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  1025. package/src/rules/kernels/moe.rules.json +0 -14
  1026. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  1027. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  1028. package/src/rules/kernels/relu.rules.json +0 -6
  1029. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  1030. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  1031. package/src/rules/kernels/residual.rules.json +0 -12
  1032. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  1033. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  1034. package/src/rules/kernels/rope-qk.rules.json +0 -6
  1035. package/src/rules/kernels/rope.rules.json +0 -6
  1036. package/src/rules/kernels/sample.rules.json +0 -6
  1037. package/src/rules/kernels/scale.rules.json +0 -8
  1038. package/src/rules/kernels/silu.rules.json +0 -22
  1039. package/src/rules/kernels/softmax.rules.json +0 -25
  1040. package/src/rules/kernels/split-qg.rules.json +0 -6
  1041. package/src/rules/kernels/split-qkv.rules.json +0 -6
  1042. package/src/rules/kernels/upsample2d.rules.json +0 -6
  1043. package/src/rules/loader/tensor-loader.rules.json +0 -42
  1044. package/src/rules/loader/weights.rules.json +0 -42
  1045. package/src/rules/tooling/command-runtime.rules.json +0 -65
  1046. package/src/tooling/hf-registry-utils.js +0 -395
  1047. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  1048. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  1049. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  1050. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  1051. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  1052. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  1053. /package/src/{utils → formats}/array-buffer.js +0 -0
  1054. /package/src/{utils → formats}/canonical-hash.js +0 -0
  1055. /package/src/{utils → formats}/clone-json.js +0 -0
  1056. /package/src/{utils → formats}/encode-utf8.js +0 -0
  1057. /package/src/{utils → formats}/load-json.js +0 -0
  1058. /package/src/{utils → formats}/plain-object.js +0 -0
  1059. /package/src/{utils → formats}/sha256.js +0 -0
  1060. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  1061. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  1062. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  1063. /package/src/{utils → storage}/runtime-env.js +0 -0
@@ -26,63 +26,10 @@ import { selectRuleValue } from '../../../rules/rule-registry.js';
26
26
  import { getDevice } from '../../../gpu/device.js';
27
27
  import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
28
28
  import { f16ToF32 } from '../../../loader/dtype-utils.js';
29
-
30
- const pleRuntimeCache = new WeakMap();
31
- const pleRangeRowCache = new WeakMap();
32
- let f16ToF32Lookup = null;
33
- const PLE_ACTIVATION_STATIC_QMIN = -128;
34
- const PLE_ACTIVATION_STATIC_QMAX = 127;
35
-
36
- function getF16ToF32Lookup() {
37
- if (f16ToF32Lookup) {
38
- return f16ToF32Lookup;
39
- }
40
- const lookup = new Float32Array(1 << 16);
41
- for (let value = 0; value < lookup.length; value += 1) {
42
- lookup[value] = f16ToF32(value);
43
- }
44
- f16ToF32Lookup = lookup;
45
- return lookup;
46
- }
47
-
48
- function getPerLayerInputWeights(context) {
49
- const weights = context.weights.get('per_layer_inputs');
50
- if (!weights || typeof weights !== 'object') {
51
- throw new Error(
52
- 'Gemma 4 per-layer inputs require global per-layer input weights, ' +
53
- 'but state.weights.get("per_layer_inputs") was missing.'
54
- );
55
- }
56
- return weights;
57
- }
58
-
59
- function normalizePleProjectionNormDtype(dtype) {
60
- if (typeof dtype !== 'string') {
61
- return null;
62
- }
63
- const value = dtype.toLowerCase();
64
- if (value === 'f16' || value === 'f32') {
65
- return value;
66
- }
67
- return null;
68
- }
69
-
70
- function getPleProjectionNormDtype(weight) {
71
- return normalizePleProjectionNormDtype(getWeightDtype(weight))
72
- ?? normalizePleProjectionNormDtype(getBufferDtype(weight))
73
- ?? null;
74
- }
75
-
76
- function isRangeBackedPleProjectionSource(value) {
77
- return isRangeBackedCpuEmbeddingSource(value);
78
- }
79
-
80
- function getPleProjectionWeightDtype(weight) {
81
- if (isCpuWeightBuffer(weight)) {
82
- return weight.dtype ?? null;
83
- }
84
- return getWeightDtype(weight);
85
- }
29
+ import { destroyPleHotVocabularyRuntime, destroyPleRuntimeCacheEntry, ensurePleScaledProjectionNormWeight, getPerLayerInputWeights, getPleHotCachePolicy, getPleRangeRowLoadConfig, getPleSplitTablePolicy, pleRuntimeCache } from './per-layer/materialize.js';
30
+ import { applyPleActivationStaticQdq, getPleProjectionWeightDtype, getPreparedTokenEntry, isRangeBackedPleProjectionSource, loadRangeBackedPleProjectionSliceBytes, loadRangeBackedPleRow, pleRangeRowCache, storePreparedTokenEntry } from './per-layer/plan.js';
31
+ export { loadRangeBackedPleProjectionSliceBytes } from './per-layer/plan.js';
32
+ export { ensurePleGpuHotVocabularyRuntime, ensurePleGpuSplitTablesRuntime, ensurePleScaledProjectionNormWeight, inferPleProjectionNormDtype, resolvePleHotVocabularyCapacity, scalePerLayerProjectionNormWeights } from './per-layer/materialize.js';
86
33
 
87
34
  function isDensePleProjectionDtype(dtype) {
88
35
  return dtype === 'f16' || dtype === 'f32';
@@ -125,69 +72,6 @@ function resolvePleActivationStaticQdqConfig(perLayerInputWeights) {
125
72
  };
126
73
  }
127
74
 
128
- function getTensorElementCount(tensor, label) {
129
- if (!Array.isArray(tensor?.shape) || tensor.shape.length === 0) {
130
- throw new Error(`${label} requires a tensor with an explicit shape.`);
131
- }
132
- const count = tensor.shape.reduce((total, dimension) => total * Number(dimension), 1);
133
- if (!Number.isFinite(count) || count <= 0) {
134
- throw new Error(`${label} resolved an invalid tensor element count (${String(count)}).`);
135
- }
136
- return count;
137
- }
138
-
139
- async function applyPleActivationStaticQdq(tensor, scale, recorder, decodeBuffers, label) {
140
- const count = getTensorElementCount(tensor, label);
141
- if (tensor.dtype === 'f32') {
142
- return recorder
143
- ? await recordActivationStaticQdq(recorder, tensor, scale, {
144
- count,
145
- qmin: PLE_ACTIVATION_STATIC_QMIN,
146
- qmax: PLE_ACTIVATION_STATIC_QMAX,
147
- })
148
- : await runActivationStaticQdq(tensor, scale, {
149
- count,
150
- qmin: PLE_ACTIVATION_STATIC_QMIN,
151
- qmax: PLE_ACTIVATION_STATIC_QMAX,
152
- });
153
- }
154
- if (tensor.dtype !== 'f16') {
155
- throw new Error(
156
- `${label} requires f16/f32 activations for static activation quantize/dequantize. ` +
157
- `Got "${String(tensor.dtype)}".`
158
- );
159
- }
160
-
161
- let widenedTensor = null;
162
- let qdqTensor = null;
163
- try {
164
- widenedTensor = recorder
165
- ? await recordCastF16ToF32(recorder, tensor)
166
- : await castF16ToF32(tensor);
167
- qdqTensor = recorder
168
- ? await recordActivationStaticQdq(recorder, widenedTensor, scale, {
169
- count,
170
- qmin: PLE_ACTIVATION_STATIC_QMIN,
171
- qmax: PLE_ACTIVATION_STATIC_QMAX,
172
- })
173
- : await runActivationStaticQdq(widenedTensor, scale, {
174
- count,
175
- qmin: PLE_ACTIVATION_STATIC_QMIN,
176
- qmax: PLE_ACTIVATION_STATIC_QMAX,
177
- });
178
- return recorder
179
- ? await recordCastF32ToF16(recorder, qdqTensor)
180
- : await castF32ToF16(qdqTensor);
181
- } finally {
182
- if (qdqTensor) {
183
- releaseOrTrack(recorder, qdqTensor.buffer, decodeBuffers);
184
- }
185
- if (widenedTensor) {
186
- releaseOrTrack(recorder, widenedTensor.buffer, decodeBuffers);
187
- }
188
- }
189
- }
190
-
191
75
  export function resolveDensePleProjectionWeight(
192
76
  weight,
193
77
  label = 'per_layer_model_projection'
@@ -225,81 +109,6 @@ export function resolveDensePleProjectionWeight(
225
109
  return weight;
226
110
  }
227
111
 
228
- export async function loadRangeBackedPleProjectionSliceBytes(
229
- weight,
230
- layerIdx,
231
- hiddenSizePerLayerInput,
232
- hiddenSize,
233
- label = 'Range-backed PLE projection slice'
234
- ) {
235
- if (!isCpuWeightBuffer(weight)) {
236
- return null;
237
- }
238
- const cpuData = weight.data;
239
- if (!isRangeBackedPleProjectionSource(cpuData)) {
240
- return null;
241
- }
242
-
243
- const dtype = getPleProjectionWeightDtype(weight);
244
- const layout = getLayout(weight) ?? weight.layout ?? 'row';
245
- const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
246
- dtype,
247
- });
248
- const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
249
- const byteOffset = hiddenOffset * hiddenSize * bytesPerElement;
250
- const byteLength = hiddenSizePerLayerInput * hiddenSize * bytesPerElement;
251
- const bytes = normalizeRangeBytes(
252
- await cpuData.loadRange(byteOffset, byteLength),
253
- label
254
- );
255
- if (bytes.byteLength !== byteLength) {
256
- throw new Error(
257
- `${label} short read for layer ${layerIdx}: expected ${byteLength} bytes, got ${bytes.byteLength}.`
258
- );
259
- }
260
- return {
261
- bytes,
262
- dtype,
263
- layout,
264
- shape: [hiddenSizePerLayerInput, hiddenSize],
265
- };
266
- }
267
-
268
- export function inferPleProjectionNormDtype(weight, hiddenSizePerLayerInput) {
269
- const explicitDtype = getPleProjectionNormDtype(weight);
270
- if (explicitDtype) {
271
- return explicitDtype;
272
- }
273
-
274
- if (!isGpuBufferInstance(weight)) {
275
- return 'f32';
276
- }
277
-
278
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
279
- throw new Error('Gemma 4 per-layer projection norm dtype inference requires hiddenSizePerLayerInput > 0.');
280
- }
281
-
282
- const bytesPerElement = weight.size / hiddenSizePerLayerInput;
283
- if (bytesPerElement !== 2 && bytesPerElement !== 4) {
284
- throw new Error(
285
- 'Gemma 4 per-layer projection norm buffer has unsupported byte size: ' +
286
- `bufferSize=${weight.size}, hiddenSizePerLayerInput=${hiddenSizePerLayerInput}.`
287
- );
288
- }
289
- return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
290
- }
291
-
292
- function createPleProjectionNormTensor(buffer, dtype, hiddenSizePerLayerInput, label) {
293
- return createTensor(buffer, dtype, [hiddenSizePerLayerInput], label);
294
- }
295
-
296
- function destroyPleRuntimeCacheEntry(entry) {
297
- const cachedBuffer = entry?.scaledProjectionNormWeight?.buffer ?? null;
298
- if (cachedBuffer) {
299
- releaseBuffer(cachedBuffer);
300
- }
301
- }
302
-
303
112
  function destroyPleSplitTables(perLayerInputWeights) {
304
113
  const splitTables = Array.isArray(perLayerInputWeights?.embedTokensPerLayerSplit)
305
114
  ? perLayerInputWeights.embedTokensPerLayerSplit
@@ -316,25 +125,6 @@ function destroyPleSplitTables(perLayerInputWeights) {
316
125
  delete perLayerInputWeights.embedTokensPerLayerSplit;
317
126
  }
318
127
 
319
- function destroyPleHotVocabularyRuntime(perLayerInputWeights) {
320
- const runtime = perLayerInputWeights?.embedTokensPerLayerHotRuntime ?? null;
321
- if (!runtime || typeof runtime !== 'object') {
322
- return;
323
- }
324
- const splitTables = Array.isArray(runtime.splitTables) ? runtime.splitTables : [];
325
- for (const table of splitTables) {
326
- const buffer = table?.buffer ?? table ?? null;
327
- if (buffer) {
328
- releaseBuffer(buffer);
329
- }
330
- }
331
- const mapBuffer = runtime.hotTokenIndexMapBuffer ?? null;
332
- if (mapBuffer) {
333
- releaseBuffer(mapBuffer);
334
- }
335
- delete perLayerInputWeights.embedTokensPerLayerHotRuntime;
336
- }
337
-
338
128
  export function destroyPleRuntimeCache(perLayerInputWeights) {
339
129
  if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
340
130
  return;
@@ -351,126 +141,6 @@ export function destroyPleRuntimeCache(perLayerInputWeights) {
351
141
  }
352
142
  }
353
143
 
354
- export function scalePerLayerProjectionNormWeights(weight, combineScale, rmsNormWeightOffset = false) {
355
- if (rmsNormWeightOffset) {
356
- return null;
357
- }
358
- const source = isCpuWeightBuffer(weight) ? weight.data : weight;
359
- const isArrayLikeView = ArrayBuffer.isView(source) && typeof source.length === 'number';
360
- if (!(source instanceof Float32Array) && !isArrayLikeView && !Array.isArray(source)) {
361
- return null;
362
- }
363
- const scaled = Float32Array.from(source);
364
- for (let i = 0; i < scaled.length; i++) {
365
- scaled[i] *= combineScale;
366
- }
367
- return scaled;
368
- }
369
-
370
- export async function ensurePleScaledProjectionNormWeight(context, combineScale = 2 ** -0.5) {
371
- const hiddenSizePerLayerInput = Number(context?.config?.hiddenSizePerLayerInput ?? 0);
372
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
373
- return null;
374
- }
375
- if (context?.weightConfig?.rmsNormWeightOffset) {
376
- return null;
377
- }
378
-
379
- const perLayerInputWeights = getPerLayerInputWeights(context);
380
- const cachedEntry = pleRuntimeCache.get(perLayerInputWeights) ?? null;
381
- if (cachedEntry?.combineScale === combineScale && cachedEntry.scaledProjectionNormWeight) {
382
- return cachedEntry.scaledProjectionNormWeight;
383
- }
384
- if (cachedEntry) {
385
- destroyPleRuntimeCacheEntry(cachedEntry);
386
- pleRuntimeCache.delete(perLayerInputWeights);
387
- }
388
-
389
- const projectionNormWeight = perLayerInputWeights.perLayerProjectionNorm;
390
- if (!projectionNormWeight) {
391
- return null;
392
- }
393
-
394
- let scaledProjectionNormWeight = null;
395
- if (isGpuBufferInstance(projectionNormWeight)) {
396
- const projectionNormDtype = inferPleProjectionNormDtype(
397
- projectionNormWeight,
398
- hiddenSizePerLayerInput
399
- );
400
- const scaledTensor = await runScale(
401
- createPleProjectionNormTensor(
402
- projectionNormWeight,
403
- projectionNormDtype,
404
- hiddenSizePerLayerInput,
405
- 'per_layer_projection_norm'
406
- ),
407
- combineScale,
408
- { count: hiddenSizePerLayerInput }
409
- );
410
- scaledProjectionNormWeight = createPleProjectionNormTensor(
411
- scaledTensor.buffer,
412
- projectionNormDtype,
413
- hiddenSizePerLayerInput,
414
- 'per_layer_projection_norm_scaled'
415
- );
416
- } else {
417
- const scaledValues = scalePerLayerProjectionNormWeights(
418
- projectionNormWeight,
419
- combineScale,
420
- false
421
- );
422
- if (!scaledValues) {
423
- return null;
424
- }
425
- if (scaledValues.length !== hiddenSizePerLayerInput) {
426
- throw new Error(
427
- 'Gemma 4 per-layer projection norm cache shape mismatch: ' +
428
- `expected ${hiddenSizePerLayerInput} values, got ${scaledValues.length}.`
429
- );
430
- }
431
- const device = getDevice();
432
- if (!device) {
433
- throw new Error('No GPU device available for Gemma 4 per-layer projection norm cache.');
434
- }
435
- const scaledBuffer = acquireBuffer(
436
- scaledValues.byteLength,
437
- undefined,
438
- 'per_layer_projection_norm_scaled'
439
- );
440
- try {
441
- device.queue.writeBuffer(scaledBuffer, 0, scaledValues);
442
- } catch (error) {
443
- releaseBuffer(scaledBuffer);
444
- throw error;
445
- }
446
- scaledProjectionNormWeight = createPleProjectionNormTensor(
447
- scaledBuffer,
448
- inferPleProjectionNormDtype(projectionNormWeight, hiddenSizePerLayerInput),
449
- hiddenSizePerLayerInput,
450
- 'per_layer_projection_norm_scaled'
451
- );
452
- }
453
-
454
- if (scaledProjectionNormWeight?.shape?.[0] !== hiddenSizePerLayerInput) {
455
- throw new Error(
456
- 'Gemma 4 per-layer projection norm cache shape mismatch after tensor creation: ' +
457
- `expected ${hiddenSizePerLayerInput}, got ${scaledProjectionNormWeight?.shape?.[0] ?? 'unknown'}.`
458
- );
459
- }
460
- if (!normalizePleProjectionNormDtype(scaledProjectionNormWeight?.dtype)) {
461
- throw new Error(
462
- 'Gemma 4 per-layer projection norm cache produced an invalid dtype: ' +
463
- `"${String(scaledProjectionNormWeight?.dtype ?? 'undefined')}".`
464
- );
465
- }
466
-
467
- pleRuntimeCache.set(perLayerInputWeights, {
468
- combineScale,
469
- scaledProjectionNormWeight,
470
- });
471
- return scaledProjectionNormWeight;
472
- }
473
-
474
144
  function getEmbeddingSource(weight, label) {
475
145
  if (isWeightBuffer(weight)) {
476
146
  return weight.buffer;
@@ -488,17 +158,6 @@ function getEmbeddingDtype(weight) {
488
158
  return getWeightDtype(weight);
489
159
  }
490
160
 
491
- function requirePleSourceDtype(rawDtype, label, allowed = ['f16', 'f32']) {
492
- if (rawDtype == null) {
493
- throw new Error(`${label} requires source dtype metadata.`);
494
- }
495
- const sourceDtype = String(rawDtype).toLowerCase();
496
- if (!allowed.includes(sourceDtype)) {
497
- throw new Error(`${label} requires ${allowed.join('/')} source rows; got "${sourceDtype}".`);
498
- }
499
- return sourceDtype;
500
- }
501
-
502
161
  function getEmbeddingTranspose(weight) {
503
162
  if (isWeightBuffer(weight) || isCpuWeightBuffer(weight)) {
504
163
  return weight.layout === 'column';
@@ -560,107 +219,6 @@ function releasePleSliceBuffer(recorder, buffer, decodeBuffers, cache) {
560
219
  releaseOrTrack(recorder, buffer, decodeBuffers);
561
220
  }
562
221
 
563
- function getPleRowCachePolicy(sessionConfig) {
564
- const rowCache = sessionConfig?.rowCache ?? null;
565
- if (!rowCache || rowCache.mode === 'off') {
566
- return null;
567
- }
568
- if (rowCache.mode !== 'lru') {
569
- throw new Error(
570
- `Gemma 4 per-layer input row cache mode "${String(rowCache.mode)}" is not implemented.`
571
- );
572
- }
573
- const decodedDtype = String(rowCache.decodedDtype ?? '').toLowerCase();
574
- if (decodedDtype !== 'f32') {
575
- throw new Error(
576
- `Gemma 4 range-backed per-layer input row cache requires rowCache.decodedDtype="f32"; ` +
577
- `got "${String(rowCache.decodedDtype)}".`
578
- );
579
- }
580
- const maxRows = Math.trunc(Number(rowCache.maxRows));
581
- const maxBytes = Math.trunc(Number(rowCache.maxBytes));
582
- if (!Number.isFinite(maxRows) || maxRows <= 0) {
583
- throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxRows > 0.');
584
- }
585
- if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
586
- throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxBytes > 0.');
587
- }
588
- return { maxRows, maxBytes };
589
- }
590
-
591
- function getPleHotCachePolicy(sessionConfig) {
592
- const hotCache = sessionConfig?.hotCache ?? null;
593
- if (!hotCache || hotCache.mode === 'off') {
594
- return null;
595
- }
596
- if (hotCache.mode === 'tokenizer_scores') {
597
- const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
598
- if (outputDtype !== 'f16' && outputDtype !== 'f32') {
599
- throw new Error(
600
- `Gemma 4 per-layer input hot vocabulary cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
601
- `got "${String(hotCache.outputDtype)}".`
602
- );
603
- }
604
- const maxTokens = Math.trunc(Number(hotCache.maxTokens));
605
- const maxBytes = Math.trunc(Number(hotCache.maxBytes));
606
- if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
607
- throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxTokens > 0.');
608
- }
609
- if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
610
- throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxBytes > 0.');
611
- }
612
- return {
613
- mode: 'tokenizer_scores',
614
- maxTokens,
615
- maxBytes,
616
- outputDtype,
617
- };
618
- }
619
- if (hotCache.mode !== 'prepared_tokens') {
620
- throw new Error(
621
- `Gemma 4 per-layer input hot cache mode "${String(hotCache.mode)}" is not implemented.`
622
- );
623
- }
624
- const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
625
- if (outputDtype !== 'f16' && outputDtype !== 'f32') {
626
- throw new Error(
627
- `Gemma 4 per-layer input hot cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
628
- `got "${String(hotCache.outputDtype)}".`
629
- );
630
- }
631
- const maxTokens = Math.trunc(Number(hotCache.maxTokens));
632
- const maxBytes = Math.trunc(Number(hotCache.maxBytes));
633
- if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
634
- throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxTokens > 0.');
635
- }
636
- if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
637
- throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxBytes > 0.');
638
- }
639
- return { mode: 'prepared_tokens', maxTokens, maxBytes, outputDtype };
640
- }
641
-
642
- export function resolvePleHotVocabularyCapacity({
643
- maxTokens,
644
- maxBytes,
645
- numLayers,
646
- hiddenSize,
647
- bytesPerElement,
648
- vocabSize,
649
- }) {
650
- const bytesPerHotRow = numLayers * hiddenSize * bytesPerElement;
651
- const tokenIndexMapBytes = vocabSize * Uint32Array.BYTES_PER_ELEMENT;
652
- const tableBudgetBytes = Math.max(0, maxBytes - tokenIndexMapBytes);
653
- const maxTableRows = bytesPerHotRow > 0
654
- ? Math.floor(tableBudgetBytes / bytesPerHotRow)
655
- : 0;
656
- const maxHotTokens = Math.max(0, Math.min(maxTokens, maxTableRows - 1));
657
- return {
658
- maxHotTokens,
659
- bytesPerHotRow,
660
- tokenIndexMapBytes,
661
- };
662
- }
663
-
664
222
  export function getPleHotVocabularyRuntime(context) {
665
223
  const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
666
224
  if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
@@ -670,212 +228,6 @@ export function getPleHotVocabularyRuntime(context) {
670
228
  return runtime && typeof runtime === 'object' ? runtime : null;
671
229
  }
672
230
 
673
- function getPleSplitTablePolicy(sessionConfig) {
674
- if (sessionConfig?.materialization !== 'gpu_split_tables') {
675
- return null;
676
- }
677
- return { mode: 'gpu_split_tables' };
678
- }
679
-
680
- function releasePreparedTokenEntry(cache, tokenId, entry) {
681
- if (!cache || !entry) {
682
- return;
683
- }
684
- cache.preparedTokenEntries?.delete(tokenId);
685
- cache.preparedTokenBytes -= entry?.bytes ?? 0;
686
- for (const buffer of entry.buffers ?? []) {
687
- if (!buffer) continue;
688
- cache.ownedBuffers?.delete(buffer);
689
- releaseBuffer(buffer);
690
- }
691
- }
692
-
693
- function prunePreparedTokenCache(cache, policy) {
694
- while (
695
- cache.preparedTokenEntries.size > policy.maxTokens
696
- || cache.preparedTokenBytes > policy.maxBytes
697
- ) {
698
- const oldest = cache.preparedTokenEntries.keys().next();
699
- if (oldest.done) {
700
- break;
701
- }
702
- const tokenId = oldest.value;
703
- const entry = cache.preparedTokenEntries.get(tokenId);
704
- releasePreparedTokenEntry(cache, tokenId, entry);
705
- }
706
- }
707
-
708
- function getPreparedTokenEntry(cache, tokenId, sessionConfig, activationDtype, stats = null) {
709
- const policy = getPleHotCachePolicy(sessionConfig);
710
- if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
711
- return null;
712
- }
713
- if (policy.outputDtype !== activationDtype) {
714
- throw new Error(
715
- `Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
716
- `got "${String(activationDtype)}".`
717
- );
718
- }
719
- const entry = cache.preparedTokenEntries.get(tokenId) ?? null;
720
- if (!entry) {
721
- if (stats) {
722
- stats.plePreparedTokenCacheMisses = (stats.plePreparedTokenCacheMisses ?? 0) + 1;
723
- stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
724
- stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
725
- }
726
- return null;
727
- }
728
- cache.preparedTokenEntries.delete(tokenId);
729
- cache.preparedTokenEntries.set(tokenId, entry);
730
- if (stats) {
731
- stats.plePreparedTokenCacheHits = (stats.plePreparedTokenCacheHits ?? 0) + 1;
732
- stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
733
- stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
734
- }
735
- return entry.buffers.slice();
736
- }
737
-
738
- function storePreparedTokenEntry(cache, tokenId, buffers, sessionConfig, activationDtype, stats = null) {
739
- const policy = getPleHotCachePolicy(sessionConfig);
740
- if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
741
- return buffers;
742
- }
743
- if (policy.outputDtype !== activationDtype) {
744
- throw new Error(
745
- `Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
746
- `got "${String(activationDtype)}".`
747
- );
748
- }
749
- const existing = cache.preparedTokenEntries.get(tokenId) ?? null;
750
- if (existing) {
751
- releasePreparedTokenEntry(cache, tokenId, existing);
752
- }
753
- const cachedBuffers = buffers.slice();
754
- const bytes = cachedBuffers.reduce((total, buffer) => total + (buffer?.size ?? 0), 0);
755
- for (const buffer of cachedBuffers) {
756
- if (buffer) {
757
- cache.ownedBuffers?.add(buffer);
758
- }
759
- }
760
- cache.preparedTokenEntries.set(tokenId, { buffers: cachedBuffers, bytes });
761
- cache.preparedTokenBytes += bytes;
762
- prunePreparedTokenCache(cache, policy);
763
- if (stats) {
764
- stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
765
- stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
766
- }
767
- return cachedBuffers.slice();
768
- }
769
-
770
- function getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize) {
771
- const sourceDtype = requirePleSourceDtype(
772
- (isCpuWeightBuffer(embedTokensPerLayer) ? embedTokensPerLayer.data?.sourceDtype : null)
773
- ?? embedTokensPerLayer?.dtype,
774
- 'Gemma 4 range-backed per-layer input rows',
775
- ['f16', 'bf16', 'f32']
776
- );
777
- const bytesPerElement = (sourceDtype === 'f16' || sourceDtype === 'bf16') ? 2 : 4;
778
- return {
779
- sourceDtype,
780
- sourceRowBytes: totalPerLayerHiddenSize * bytesPerElement,
781
- };
782
- }
783
-
784
- function getPleRangeRowCache(embedTokensPerLayer, sessionConfig) {
785
- const policy = getPleRowCachePolicy(sessionConfig);
786
- if (!policy) {
787
- return null;
788
- }
789
- const cached = pleRangeRowCache.get(embedTokensPerLayer);
790
- if (cached && cached.maxRows === policy.maxRows && cached.maxBytes === policy.maxBytes) {
791
- return cached;
792
- }
793
- const next = {
794
- maxRows: policy.maxRows,
795
- maxBytes: policy.maxBytes,
796
- totalBytes: 0,
797
- rows: new Map(),
798
- };
799
- pleRangeRowCache.set(embedTokensPerLayer, next);
800
- return next;
801
- }
802
-
803
- function touchPleCachedRow(cache, tokenId) {
804
- const hit = cache?.rows?.get(tokenId) ?? null;
805
- if (!hit) {
806
- return null;
807
- }
808
- cache.rows.delete(tokenId);
809
- cache.rows.set(tokenId, hit);
810
- return hit.row;
811
- }
812
-
813
- function prunePleRangeRowCache(cache) {
814
- while (cache.rows.size > cache.maxRows || cache.totalBytes > cache.maxBytes) {
815
- const oldest = cache.rows.keys().next();
816
- if (oldest.done) {
817
- break;
818
- }
819
- const entry = cache.rows.get(oldest.value);
820
- cache.rows.delete(oldest.value);
821
- cache.totalBytes -= entry?.bytes ?? 0;
822
- }
823
- }
824
-
825
- function cachePleRangeRow(cache, tokenId, row) {
826
- if (!cache) {
827
- return row;
828
- }
829
- const existing = cache.rows.get(tokenId);
830
- if (existing) {
831
- cache.totalBytes -= existing.bytes;
832
- cache.rows.delete(tokenId);
833
- }
834
- cache.rows.set(tokenId, { row, bytes: row.byteLength });
835
- cache.totalBytes += row.byteLength;
836
- prunePleRangeRowCache(cache);
837
- return row;
838
- }
839
-
840
- async function loadRangeBackedPleRow(
841
- tokenId,
842
- embedTokensPerLayer,
843
- totalPerLayerHiddenSize,
844
- sessionConfig,
845
- label,
846
- prefetchedRow = null
847
- ) {
848
- if (!isCpuWeightBuffer(embedTokensPerLayer)) {
849
- return null;
850
- }
851
- const cpuData = embedTokensPerLayer.data;
852
- if (!isRangeBackedCpuEmbeddingSource(cpuData)) {
853
- return null;
854
- }
855
-
856
- const cache = getPleRangeRowCache(embedTokensPerLayer, sessionConfig);
857
- if (prefetchedRow && prefetchedRow.tokenId === tokenId) {
858
- return cachePleRangeRow(cache, tokenId, prefetchedRow.row);
859
- }
860
-
861
- const cached = touchPleCachedRow(cache, tokenId);
862
- if (cached) {
863
- return cached;
864
- }
865
-
866
- const { sourceDtype, sourceRowBytes } = getPleRangeRowLoadConfig(
867
- embedTokensPerLayer,
868
- totalPerLayerHiddenSize
869
- );
870
- const chunk = normalizeRangeBytes(
871
- await cpuData.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
872
- label
873
- );
874
- const row = new Float32Array(totalPerLayerHiddenSize);
875
- decodeRangeChunkIntoOutput(chunk, sourceDtype, row, 0, totalPerLayerHiddenSize);
876
- return cachePleRangeRow(cache, tokenId, row);
877
- }
878
-
879
231
  // Step 5: Prefetch next token's PLE row during current decode step.
880
232
  // Returns a promise resolving to { tokenId, row: Float32Array } or null.
881
233
  // Call after sampling produces the next token; pass result as options.prefetchedRow
@@ -934,391 +286,6 @@ export function hasGpuSplitPerLayerInputEmbeddings(context) {
934
286
  && perLayerInputWeights.embedTokensPerLayerSplit.length > 0;
935
287
  }
936
288
 
937
- export async function ensurePleGpuSplitTablesRuntime(context) {
938
- const policy = getPleSplitTablePolicy(context?.perLayerInputsSession ?? null);
939
- if (!policy) {
940
- return null;
941
- }
942
-
943
- const config = context?.config ?? null;
944
- const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
945
- const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
946
- const numLayers = Number(config?.numLayers ?? 0);
947
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
948
- return null;
949
- }
950
- if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
951
- return null;
952
- }
953
- if (!Number.isFinite(numLayers) || numLayers <= 0) {
954
- return null;
955
- }
956
-
957
- const perLayerInputWeights = getPerLayerInputWeights(context);
958
- if (Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit) && perLayerInputWeights.embedTokensPerLayerSplit.length === numLayers) {
959
- return perLayerInputWeights.embedTokensPerLayerSplit;
960
- }
961
-
962
- const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
963
- if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
964
- throw new Error('Gemma 4 gpu_split_tables materialization requires a range-backed CPU embedTokensPerLayer source.');
965
- }
966
-
967
- const sourceDtype = requirePleSourceDtype(
968
- embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
969
- 'Gemma 4 gpu_split_tables materialization'
970
- );
971
-
972
- const device = getDevice();
973
- if (!device) {
974
- throw new Error('No GPU device available for Gemma 4 gpu_split_tables materialization.');
975
- }
976
-
977
- const bytesPerElement = sourceDtype === 'f16' ? 2 : 4;
978
- const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
979
- const tableBytes = vocabSizePerLayerInput * hiddenSizePerLayerInput * bytesPerElement;
980
- const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
981
- acquireBuffer(tableBytes, undefined, `L${layerIdx}.ple_table_split`),
982
- sourceDtype,
983
- 'row',
984
- [vocabSizePerLayerInput, hiddenSizePerLayerInput],
985
- `L${layerIdx}.embed_tokens_per_layer_split`
986
- ));
987
-
988
- try {
989
- const rowsPerChunk = 128;
990
- for (let rowStart = 0; rowStart < vocabSizePerLayerInput; rowStart += rowsPerChunk) {
991
- const rowCount = Math.min(rowsPerChunk, vocabSizePerLayerInput - rowStart);
992
- const chunkByteOffset = rowStart * totalPerLayerHiddenSize * bytesPerElement;
993
- const chunkByteLength = rowCount * totalPerLayerHiddenSize * bytesPerElement;
994
- const chunk = normalizeRangeBytes(
995
- await embedTokensPerLayer.data.loadRange(chunkByteOffset, chunkByteLength),
996
- 'Gemma 4 split GPU PLE chunk'
997
- );
998
-
999
- if (sourceDtype === 'f16') {
1000
- const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
1001
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1002
- const layerWords = new Uint16Array(rowCount * hiddenSizePerLayerInput);
1003
- for (let row = 0; row < rowCount; row += 1) {
1004
- const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
1005
- layerWords.set(
1006
- sourceWords.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
1007
- row * hiddenSizePerLayerInput
1008
- );
1009
- }
1010
- device.queue.writeBuffer(
1011
- splitTables[layerIdx].buffer,
1012
- rowStart * hiddenSizePerLayerInput * bytesPerElement,
1013
- layerWords.buffer,
1014
- layerWords.byteOffset,
1015
- layerWords.byteLength
1016
- );
1017
- }
1018
- } else {
1019
- const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
1020
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1021
- const layerValues = new Float32Array(rowCount * hiddenSizePerLayerInput);
1022
- for (let row = 0; row < rowCount; row += 1) {
1023
- const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
1024
- layerValues.set(
1025
- sourceValues.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
1026
- row * hiddenSizePerLayerInput
1027
- );
1028
- }
1029
- device.queue.writeBuffer(
1030
- splitTables[layerIdx].buffer,
1031
- rowStart * hiddenSizePerLayerInput * bytesPerElement,
1032
- layerValues.buffer,
1033
- layerValues.byteOffset,
1034
- layerValues.byteLength
1035
- );
1036
- }
1037
- }
1038
- }
1039
- } catch (error) {
1040
- for (const table of splitTables) {
1041
- releaseBuffer(table.buffer);
1042
- }
1043
- throw error;
1044
- }
1045
-
1046
- perLayerInputWeights.embedTokensPerLayerSplit = splitTables;
1047
- return splitTables;
1048
- }
1049
-
1050
- function resolvePleHotVocabularySeedTokenIds(context, maxTokens, vocabSizePerLayerInput) {
1051
- const rawSeedTokenIds = Array.isArray(context?.seedTokenIds) ? context.seedTokenIds : null;
1052
- if (!rawSeedTokenIds || rawSeedTokenIds.length === 0) {
1053
- return [];
1054
- }
1055
-
1056
- const specialTokenIds = new Set();
1057
- const tokenizerSpecialTokens = context?.tokenizer?.getSpecialTokens?.() ?? null;
1058
- if (tokenizerSpecialTokens && typeof tokenizerSpecialTokens === 'object') {
1059
- for (const value of Object.values(tokenizerSpecialTokens)) {
1060
- if (Number.isInteger(value)) {
1061
- specialTokenIds.add(value);
1062
- }
1063
- }
1064
- }
1065
-
1066
- const seeded = [];
1067
- const seen = new Set();
1068
- for (let i = rawSeedTokenIds.length - 1; i >= 0; i -= 1) {
1069
- const tokenId = rawSeedTokenIds[i];
1070
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput) {
1071
- continue;
1072
- }
1073
- if (specialTokenIds.has(tokenId) || seen.has(tokenId)) {
1074
- continue;
1075
- }
1076
- seeded.push(tokenId);
1077
- seen.add(tokenId);
1078
- if (seeded.length >= maxTokens) {
1079
- break;
1080
- }
1081
- }
1082
- return seeded;
1083
- }
1084
-
1085
- function mergePleHotVocabularyTokenIds(seedTokenIds, tokenizerHotTokenIds, maxTokens, vocabSizePerLayerInput) {
1086
- const merged = [];
1087
- const seen = new Set();
1088
-
1089
- for (const tokenId of seedTokenIds ?? []) {
1090
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
1091
- continue;
1092
- }
1093
- merged.push(tokenId);
1094
- seen.add(tokenId);
1095
- if (merged.length >= maxTokens) {
1096
- return merged;
1097
- }
1098
- }
1099
-
1100
- for (const tokenId of tokenizerHotTokenIds ?? []) {
1101
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
1102
- continue;
1103
- }
1104
- merged.push(tokenId);
1105
- seen.add(tokenId);
1106
- if (merged.length >= maxTokens) {
1107
- break;
1108
- }
1109
- }
1110
-
1111
- return merged;
1112
- }
1113
-
1114
- export async function ensurePleGpuHotVocabularyRuntime(context) {
1115
- const policy = getPleHotCachePolicy(context?.perLayerInputsSession ?? null);
1116
- if (!policy || policy.mode !== 'tokenizer_scores') {
1117
- return null;
1118
- }
1119
-
1120
- const config = context?.config ?? null;
1121
- const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
1122
- const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
1123
- const numLayers = Number(config?.numLayers ?? 0);
1124
- if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
1125
- return null;
1126
- }
1127
- if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
1128
- return null;
1129
- }
1130
- if (!Number.isFinite(numLayers) || numLayers <= 0) {
1131
- return null;
1132
- }
1133
-
1134
- const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
1135
- const capacity = resolvePleHotVocabularyCapacity({
1136
- maxTokens: policy.maxTokens,
1137
- maxBytes: policy.maxBytes,
1138
- numLayers,
1139
- hiddenSize: hiddenSizePerLayerInput,
1140
- bytesPerElement,
1141
- vocabSize: vocabSizePerLayerInput,
1142
- });
1143
- if (capacity.maxHotTokens <= 0) {
1144
- return null;
1145
- }
1146
-
1147
- const tokenizer = context?.tokenizer ?? null;
1148
- const tokenizerHotTokenIds = typeof tokenizer?.getHotTokenIds === 'function'
1149
- ? tokenizer.getHotTokenIds(capacity.maxHotTokens)
1150
- : null;
1151
- const seedTokenIds = resolvePleHotVocabularySeedTokenIds(
1152
- context,
1153
- capacity.maxHotTokens,
1154
- vocabSizePerLayerInput
1155
- );
1156
- const hotTokenIds = mergePleHotVocabularyTokenIds(
1157
- seedTokenIds,
1158
- tokenizerHotTokenIds,
1159
- capacity.maxHotTokens,
1160
- vocabSizePerLayerInput
1161
- );
1162
- if (hotTokenIds.length === 0) {
1163
- return null;
1164
- }
1165
- const hotTokenIdsSignature = hotTokenIds.join(',');
1166
-
1167
- const perLayerInputWeights = getPerLayerInputWeights(context);
1168
- const cached = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
1169
- if (
1170
- cached
1171
- && cached.maxTokens === capacity.maxHotTokens
1172
- && cached.maxBytes === policy.maxBytes
1173
- && cached.outputDtype === policy.outputDtype
1174
- && cached.vocabSize === vocabSizePerLayerInput
1175
- && cached.numLayers === numLayers
1176
- && cached.hotTokenIdsSignature === hotTokenIdsSignature
1177
- ) {
1178
- return cached;
1179
- }
1180
- destroyPleHotVocabularyRuntime(perLayerInputWeights);
1181
-
1182
- const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
1183
- if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
1184
- return null;
1185
- }
1186
-
1187
- const sourceDtype = requirePleSourceDtype(
1188
- embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
1189
- 'Gemma 4 hot vocabulary cache'
1190
- );
1191
- const expandsF16ToF32 = sourceDtype === 'f16' && policy.outputDtype === 'f32';
1192
- if (sourceDtype !== policy.outputDtype && !expandsF16ToF32) {
1193
- throw new Error(
1194
- `Gemma 4 hot vocabulary cache cannot convert source dtype "${sourceDtype}" ` +
1195
- `to output dtype "${policy.outputDtype}".`
1196
- );
1197
- }
1198
-
1199
- const device = getDevice();
1200
- if (!device) {
1201
- throw new Error('No GPU device available for Gemma 4 hot vocabulary cache.');
1202
- }
1203
-
1204
- const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
1205
- const sentinelIndex = hotTokenIds.length;
1206
- const hotRowCount = sentinelIndex + 1;
1207
- const splitTableBytes = hotRowCount * hiddenSizePerLayerInput * bytesPerElement;
1208
- const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
1209
- acquireBuffer(splitTableBytes, undefined, `L${layerIdx}.ple_hot_vocab_table`),
1210
- policy.outputDtype,
1211
- 'row',
1212
- [hotRowCount, hiddenSizePerLayerInput],
1213
- `L${layerIdx}.embed_tokens_per_layer_hot_vocab`
1214
- ));
1215
- const hotTokenIndexMap = new Uint32Array(vocabSizePerLayerInput);
1216
- hotTokenIndexMap.fill(sentinelIndex);
1217
- for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
1218
- const tokenId = hotTokenIds[hotIndex];
1219
- if (Number.isInteger(tokenId) && tokenId >= 0 && tokenId < vocabSizePerLayerInput) {
1220
- hotTokenIndexMap[tokenId] = hotIndex;
1221
- }
1222
- }
1223
- const hotTokenIndexMapBuffer = acquireBuffer(
1224
- hotTokenIndexMap.byteLength,
1225
- undefined,
1226
- 'ple_hot_token_index_map'
1227
- );
1228
-
1229
- try {
1230
- device.queue.writeBuffer(hotTokenIndexMapBuffer, 0, hotTokenIndexMap);
1231
- const zeroRow = policy.outputDtype === 'f16'
1232
- ? new Uint16Array(hiddenSizePerLayerInput)
1233
- : new Float32Array(hiddenSizePerLayerInput);
1234
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1235
- device.queue.writeBuffer(
1236
- splitTables[layerIdx].buffer,
1237
- sentinelIndex * hiddenSizePerLayerInput * bytesPerElement,
1238
- zeroRow.buffer,
1239
- zeroRow.byteOffset,
1240
- zeroRow.byteLength
1241
- );
1242
- }
1243
-
1244
- const { sourceRowBytes } = getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize);
1245
- const expandedF32Row = expandsF16ToF32
1246
- ? new Float32Array(totalPerLayerHiddenSize)
1247
- : null;
1248
- const f16Lookup = expandsF16ToF32 ? getF16ToF32Lookup() : null;
1249
- for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
1250
- const tokenId = hotTokenIds[hotIndex];
1251
- const chunk = normalizeRangeBytes(
1252
- await embedTokensPerLayer.data.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
1253
- 'Gemma 4 hot vocabulary PLE row'
1254
- );
1255
- if (policy.outputDtype === 'f16') {
1256
- const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1257
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1258
- const sourceStart = layerIdx * hiddenSizePerLayerInput;
1259
- device.queue.writeBuffer(
1260
- splitTables[layerIdx].buffer,
1261
- hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1262
- sourceWords.buffer,
1263
- sourceWords.byteOffset + sourceStart * bytesPerElement,
1264
- hiddenSizePerLayerInput * bytesPerElement
1265
- );
1266
- }
1267
- } else if (sourceDtype === 'f16') {
1268
- const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1269
- for (let valueIdx = 0; valueIdx < totalPerLayerHiddenSize; valueIdx += 1) {
1270
- expandedF32Row[valueIdx] = f16Lookup[sourceWords[valueIdx]];
1271
- }
1272
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1273
- const sourceStart = layerIdx * hiddenSizePerLayerInput;
1274
- device.queue.writeBuffer(
1275
- splitTables[layerIdx].buffer,
1276
- hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1277
- expandedF32Row.buffer,
1278
- sourceStart * bytesPerElement,
1279
- hiddenSizePerLayerInput * bytesPerElement
1280
- );
1281
- }
1282
- } else {
1283
- const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1284
- for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1285
- const sourceStart = layerIdx * hiddenSizePerLayerInput;
1286
- device.queue.writeBuffer(
1287
- splitTables[layerIdx].buffer,
1288
- hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1289
- sourceValues.buffer,
1290
- sourceValues.byteOffset + sourceStart * bytesPerElement,
1291
- hiddenSizePerLayerInput * bytesPerElement
1292
- );
1293
- }
1294
- }
1295
- }
1296
- } catch (error) {
1297
- for (const table of splitTables) {
1298
- releaseBuffer(table.buffer);
1299
- }
1300
- releaseBuffer(hotTokenIndexMapBuffer);
1301
- throw error;
1302
- }
1303
-
1304
- const runtime = {
1305
- mode: 'tokenizer_scores',
1306
- maxTokens: capacity.maxHotTokens,
1307
- maxBytes: policy.maxBytes,
1308
- outputDtype: policy.outputDtype,
1309
- vocabSize: vocabSizePerLayerInput,
1310
- numLayers,
1311
- hotTokenIdsSignature,
1312
- hotTokenIds: Uint32Array.from(hotTokenIds),
1313
- hotTokenIndexMap,
1314
- hotTokenIndexMapBuffer,
1315
- sentinelIndex,
1316
- splitTables,
1317
- };
1318
- perLayerInputWeights.embedTokensPerLayerHotRuntime = runtime;
1319
- return runtime;
1320
- }
1321
-
1322
289
  export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context, options = {}) {
1323
290
  const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
1324
291
  const hiddenSizePerLayerInput = Number(config.hiddenSizePerLayerInput ?? 0);