doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -0,0 +1,1581 @@
1
+ import {
2
+ createWeightBuffer,
3
+ getBufferDtype,
4
+ getLayout,
5
+ getWeightDtype,
6
+ isCpuWeightBuffer,
7
+ isGpuBufferInstance,
8
+ isWeightBuffer,
9
+ } from '../../../gpu/weight-buffer.js';
10
+ import { createTensor } from '../../../gpu/tensor.js';
11
+ import { recordScale, runScale } from '../../../gpu/kernel-selector.js';
12
+ import { getNormWeightBuffer, getWeightBuffer } from './weights.js';
13
+ import { doMatmul, doRMSNorm, releaseOrTrack } from './ops.js';
14
+ import { runProbes } from './probes.js';
15
+ import { embed, isRangeBackedCpuEmbeddingSource, normalizeRangeBytes, decodeRangeChunkIntoOutput } from './embed.js';
16
+ import { selectRuleValue } from '../../../rules/rule-registry.js';
17
+ import { getDevice } from '../../../gpu/device.js';
18
+ import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
19
+
20
+ const pleRuntimeCache = new WeakMap();
21
+ const pleRangeRowCache = new WeakMap();
22
+
23
+ function getPerLayerInputWeights(context) {
24
+ const weights = context.weights.get('per_layer_inputs');
25
+ if (!weights || typeof weights !== 'object') {
26
+ throw new Error(
27
+ 'Gemma 4 per-layer inputs require global per-layer input weights, ' +
28
+ 'but state.weights.get("per_layer_inputs") was missing.'
29
+ );
30
+ }
31
+ return weights;
32
+ }
33
+
34
+ function normalizePleProjectionNormDtype(dtype) {
35
+ if (typeof dtype !== 'string') {
36
+ return null;
37
+ }
38
+ const value = dtype.toLowerCase();
39
+ if (value === 'f16' || value === 'f32') {
40
+ return value;
41
+ }
42
+ return null;
43
+ }
44
+
45
+ function getPleProjectionNormDtype(weight) {
46
+ return normalizePleProjectionNormDtype(getWeightDtype(weight))
47
+ ?? normalizePleProjectionNormDtype(getBufferDtype(weight))
48
+ ?? null;
49
+ }
50
+
51
+ function isRangeBackedPleProjectionSource(value) {
52
+ return isRangeBackedCpuEmbeddingSource(value);
53
+ }
54
+
55
+ function getPleProjectionWeightDtype(weight) {
56
+ if (isCpuWeightBuffer(weight)) {
57
+ return weight.dtype ?? null;
58
+ }
59
+ return getWeightDtype(weight);
60
+ }
61
+
62
+ function isDensePleProjectionDtype(dtype) {
63
+ return dtype === 'f16' || dtype === 'f32';
64
+ }
65
+
66
+ export function resolveDensePleProjectionWeight(
67
+ weight,
68
+ label = 'per_layer_model_projection'
69
+ ) {
70
+ if (isWeightBuffer(weight)) {
71
+ const dtype = String(weight.dtype ?? '').toLowerCase();
72
+ if (!isDensePleProjectionDtype(dtype)) {
73
+ throw new Error(
74
+ 'Gemma 4 sliced per-layer projection requires a dense f16/f32 base materialization. ' +
75
+ `Got dtype="${String(weight.dtype)}" for "${label}".`
76
+ );
77
+ }
78
+ if (!weight.materializations?.q4k?.buffer) {
79
+ return weight;
80
+ }
81
+ return createWeightBuffer(
82
+ weight.buffer,
83
+ dtype,
84
+ weight.layout,
85
+ [...weight.shape],
86
+ weight.label ?? label
87
+ );
88
+ }
89
+
90
+ if (isGpuBufferInstance(weight)) {
91
+ const dtype = String(getBufferDtype(weight) ?? '').toLowerCase();
92
+ if (!isDensePleProjectionDtype(dtype)) {
93
+ throw new Error(
94
+ 'Gemma 4 sliced per-layer projection requires dense f16/f32 GPU weights with runtime dtype metadata. ' +
95
+ `Got dtype="${String(getBufferDtype(weight) ?? 'unknown')}" for "${label}".`
96
+ );
97
+ }
98
+ }
99
+
100
+ return weight;
101
+ }
102
+
103
+ export async function loadRangeBackedPleProjectionSliceBytes(
104
+ weight,
105
+ layerIdx,
106
+ hiddenSizePerLayerInput,
107
+ hiddenSize,
108
+ label = 'Range-backed PLE projection slice'
109
+ ) {
110
+ if (!isCpuWeightBuffer(weight)) {
111
+ return null;
112
+ }
113
+ const cpuData = weight.data;
114
+ if (!isRangeBackedPleProjectionSource(cpuData)) {
115
+ return null;
116
+ }
117
+
118
+ const dtype = getPleProjectionWeightDtype(weight);
119
+ const layout = getLayout(weight) ?? weight.layout ?? 'row';
120
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
121
+ dtype,
122
+ });
123
+ const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
124
+ const byteOffset = hiddenOffset * hiddenSize * bytesPerElement;
125
+ const byteLength = hiddenSizePerLayerInput * hiddenSize * bytesPerElement;
126
+ const bytes = normalizeRangeBytes(
127
+ await cpuData.loadRange(byteOffset, byteLength),
128
+ label
129
+ );
130
+ if (bytes.byteLength !== byteLength) {
131
+ throw new Error(
132
+ `${label} short read for layer ${layerIdx}: expected ${byteLength} bytes, got ${bytes.byteLength}.`
133
+ );
134
+ }
135
+ return {
136
+ bytes,
137
+ dtype,
138
+ layout,
139
+ shape: [hiddenSizePerLayerInput, hiddenSize],
140
+ };
141
+ }
142
+
143
+ export function inferPleProjectionNormDtype(weight, hiddenSizePerLayerInput) {
144
+ const explicitDtype = getPleProjectionNormDtype(weight);
145
+ if (explicitDtype) {
146
+ return explicitDtype;
147
+ }
148
+
149
+ if (!isGpuBufferInstance(weight)) {
150
+ return 'f32';
151
+ }
152
+
153
+ if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
154
+ throw new Error('Gemma 4 per-layer projection norm dtype inference requires hiddenSizePerLayerInput > 0.');
155
+ }
156
+
157
+ const bytesPerElement = weight.size / hiddenSizePerLayerInput;
158
+ if (bytesPerElement !== 2 && bytesPerElement !== 4) {
159
+ throw new Error(
160
+ 'Gemma 4 per-layer projection norm buffer has unsupported byte size: ' +
161
+ `bufferSize=${weight.size}, hiddenSizePerLayerInput=${hiddenSizePerLayerInput}.`
162
+ );
163
+ }
164
+ return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
165
+ }
166
+
167
+ function createPleProjectionNormTensor(buffer, dtype, hiddenSizePerLayerInput, label) {
168
+ return createTensor(buffer, dtype, [hiddenSizePerLayerInput], label);
169
+ }
170
+
171
+ function destroyPleRuntimeCacheEntry(entry) {
172
+ const cachedBuffer = entry?.scaledProjectionNormWeight?.buffer ?? null;
173
+ if (cachedBuffer) {
174
+ releaseBuffer(cachedBuffer);
175
+ }
176
+ }
177
+
178
+ function destroyPleSplitTables(perLayerInputWeights) {
179
+ const splitTables = Array.isArray(perLayerInputWeights?.embedTokensPerLayerSplit)
180
+ ? perLayerInputWeights.embedTokensPerLayerSplit
181
+ : null;
182
+ if (!splitTables) {
183
+ return;
184
+ }
185
+ for (const table of splitTables) {
186
+ const buffer = table?.buffer ?? table ?? null;
187
+ if (buffer) {
188
+ releaseBuffer(buffer);
189
+ }
190
+ }
191
+ delete perLayerInputWeights.embedTokensPerLayerSplit;
192
+ }
193
+
194
+ function destroyPleHotVocabularyRuntime(perLayerInputWeights) {
195
+ const runtime = perLayerInputWeights?.embedTokensPerLayerHotRuntime ?? null;
196
+ if (!runtime || typeof runtime !== 'object') {
197
+ return;
198
+ }
199
+ const splitTables = Array.isArray(runtime.splitTables) ? runtime.splitTables : [];
200
+ for (const table of splitTables) {
201
+ const buffer = table?.buffer ?? table ?? null;
202
+ if (buffer) {
203
+ releaseBuffer(buffer);
204
+ }
205
+ }
206
+ const mapBuffer = runtime.hotTokenIndexMapBuffer ?? null;
207
+ if (mapBuffer) {
208
+ releaseBuffer(mapBuffer);
209
+ }
210
+ delete perLayerInputWeights.embedTokensPerLayerHotRuntime;
211
+ }
212
+
213
+ export function destroyPleRuntimeCache(perLayerInputWeights) {
214
+ if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
215
+ return;
216
+ }
217
+ const entry = pleRuntimeCache.get(perLayerInputWeights);
218
+ if (entry) {
219
+ destroyPleRuntimeCacheEntry(entry);
220
+ pleRuntimeCache.delete(perLayerInputWeights);
221
+ }
222
+ destroyPleSplitTables(perLayerInputWeights);
223
+ destroyPleHotVocabularyRuntime(perLayerInputWeights);
224
+ if (perLayerInputWeights.embedTokensPerLayer) {
225
+ pleRangeRowCache.delete(perLayerInputWeights.embedTokensPerLayer);
226
+ }
227
+ }
228
+
229
+ export function scalePerLayerProjectionNormWeights(weight, combineScale, rmsNormWeightOffset = false) {
230
+ if (rmsNormWeightOffset) {
231
+ return null;
232
+ }
233
+ const source = isCpuWeightBuffer(weight) ? weight.data : weight;
234
+ const isArrayLikeView = ArrayBuffer.isView(source) && typeof source.length === 'number';
235
+ if (!(source instanceof Float32Array) && !isArrayLikeView && !Array.isArray(source)) {
236
+ return null;
237
+ }
238
+ const scaled = Float32Array.from(source);
239
+ for (let i = 0; i < scaled.length; i++) {
240
+ scaled[i] *= combineScale;
241
+ }
242
+ return scaled;
243
+ }
244
+
245
+ export async function ensurePleScaledProjectionNormWeight(context, combineScale = 2 ** -0.5) {
246
+ const hiddenSizePerLayerInput = Number(context?.config?.hiddenSizePerLayerInput ?? 0);
247
+ if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
248
+ return null;
249
+ }
250
+ if (context?.weightConfig?.rmsNormWeightOffset) {
251
+ return null;
252
+ }
253
+
254
+ const perLayerInputWeights = getPerLayerInputWeights(context);
255
+ const cachedEntry = pleRuntimeCache.get(perLayerInputWeights) ?? null;
256
+ if (cachedEntry?.combineScale === combineScale && cachedEntry.scaledProjectionNormWeight) {
257
+ return cachedEntry.scaledProjectionNormWeight;
258
+ }
259
+ if (cachedEntry) {
260
+ destroyPleRuntimeCacheEntry(cachedEntry);
261
+ pleRuntimeCache.delete(perLayerInputWeights);
262
+ }
263
+
264
+ const projectionNormWeight = perLayerInputWeights.perLayerProjectionNorm;
265
+ if (!projectionNormWeight) {
266
+ return null;
267
+ }
268
+
269
+ let scaledProjectionNormWeight = null;
270
+ if (isGpuBufferInstance(projectionNormWeight)) {
271
+ const projectionNormDtype = inferPleProjectionNormDtype(
272
+ projectionNormWeight,
273
+ hiddenSizePerLayerInput
274
+ );
275
+ const scaledTensor = await runScale(
276
+ createPleProjectionNormTensor(
277
+ projectionNormWeight,
278
+ projectionNormDtype,
279
+ hiddenSizePerLayerInput,
280
+ 'per_layer_projection_norm'
281
+ ),
282
+ combineScale,
283
+ { count: hiddenSizePerLayerInput }
284
+ );
285
+ scaledProjectionNormWeight = createPleProjectionNormTensor(
286
+ scaledTensor.buffer,
287
+ projectionNormDtype,
288
+ hiddenSizePerLayerInput,
289
+ 'per_layer_projection_norm_scaled'
290
+ );
291
+ } else {
292
+ const scaledValues = scalePerLayerProjectionNormWeights(
293
+ projectionNormWeight,
294
+ combineScale,
295
+ false
296
+ );
297
+ if (!scaledValues) {
298
+ return null;
299
+ }
300
+ if (scaledValues.length !== hiddenSizePerLayerInput) {
301
+ throw new Error(
302
+ 'Gemma 4 per-layer projection norm cache shape mismatch: ' +
303
+ `expected ${hiddenSizePerLayerInput} values, got ${scaledValues.length}.`
304
+ );
305
+ }
306
+ const device = getDevice();
307
+ if (!device) {
308
+ throw new Error('No GPU device available for Gemma 4 per-layer projection norm cache.');
309
+ }
310
+ const scaledBuffer = acquireBuffer(
311
+ scaledValues.byteLength,
312
+ undefined,
313
+ 'per_layer_projection_norm_scaled'
314
+ );
315
+ try {
316
+ device.queue.writeBuffer(scaledBuffer, 0, scaledValues);
317
+ } catch (error) {
318
+ releaseBuffer(scaledBuffer);
319
+ throw error;
320
+ }
321
+ scaledProjectionNormWeight = createPleProjectionNormTensor(
322
+ scaledBuffer,
323
+ getPleProjectionNormDtype(projectionNormWeight) ?? 'f32',
324
+ hiddenSizePerLayerInput,
325
+ 'per_layer_projection_norm_scaled'
326
+ );
327
+ }
328
+
329
+ if (scaledProjectionNormWeight?.shape?.[0] !== hiddenSizePerLayerInput) {
330
+ throw new Error(
331
+ 'Gemma 4 per-layer projection norm cache shape mismatch after tensor creation: ' +
332
+ `expected ${hiddenSizePerLayerInput}, got ${scaledProjectionNormWeight?.shape?.[0] ?? 'unknown'}.`
333
+ );
334
+ }
335
+ if (!normalizePleProjectionNormDtype(scaledProjectionNormWeight?.dtype)) {
336
+ throw new Error(
337
+ 'Gemma 4 per-layer projection norm cache produced an invalid dtype: ' +
338
+ `"${String(scaledProjectionNormWeight?.dtype ?? 'undefined')}".`
339
+ );
340
+ }
341
+
342
+ pleRuntimeCache.set(perLayerInputWeights, {
343
+ combineScale,
344
+ scaledProjectionNormWeight,
345
+ });
346
+ return scaledProjectionNormWeight;
347
+ }
348
+
349
+ function getEmbeddingSource(weight, label) {
350
+ if (isWeightBuffer(weight)) {
351
+ return weight.buffer;
352
+ }
353
+ if (isCpuWeightBuffer(weight) || isGpuBufferInstance(weight) || weight instanceof Float32Array) {
354
+ return weight;
355
+ }
356
+ throw new Error(`Gemma 4 per-layer input ${label} has unsupported type "${weight?.constructor?.name ?? typeof weight}".`);
357
+ }
358
+
359
+ function getEmbeddingDtype(weight) {
360
+ if (isCpuWeightBuffer(weight)) {
361
+ return weight.dtype;
362
+ }
363
+ return getWeightDtype(weight);
364
+ }
365
+
366
+ function getEmbeddingTranspose(weight) {
367
+ if (isWeightBuffer(weight) || isCpuWeightBuffer(weight)) {
368
+ return weight.layout === 'column';
369
+ }
370
+ return false;
371
+ }
372
+
373
+ // Step 4: Pre-allocated buffer cache for decode-path fused projection slices.
374
+ // Avoids per-step acquireBuffer/releaseBuffer churn for the 35 slice buffers.
375
+ export function createPleBufferCache(numLayers, sliceBytes) {
376
+ const sliceBuffers = Array.from({ length: numLayers }, (_, l) =>
377
+ acquireBuffer(sliceBytes, undefined, `L${l}.ple_slice_cached`));
378
+ const gatherSliceBuffers = Array.from({ length: numLayers }, (_, l) =>
379
+ acquireBuffer(sliceBytes, undefined, `L${l}.ple_gather_slice_cached`));
380
+ const ownedBuffers = new Set();
381
+ for (const buffer of sliceBuffers) {
382
+ if (buffer) ownedBuffers.add(buffer);
383
+ }
384
+ for (const buffer of gatherSliceBuffers) {
385
+ if (buffer) ownedBuffers.add(buffer);
386
+ }
387
+ return {
388
+ sliceBuffers,
389
+ gatherSliceBuffers,
390
+ preparedTokenEntries: new Map(),
391
+ preparedTokenBytes: 0,
392
+ ownedBuffers,
393
+ };
394
+ }
395
+
396
+ export function destroyPleBufferCache(cache) {
397
+ if (!cache?.sliceBuffers && !cache?.gatherSliceBuffers) return;
398
+ for (const buf of cache?.sliceBuffers ?? []) {
399
+ if (buf) releaseBuffer(buf);
400
+ }
401
+ for (const buf of cache?.gatherSliceBuffers ?? []) {
402
+ if (buf) releaseBuffer(buf);
403
+ }
404
+ for (const entry of cache?.preparedTokenEntries?.values?.() ?? []) {
405
+ for (const buf of entry?.buffers ?? []) {
406
+ if (buf) releaseBuffer(buf);
407
+ }
408
+ }
409
+ cache.sliceBuffers = null;
410
+ cache.gatherSliceBuffers = null;
411
+ cache.preparedTokenEntries = null;
412
+ cache.preparedTokenBytes = 0;
413
+ cache.ownedBuffers = null;
414
+ }
415
+
416
+ function isCachedPleSliceBuffer(cache, buffer) {
417
+ return cache?.ownedBuffers instanceof Set && cache.ownedBuffers.has(buffer);
418
+ }
419
+
420
+ function releasePleSliceBuffer(recorder, buffer, decodeBuffers, cache) {
421
+ if (!buffer || isCachedPleSliceBuffer(cache, buffer)) {
422
+ return;
423
+ }
424
+ releaseOrTrack(recorder, buffer, decodeBuffers);
425
+ }
426
+
427
+ function getPleRowCachePolicy(sessionConfig) {
428
+ const rowCache = sessionConfig?.rowCache ?? null;
429
+ if (!rowCache || rowCache.mode === 'off') {
430
+ return null;
431
+ }
432
+ if (rowCache.mode !== 'lru') {
433
+ throw new Error(
434
+ `Gemma 4 per-layer input row cache mode "${String(rowCache.mode)}" is not implemented.`
435
+ );
436
+ }
437
+ const decodedDtype = String(rowCache.decodedDtype ?? '').toLowerCase();
438
+ if (decodedDtype !== 'f32') {
439
+ throw new Error(
440
+ `Gemma 4 range-backed per-layer input row cache requires rowCache.decodedDtype="f32"; ` +
441
+ `got "${String(rowCache.decodedDtype)}".`
442
+ );
443
+ }
444
+ const maxRows = Math.trunc(Number(rowCache.maxRows));
445
+ const maxBytes = Math.trunc(Number(rowCache.maxBytes));
446
+ if (!Number.isFinite(maxRows) || maxRows <= 0) {
447
+ throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxRows > 0.');
448
+ }
449
+ if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
450
+ throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxBytes > 0.');
451
+ }
452
+ return { maxRows, maxBytes };
453
+ }
454
+
455
+ function getPleHotCachePolicy(sessionConfig) {
456
+ const hotCache = sessionConfig?.hotCache ?? null;
457
+ if (!hotCache || hotCache.mode === 'off') {
458
+ return null;
459
+ }
460
+ if (hotCache.mode === 'tokenizer_scores') {
461
+ const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
462
+ if (outputDtype !== 'f16' && outputDtype !== 'f32') {
463
+ throw new Error(
464
+ `Gemma 4 per-layer input hot vocabulary cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
465
+ `got "${String(hotCache.outputDtype)}".`
466
+ );
467
+ }
468
+ const maxTokens = Math.trunc(Number(hotCache.maxTokens));
469
+ const maxBytes = Math.trunc(Number(hotCache.maxBytes));
470
+ if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
471
+ throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxTokens > 0.');
472
+ }
473
+ if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
474
+ throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxBytes > 0.');
475
+ }
476
+ return {
477
+ mode: 'tokenizer_scores',
478
+ maxTokens,
479
+ maxBytes,
480
+ outputDtype,
481
+ };
482
+ }
483
+ if (hotCache.mode !== 'prepared_tokens') {
484
+ throw new Error(
485
+ `Gemma 4 per-layer input hot cache mode "${String(hotCache.mode)}" is not implemented.`
486
+ );
487
+ }
488
+ const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
489
+ if (outputDtype !== 'f16' && outputDtype !== 'f32') {
490
+ throw new Error(
491
+ `Gemma 4 per-layer input hot cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
492
+ `got "${String(hotCache.outputDtype)}".`
493
+ );
494
+ }
495
+ const maxTokens = Math.trunc(Number(hotCache.maxTokens));
496
+ const maxBytes = Math.trunc(Number(hotCache.maxBytes));
497
+ if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
498
+ throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxTokens > 0.');
499
+ }
500
+ if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
501
+ throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxBytes > 0.');
502
+ }
503
+ return { mode: 'prepared_tokens', maxTokens, maxBytes, outputDtype };
504
+ }
505
+
506
+ export function getPleHotVocabularyRuntime(context) {
507
+ const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
508
+ if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
509
+ return null;
510
+ }
511
+ const runtime = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
512
+ return runtime && typeof runtime === 'object' ? runtime : null;
513
+ }
514
+
515
+ function getPleSplitTablePolicy(sessionConfig) {
516
+ if (sessionConfig?.materialization !== 'gpu_split_tables') {
517
+ return null;
518
+ }
519
+ return { mode: 'gpu_split_tables' };
520
+ }
521
+
522
+ function releasePreparedTokenEntry(cache, tokenId, entry) {
523
+ if (!cache || !entry) {
524
+ return;
525
+ }
526
+ cache.preparedTokenEntries?.delete(tokenId);
527
+ cache.preparedTokenBytes -= entry?.bytes ?? 0;
528
+ for (const buffer of entry.buffers ?? []) {
529
+ if (!buffer) continue;
530
+ cache.ownedBuffers?.delete(buffer);
531
+ releaseBuffer(buffer);
532
+ }
533
+ }
534
+
535
+ function prunePreparedTokenCache(cache, policy) {
536
+ while (
537
+ cache.preparedTokenEntries.size > policy.maxTokens
538
+ || cache.preparedTokenBytes > policy.maxBytes
539
+ ) {
540
+ const oldest = cache.preparedTokenEntries.keys().next();
541
+ if (oldest.done) {
542
+ break;
543
+ }
544
+ const tokenId = oldest.value;
545
+ const entry = cache.preparedTokenEntries.get(tokenId);
546
+ releasePreparedTokenEntry(cache, tokenId, entry);
547
+ }
548
+ }
549
+
550
+ function getPreparedTokenEntry(cache, tokenId, sessionConfig, activationDtype, stats = null) {
551
+ const policy = getPleHotCachePolicy(sessionConfig);
552
+ if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
553
+ return null;
554
+ }
555
+ if (policy.outputDtype !== activationDtype) {
556
+ throw new Error(
557
+ `Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
558
+ `got "${String(activationDtype)}".`
559
+ );
560
+ }
561
+ const entry = cache.preparedTokenEntries.get(tokenId) ?? null;
562
+ if (!entry) {
563
+ if (stats) {
564
+ stats.plePreparedTokenCacheMisses = (stats.plePreparedTokenCacheMisses ?? 0) + 1;
565
+ stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
566
+ stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
567
+ }
568
+ return null;
569
+ }
570
+ cache.preparedTokenEntries.delete(tokenId);
571
+ cache.preparedTokenEntries.set(tokenId, entry);
572
+ if (stats) {
573
+ stats.plePreparedTokenCacheHits = (stats.plePreparedTokenCacheHits ?? 0) + 1;
574
+ stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
575
+ stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
576
+ }
577
+ return entry.buffers.slice();
578
+ }
579
+
580
+ function storePreparedTokenEntry(cache, tokenId, buffers, sessionConfig, activationDtype, stats = null) {
581
+ const policy = getPleHotCachePolicy(sessionConfig);
582
+ if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
583
+ return buffers;
584
+ }
585
+ if (policy.outputDtype !== activationDtype) {
586
+ throw new Error(
587
+ `Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
588
+ `got "${String(activationDtype)}".`
589
+ );
590
+ }
591
+ const existing = cache.preparedTokenEntries.get(tokenId) ?? null;
592
+ if (existing) {
593
+ releasePreparedTokenEntry(cache, tokenId, existing);
594
+ }
595
+ const cachedBuffers = buffers.slice();
596
+ const bytes = cachedBuffers.reduce((total, buffer) => total + (buffer?.size ?? 0), 0);
597
+ for (const buffer of cachedBuffers) {
598
+ if (buffer) {
599
+ cache.ownedBuffers?.add(buffer);
600
+ }
601
+ }
602
+ cache.preparedTokenEntries.set(tokenId, { buffers: cachedBuffers, bytes });
603
+ cache.preparedTokenBytes += bytes;
604
+ prunePreparedTokenCache(cache, policy);
605
+ if (stats) {
606
+ stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
607
+ stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
608
+ }
609
+ return cachedBuffers.slice();
610
+ }
611
+
612
+ function getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize) {
613
+ const sourceDtype = String(
614
+ (isCpuWeightBuffer(embedTokensPerLayer) ? embedTokensPerLayer.data?.sourceDtype : null)
615
+ ?? embedTokensPerLayer?.dtype
616
+ ?? 'f32'
617
+ ).toLowerCase();
618
+ const bytesPerElement = (sourceDtype === 'f16' || sourceDtype === 'bf16') ? 2 : 4;
619
+ return {
620
+ sourceDtype,
621
+ sourceRowBytes: totalPerLayerHiddenSize * bytesPerElement,
622
+ };
623
+ }
624
+
625
+ function getPleRangeRowCache(embedTokensPerLayer, sessionConfig) {
626
+ const policy = getPleRowCachePolicy(sessionConfig);
627
+ if (!policy) {
628
+ return null;
629
+ }
630
+ const cached = pleRangeRowCache.get(embedTokensPerLayer);
631
+ if (cached && cached.maxRows === policy.maxRows && cached.maxBytes === policy.maxBytes) {
632
+ return cached;
633
+ }
634
+ const next = {
635
+ maxRows: policy.maxRows,
636
+ maxBytes: policy.maxBytes,
637
+ totalBytes: 0,
638
+ rows: new Map(),
639
+ };
640
+ pleRangeRowCache.set(embedTokensPerLayer, next);
641
+ return next;
642
+ }
643
+
644
+ function touchPleCachedRow(cache, tokenId) {
645
+ const hit = cache?.rows?.get(tokenId) ?? null;
646
+ if (!hit) {
647
+ return null;
648
+ }
649
+ cache.rows.delete(tokenId);
650
+ cache.rows.set(tokenId, hit);
651
+ return hit.row;
652
+ }
653
+
654
+ function prunePleRangeRowCache(cache) {
655
+ while (cache.rows.size > cache.maxRows || cache.totalBytes > cache.maxBytes) {
656
+ const oldest = cache.rows.keys().next();
657
+ if (oldest.done) {
658
+ break;
659
+ }
660
+ const entry = cache.rows.get(oldest.value);
661
+ cache.rows.delete(oldest.value);
662
+ cache.totalBytes -= entry?.bytes ?? 0;
663
+ }
664
+ }
665
+
666
+ function cachePleRangeRow(cache, tokenId, row) {
667
+ if (!cache) {
668
+ return row;
669
+ }
670
+ const existing = cache.rows.get(tokenId);
671
+ if (existing) {
672
+ cache.totalBytes -= existing.bytes;
673
+ cache.rows.delete(tokenId);
674
+ }
675
+ cache.rows.set(tokenId, { row, bytes: row.byteLength });
676
+ cache.totalBytes += row.byteLength;
677
+ prunePleRangeRowCache(cache);
678
+ return row;
679
+ }
680
+
681
+ async function loadRangeBackedPleRow(
682
+ tokenId,
683
+ embedTokensPerLayer,
684
+ totalPerLayerHiddenSize,
685
+ sessionConfig,
686
+ label,
687
+ prefetchedRow = null
688
+ ) {
689
+ if (!isCpuWeightBuffer(embedTokensPerLayer)) {
690
+ return null;
691
+ }
692
+ const cpuData = embedTokensPerLayer.data;
693
+ if (!isRangeBackedCpuEmbeddingSource(cpuData)) {
694
+ return null;
695
+ }
696
+
697
+ const cache = getPleRangeRowCache(embedTokensPerLayer, sessionConfig);
698
+ if (prefetchedRow && prefetchedRow.tokenId === tokenId) {
699
+ return cachePleRangeRow(cache, tokenId, prefetchedRow.row);
700
+ }
701
+
702
+ const cached = touchPleCachedRow(cache, tokenId);
703
+ if (cached) {
704
+ return cached;
705
+ }
706
+
707
+ const { sourceDtype, sourceRowBytes } = getPleRangeRowLoadConfig(
708
+ embedTokensPerLayer,
709
+ totalPerLayerHiddenSize
710
+ );
711
+ const chunk = normalizeRangeBytes(
712
+ await cpuData.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
713
+ label
714
+ );
715
+ const row = new Float32Array(totalPerLayerHiddenSize);
716
+ decodeRangeChunkIntoOutput(chunk, sourceDtype, row, 0, totalPerLayerHiddenSize);
717
+ return cachePleRangeRow(cache, tokenId, row);
718
+ }
719
+
720
+ // Step 5: Prefetch next token's PLE row during current decode step.
721
+ // Returns a promise resolving to { tokenId, row: Float32Array } or null.
722
+ // Call after sampling produces the next token; pass result as options.prefetchedRow
723
+ // to the next preparePerLayerInputs call.
724
+ export function prefetchPerLayerRow(tokenId, embedTokensPerLayer, totalPerLayerHiddenSize, sessionConfig = null) {
725
+ if (!isCpuWeightBuffer(embedTokensPerLayer)) return null;
726
+ const cpuData = embedTokensPerLayer.data;
727
+ if (!isRangeBackedCpuEmbeddingSource(cpuData)) return null;
728
+ return loadRangeBackedPleRow(
729
+ tokenId,
730
+ embedTokensPerLayer,
731
+ totalPerLayerHiddenSize,
732
+ sessionConfig,
733
+ 'Prefetched PLE row'
734
+ )
735
+ .then(row => (row ? { tokenId, row } : null))
736
+ .catch(() => null);
737
+ }
738
+
739
+ export function hasRangeBackedPerLayerInputEmbeddings(context) {
740
+ const hiddenSizePerLayerInput = Number(context?.config?.hiddenSizePerLayerInput ?? 0);
741
+ if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
742
+ return false;
743
+ }
744
+
745
+ const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
746
+ if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
747
+ return false;
748
+ }
749
+ if (Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit) && perLayerInputWeights.embedTokensPerLayerSplit.length > 0) {
750
+ return false;
751
+ }
752
+
753
+ const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
754
+ return isCpuWeightBuffer(embedTokensPerLayer)
755
+ && isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data);
756
+ }
757
+
758
+ export function hasGpuSplitPerLayerInputEmbeddings(context) {
759
+ const hiddenSizePerLayerInput = Number(context?.config?.hiddenSizePerLayerInput ?? 0);
760
+ if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
761
+ return false;
762
+ }
763
+
764
+ const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
765
+ if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
766
+ return false;
767
+ }
768
+
769
+ return Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit)
770
+ && perLayerInputWeights.embedTokensPerLayerSplit.length > 0;
771
+ }
772
+
773
+ export async function ensurePleGpuSplitTablesRuntime(context) {
774
+ const policy = getPleSplitTablePolicy(context?.perLayerInputsSession ?? null);
775
+ if (!policy) {
776
+ return null;
777
+ }
778
+
779
+ const config = context?.config ?? null;
780
+ const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
781
+ const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
782
+ const numLayers = Number(config?.numLayers ?? 0);
783
+ if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
784
+ return null;
785
+ }
786
+ if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
787
+ return null;
788
+ }
789
+ if (!Number.isFinite(numLayers) || numLayers <= 0) {
790
+ return null;
791
+ }
792
+
793
+ const perLayerInputWeights = getPerLayerInputWeights(context);
794
+ if (Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit) && perLayerInputWeights.embedTokensPerLayerSplit.length === numLayers) {
795
+ return perLayerInputWeights.embedTokensPerLayerSplit;
796
+ }
797
+
798
+ const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
799
+ if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
800
+ throw new Error('Gemma 4 gpu_split_tables materialization requires a range-backed CPU embedTokensPerLayer source.');
801
+ }
802
+
803
+ const sourceDtype = String(embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype ?? 'f32').toLowerCase();
804
+ if (sourceDtype !== 'f16' && sourceDtype !== 'f32') {
805
+ throw new Error(
806
+ `Gemma 4 gpu_split_tables materialization requires f16/f32 source rows; got "${sourceDtype}".`
807
+ );
808
+ }
809
+
810
+ const device = getDevice();
811
+ if (!device) {
812
+ throw new Error('No GPU device available for Gemma 4 gpu_split_tables materialization.');
813
+ }
814
+
815
+ const bytesPerElement = sourceDtype === 'f16' ? 2 : 4;
816
+ const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
817
+ const tableBytes = vocabSizePerLayerInput * hiddenSizePerLayerInput * bytesPerElement;
818
+ const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
819
+ acquireBuffer(tableBytes, undefined, `L${layerIdx}.ple_table_split`),
820
+ sourceDtype,
821
+ 'row',
822
+ [vocabSizePerLayerInput, hiddenSizePerLayerInput],
823
+ `L${layerIdx}.embed_tokens_per_layer_split`
824
+ ));
825
+
826
+ try {
827
+ const rowsPerChunk = 128;
828
+ for (let rowStart = 0; rowStart < vocabSizePerLayerInput; rowStart += rowsPerChunk) {
829
+ const rowCount = Math.min(rowsPerChunk, vocabSizePerLayerInput - rowStart);
830
+ const chunkByteOffset = rowStart * totalPerLayerHiddenSize * bytesPerElement;
831
+ const chunkByteLength = rowCount * totalPerLayerHiddenSize * bytesPerElement;
832
+ const chunk = normalizeRangeBytes(
833
+ await embedTokensPerLayer.data.loadRange(chunkByteOffset, chunkByteLength),
834
+ 'Gemma 4 split GPU PLE chunk'
835
+ );
836
+
837
+ if (sourceDtype === 'f16') {
838
+ const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
839
+ for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
840
+ const layerWords = new Uint16Array(rowCount * hiddenSizePerLayerInput);
841
+ for (let row = 0; row < rowCount; row += 1) {
842
+ const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
843
+ layerWords.set(
844
+ sourceWords.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
845
+ row * hiddenSizePerLayerInput
846
+ );
847
+ }
848
+ device.queue.writeBuffer(
849
+ splitTables[layerIdx].buffer,
850
+ rowStart * hiddenSizePerLayerInput * bytesPerElement,
851
+ layerWords.buffer,
852
+ layerWords.byteOffset,
853
+ layerWords.byteLength
854
+ );
855
+ }
856
+ } else {
857
+ const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
858
+ for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
859
+ const layerValues = new Float32Array(rowCount * hiddenSizePerLayerInput);
860
+ for (let row = 0; row < rowCount; row += 1) {
861
+ const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
862
+ layerValues.set(
863
+ sourceValues.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
864
+ row * hiddenSizePerLayerInput
865
+ );
866
+ }
867
+ device.queue.writeBuffer(
868
+ splitTables[layerIdx].buffer,
869
+ rowStart * hiddenSizePerLayerInput * bytesPerElement,
870
+ layerValues.buffer,
871
+ layerValues.byteOffset,
872
+ layerValues.byteLength
873
+ );
874
+ }
875
+ }
876
+ }
877
+ } catch (error) {
878
+ for (const table of splitTables) {
879
+ releaseBuffer(table.buffer);
880
+ }
881
+ throw error;
882
+ }
883
+
884
+ perLayerInputWeights.embedTokensPerLayerSplit = splitTables;
885
+ return splitTables;
886
+ }
887
+
888
+ function resolvePleHotVocabularySeedTokenIds(context, maxTokens, vocabSizePerLayerInput) {
889
+ const rawSeedTokenIds = Array.isArray(context?.seedTokenIds) ? context.seedTokenIds : null;
890
+ if (!rawSeedTokenIds || rawSeedTokenIds.length === 0) {
891
+ return [];
892
+ }
893
+
894
+ const specialTokenIds = new Set();
895
+ const tokenizerSpecialTokens = context?.tokenizer?.getSpecialTokens?.() ?? null;
896
+ if (tokenizerSpecialTokens && typeof tokenizerSpecialTokens === 'object') {
897
+ for (const value of Object.values(tokenizerSpecialTokens)) {
898
+ if (Number.isInteger(value)) {
899
+ specialTokenIds.add(value);
900
+ }
901
+ }
902
+ }
903
+
904
+ const seeded = [];
905
+ const seen = new Set();
906
+ for (let i = rawSeedTokenIds.length - 1; i >= 0; i -= 1) {
907
+ const tokenId = rawSeedTokenIds[i];
908
+ if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput) {
909
+ continue;
910
+ }
911
+ if (specialTokenIds.has(tokenId) || seen.has(tokenId)) {
912
+ continue;
913
+ }
914
+ seeded.push(tokenId);
915
+ seen.add(tokenId);
916
+ if (seeded.length >= maxTokens) {
917
+ break;
918
+ }
919
+ }
920
+ return seeded;
921
+ }
922
+
923
+ function mergePleHotVocabularyTokenIds(seedTokenIds, tokenizerHotTokenIds, maxTokens, vocabSizePerLayerInput) {
924
+ const merged = [];
925
+ const seen = new Set();
926
+
927
+ for (const tokenId of seedTokenIds ?? []) {
928
+ if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
929
+ continue;
930
+ }
931
+ merged.push(tokenId);
932
+ seen.add(tokenId);
933
+ if (merged.length >= maxTokens) {
934
+ return merged;
935
+ }
936
+ }
937
+
938
+ for (const tokenId of tokenizerHotTokenIds ?? []) {
939
+ if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
940
+ continue;
941
+ }
942
+ merged.push(tokenId);
943
+ seen.add(tokenId);
944
+ if (merged.length >= maxTokens) {
945
+ break;
946
+ }
947
+ }
948
+
949
+ return merged;
950
+ }
951
+
952
+ export async function ensurePleGpuHotVocabularyRuntime(context) {
953
+ const policy = getPleHotCachePolicy(context?.perLayerInputsSession ?? null);
954
+ if (!policy || policy.mode !== 'tokenizer_scores') {
955
+ return null;
956
+ }
957
+
958
+ const config = context?.config ?? null;
959
+ const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
960
+ const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
961
+ const numLayers = Number(config?.numLayers ?? 0);
962
+ if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
963
+ return null;
964
+ }
965
+ if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
966
+ return null;
967
+ }
968
+ if (!Number.isFinite(numLayers) || numLayers <= 0) {
969
+ return null;
970
+ }
971
+
972
+ const tokenizer = context?.tokenizer ?? null;
973
+ const tokenizerHotTokenIds = typeof tokenizer?.getHotTokenIds === 'function'
974
+ ? tokenizer.getHotTokenIds(policy.maxTokens)
975
+ : null;
976
+ const seedTokenIds = resolvePleHotVocabularySeedTokenIds(
977
+ context,
978
+ policy.maxTokens,
979
+ vocabSizePerLayerInput
980
+ );
981
+ const hotTokenIds = mergePleHotVocabularyTokenIds(
982
+ seedTokenIds,
983
+ tokenizerHotTokenIds,
984
+ policy.maxTokens,
985
+ vocabSizePerLayerInput
986
+ );
987
+ if (hotTokenIds.length === 0) {
988
+ return null;
989
+ }
990
+ const hotTokenIdsSignature = hotTokenIds.join(',');
991
+
992
+ const perLayerInputWeights = getPerLayerInputWeights(context);
993
+ const cached = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
994
+ if (
995
+ cached
996
+ && cached.maxTokens === policy.maxTokens
997
+ && cached.outputDtype === policy.outputDtype
998
+ && cached.vocabSize === vocabSizePerLayerInput
999
+ && cached.numLayers === numLayers
1000
+ && cached.hotTokenIdsSignature === hotTokenIdsSignature
1001
+ ) {
1002
+ return cached;
1003
+ }
1004
+ destroyPleHotVocabularyRuntime(perLayerInputWeights);
1005
+
1006
+ const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
1007
+ if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
1008
+ return null;
1009
+ }
1010
+
1011
+ const sourceDtype = String(embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype ?? 'f32').toLowerCase();
1012
+ if (sourceDtype !== policy.outputDtype) {
1013
+ throw new Error(
1014
+ `Gemma 4 hot vocabulary cache requires source dtype "${policy.outputDtype}" for zero-copy row packing; ` +
1015
+ `got "${sourceDtype}".`
1016
+ );
1017
+ }
1018
+
1019
+ const device = getDevice();
1020
+ if (!device) {
1021
+ throw new Error('No GPU device available for Gemma 4 hot vocabulary cache.');
1022
+ }
1023
+
1024
+ const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
1025
+ const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
1026
+ const sentinelIndex = hotTokenIds.length;
1027
+ const hotRowCount = sentinelIndex + 1;
1028
+ const splitTableBytes = hotRowCount * hiddenSizePerLayerInput * bytesPerElement;
1029
+ const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
1030
+ acquireBuffer(splitTableBytes, undefined, `L${layerIdx}.ple_hot_vocab_table`),
1031
+ policy.outputDtype,
1032
+ 'row',
1033
+ [hotRowCount, hiddenSizePerLayerInput],
1034
+ `L${layerIdx}.embed_tokens_per_layer_hot_vocab`
1035
+ ));
1036
+ const hotTokenIndexMap = new Uint32Array(vocabSizePerLayerInput);
1037
+ hotTokenIndexMap.fill(sentinelIndex);
1038
+ for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
1039
+ const tokenId = hotTokenIds[hotIndex];
1040
+ if (Number.isInteger(tokenId) && tokenId >= 0 && tokenId < vocabSizePerLayerInput) {
1041
+ hotTokenIndexMap[tokenId] = hotIndex;
1042
+ }
1043
+ }
1044
+ const hotTokenIndexMapBuffer = acquireBuffer(
1045
+ hotTokenIndexMap.byteLength,
1046
+ undefined,
1047
+ 'ple_hot_token_index_map'
1048
+ );
1049
+
1050
+ try {
1051
+ device.queue.writeBuffer(hotTokenIndexMapBuffer, 0, hotTokenIndexMap);
1052
+ const zeroRow = policy.outputDtype === 'f16'
1053
+ ? new Uint16Array(hiddenSizePerLayerInput)
1054
+ : new Float32Array(hiddenSizePerLayerInput);
1055
+ for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1056
+ device.queue.writeBuffer(
1057
+ splitTables[layerIdx].buffer,
1058
+ sentinelIndex * hiddenSizePerLayerInput * bytesPerElement,
1059
+ zeroRow.buffer,
1060
+ zeroRow.byteOffset,
1061
+ zeroRow.byteLength
1062
+ );
1063
+ }
1064
+
1065
+ const { sourceRowBytes } = getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize);
1066
+ for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
1067
+ const tokenId = hotTokenIds[hotIndex];
1068
+ const chunk = normalizeRangeBytes(
1069
+ await embedTokensPerLayer.data.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
1070
+ 'Gemma 4 hot vocabulary PLE row'
1071
+ );
1072
+ if (policy.outputDtype === 'f16') {
1073
+ const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1074
+ for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1075
+ const sourceStart = layerIdx * hiddenSizePerLayerInput;
1076
+ device.queue.writeBuffer(
1077
+ splitTables[layerIdx].buffer,
1078
+ hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1079
+ sourceWords.buffer,
1080
+ sourceWords.byteOffset + sourceStart * bytesPerElement,
1081
+ hiddenSizePerLayerInput * bytesPerElement
1082
+ );
1083
+ }
1084
+ } else {
1085
+ const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1086
+ for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1087
+ const sourceStart = layerIdx * hiddenSizePerLayerInput;
1088
+ device.queue.writeBuffer(
1089
+ splitTables[layerIdx].buffer,
1090
+ hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1091
+ sourceValues.buffer,
1092
+ sourceValues.byteOffset + sourceStart * bytesPerElement,
1093
+ hiddenSizePerLayerInput * bytesPerElement
1094
+ );
1095
+ }
1096
+ }
1097
+ }
1098
+ } catch (error) {
1099
+ for (const table of splitTables) {
1100
+ releaseBuffer(table.buffer);
1101
+ }
1102
+ releaseBuffer(hotTokenIndexMapBuffer);
1103
+ throw error;
1104
+ }
1105
+
1106
+ const runtime = {
1107
+ mode: 'tokenizer_scores',
1108
+ maxTokens: policy.maxTokens,
1109
+ outputDtype: policy.outputDtype,
1110
+ vocabSize: vocabSizePerLayerInput,
1111
+ numLayers,
1112
+ hotTokenIdsSignature,
1113
+ hotTokenIds: Uint32Array.from(hotTokenIds),
1114
+ hotTokenIndexMap,
1115
+ hotTokenIndexMapBuffer,
1116
+ sentinelIndex,
1117
+ splitTables,
1118
+ };
1119
+ perLayerInputWeights.embedTokensPerLayerHotRuntime = runtime;
1120
+ return runtime;
1121
+ }
1122
+
1123
+ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context, options = {}) {
1124
+ const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
1125
+ const hiddenSizePerLayerInput = Number(config.hiddenSizePerLayerInput ?? 0);
1126
+ if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
1127
+ return null;
1128
+ }
1129
+
1130
+ const vocabSizePerLayerInput = Number(config.vocabSizePerLayerInput ?? 0);
1131
+ if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
1132
+ throw new Error(
1133
+ `Gemma 4 model "${config.modelId ?? 'unknown'}" requires architecture.vocabSizePerLayerInput ` +
1134
+ 'when hiddenSizePerLayerInput is enabled.'
1135
+ );
1136
+ }
1137
+
1138
+ const perLayerInputWeights = getPerLayerInputWeights(context);
1139
+ const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
1140
+ const embedTokensPerLayerSplit = Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit)
1141
+ ? perLayerInputWeights.embedTokensPerLayerSplit
1142
+ : null;
1143
+ const hotVocabularyRuntime = getPleHotVocabularyRuntime(context);
1144
+ const perLayerModelProjection = perLayerInputWeights.perLayerModelProjection;
1145
+ const perLayerProjectionNorm = perLayerInputWeights.perLayerProjectionNorm;
1146
+ if (!embedTokensPerLayer || !perLayerModelProjection || !perLayerProjectionNorm) {
1147
+ throw new Error(
1148
+ 'Gemma 4 per-layer inputs require embedTokensPerLayer, perLayerModelProjection, ' +
1149
+ 'and perLayerProjectionNorm weights.'
1150
+ );
1151
+ }
1152
+
1153
+ const numLayers = config.numLayers;
1154
+ const hasSplitEmbeddingTables = Array.isArray(embedTokensPerLayerSplit)
1155
+ && embedTokensPerLayerSplit.length === numLayers;
1156
+ const hasHotVocabularyTables = Array.isArray(hotVocabularyRuntime?.splitTables)
1157
+ && hotVocabularyRuntime.splitTables.length === numLayers;
1158
+ const numTokens = Number.isFinite(options.numTokens) ? options.numTokens : inputEmbedsTensor.shape?.[0];
1159
+ const indexOffset = Number.isFinite(options.indexOffset) ? options.indexOffset : 0;
1160
+ const perLayerIndexOffset = Number.isFinite(options.perLayerIndexOffset)
1161
+ ? options.perLayerIndexOffset
1162
+ : indexOffset;
1163
+ if (!Number.isFinite(numTokens) || numTokens <= 0) {
1164
+ throw new Error('Gemma 4 per-layer inputs require a positive numTokens value.');
1165
+ }
1166
+
1167
+ const activationDtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', {
1168
+ dtype: inputEmbedsTensor.dtype,
1169
+ });
1170
+ const perLayerTokenIdsOption = options.perLayerTokenIds ?? null;
1171
+ let hotLocalTokenIds = null;
1172
+ if (
1173
+ !perLayerTokenIdsOption
1174
+ && hasHotVocabularyTables
1175
+ && numTokens === 1
1176
+ && !isGpuBufferInstance(tokenIds)
1177
+ && Array.isArray(tokenIds)
1178
+ && Number.isInteger(tokenIds[0])
1179
+ ) {
1180
+ const hotIndex = hotVocabularyRuntime.hotTokenIndexMap?.[tokenIds[0]] ?? hotVocabularyRuntime.sentinelIndex;
1181
+ if (hotIndex !== hotVocabularyRuntime.sentinelIndex) {
1182
+ hotLocalTokenIds = new Uint32Array([hotIndex]);
1183
+ if (context.stats) {
1184
+ context.stats.pleHotVocabularyHits = (context.stats.pleHotVocabularyHits ?? 0) + 1;
1185
+ }
1186
+ } else if (context.stats) {
1187
+ context.stats.pleHotVocabularyMisses = (context.stats.pleHotVocabularyMisses ?? 0) + 1;
1188
+ }
1189
+ }
1190
+ const perLayerTokenIds = perLayerTokenIdsOption ?? hotLocalTokenIds;
1191
+ const useHotVocabularyTables = hasHotVocabularyTables && perLayerTokenIds != null;
1192
+
1193
+ const perLayerEmbeddingDtype = useHotVocabularyTables
1194
+ ? hotVocabularyRuntime.outputDtype
1195
+ : hasSplitEmbeddingTables
1196
+ ? getEmbeddingDtype(embedTokensPerLayerSplit[0])
1197
+ : getEmbeddingDtype(embedTokensPerLayer);
1198
+ const embedSource = hasSplitEmbeddingTables || useHotVocabularyTables
1199
+ ? null
1200
+ : getEmbeddingSource(embedTokensPerLayer, 'embedTokensPerLayer');
1201
+ const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
1202
+ const hasRangeBackedProjection = isCpuWeightBuffer(perLayerModelProjection)
1203
+ && isRangeBackedPleProjectionSource(perLayerModelProjection.data);
1204
+ const projectionWeight = hasRangeBackedProjection
1205
+ ? null
1206
+ : getWeightBuffer(perLayerModelProjection, 'per_layer_model_projection');
1207
+ if (isWeightBuffer(perLayerModelProjection) && perLayerModelProjection.layout !== 'row') {
1208
+ throw new Error(
1209
+ 'Gemma 4 per-layer input projection requires a row-major per_layer_model_projection weight. ' +
1210
+ `Got layout="${perLayerModelProjection.layout}".`
1211
+ );
1212
+ }
1213
+ const projectionScale = config.hiddenSize ** -0.5;
1214
+ const combineScale = 2 ** -0.5;
1215
+ const scaledProjectionNormWeight = await ensurePleScaledProjectionNormWeight(context, combineScale);
1216
+ const usesCachedScaledProjectionNormWeight = !!scaledProjectionNormWeight;
1217
+ const projectionNormWeight = scaledProjectionNormWeight ?? getNormWeightBuffer(
1218
+ perLayerProjectionNorm,
1219
+ 'per_layer_projection_norm',
1220
+ weightConfig,
1221
+ debugFlags
1222
+ );
1223
+ const perLayerBuffers = new Array(numLayers).fill(null);
1224
+ const pleCache = options.pleCache ?? null;
1225
+ const activationBytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
1226
+ dtype: activationDtype,
1227
+ });
1228
+ const projectionDevice = getDevice();
1229
+ if (projectionDevice == null) {
1230
+ throw new Error('Gemma 4 per-layer input projection requires an initialized GPU device.');
1231
+ }
1232
+
1233
+ // Decode-path optimizations: coalesced PLE read + fused projection matmul.
1234
+ // Gated on numTokens === 1 (decode) and row-major embeddings (non-transpose).
1235
+ // For numTokens > 1 (prefill), the fused matmul output is strided per-layer,
1236
+ // so we fall back to the per-layer path.
1237
+ const embedTranspose = (hasSplitEmbeddingTables || useHotVocabularyTables) ? false : getEmbeddingTranspose(embedTokensPerLayer);
1238
+ const canFuseDecodeOps = numTokens === 1 && !embedTranspose && !hasRangeBackedProjection;
1239
+ const tokenIdsAreGpuBuffer = isGpuBufferInstance(tokenIds);
1240
+ const decodeTokenId = canFuseDecodeOps && !tokenIdsAreGpuBuffer
1241
+ ? Number(tokenIds[0])
1242
+ : null;
1243
+
1244
+ if (decodeTokenId != null) {
1245
+ const preparedTokenHit = getPreparedTokenEntry(
1246
+ pleCache,
1247
+ decodeTokenId,
1248
+ context.perLayerInputsSession ?? null,
1249
+ activationDtype,
1250
+ context.stats ?? null
1251
+ );
1252
+ if (preparedTokenHit) {
1253
+ return preparedTokenHit;
1254
+ }
1255
+ }
1256
+
1257
+ // Step 5: Use prefetched PLE row if available and token matches.
1258
+ // Falls back to inline coalesced read (step 3) otherwise.
1259
+ let preloadedCpuRow = null;
1260
+ if (canFuseDecodeOps && !perLayerTokenIds) {
1261
+ const embedCpuData = !hasSplitEmbeddingTables && isCpuWeightBuffer(embedSource) ? embedSource.data : null;
1262
+ if (embedCpuData && isRangeBackedCpuEmbeddingSource(embedCpuData)) {
1263
+ if (tokenIdsAreGpuBuffer) {
1264
+ throw new Error(
1265
+ 'Gemma 4 per-layer input decode with range-backed CPU embeddings requires CPU token IDs. ' +
1266
+ 'Disable batch decode or use GPU-resident per-layer inputs.'
1267
+ );
1268
+ }
1269
+ preloadedCpuRow = await loadRangeBackedPleRow(
1270
+ tokenIds[0],
1271
+ embedTokensPerLayer,
1272
+ totalPerLayerHiddenSize,
1273
+ context.perLayerInputsSession ?? null,
1274
+ 'Coalesced PLE row',
1275
+ options.prefetchedRow ?? null
1276
+ );
1277
+ if (!preloadedCpuRow) {
1278
+ throw new Error('Gemma 4 range-backed per-layer input row load returned null unexpectedly.');
1279
+ }
1280
+ }
1281
+ }
1282
+
1283
+ // Step 6: Batched prefill gather. When numTokens > 1 and the PLE source is
1284
+ // range-backed + row-major, read all tokens' full PLE rows into a single CPU
1285
+ // buffer. This avoids numTokens × numLayers separate loadRange calls during
1286
+ // the per-layer embed loop.
1287
+ let prefillBatchedRows = null;
1288
+ if (!canFuseDecodeOps && numTokens > 1 && !perLayerTokenIds && !getEmbeddingTranspose(embedTokensPerLayer)) {
1289
+ const embedCpuData = !hasSplitEmbeddingTables && isCpuWeightBuffer(embedSource) ? embedSource.data : null;
1290
+ if (embedCpuData && isRangeBackedCpuEmbeddingSource(embedCpuData)) {
1291
+ const tokenIdArray = Array.isArray(tokenIds) ? tokenIds : Array.from(tokenIds);
1292
+ prefillBatchedRows = new Float32Array(numTokens * totalPerLayerHiddenSize);
1293
+ for (let t = 0; t < numTokens; t++) {
1294
+ const row = await loadRangeBackedPleRow(
1295
+ tokenIdArray[t],
1296
+ embedTokensPerLayer,
1297
+ totalPerLayerHiddenSize,
1298
+ context.perLayerInputsSession ?? null,
1299
+ 'Batched PLE prefill row'
1300
+ );
1301
+ if (!row) {
1302
+ throw new Error('Gemma 4 batched per-layer input row load returned null unexpectedly.');
1303
+ }
1304
+ prefillBatchedRows.set(row, t * totalPerLayerHiddenSize);
1305
+ }
1306
+ }
1307
+ }
1308
+
1309
+ // Fused projection matmul: one dispatch for all layers instead of numLayers dispatches.
1310
+ // Produces [numTokens × totalPerLayerHiddenSize], then scales the full output and
1311
+ // extracts per-layer slices via GPU buffer copies.
1312
+ let fusedProjectionSlices = null;
1313
+ if (canFuseDecodeOps) {
1314
+ const fusedProjection = await doMatmul(
1315
+ inputEmbedsTensor,
1316
+ projectionWeight,
1317
+ numTokens,
1318
+ totalPerLayerHiddenSize,
1319
+ config.hiddenSize,
1320
+ {
1321
+ transposeB: 'auto',
1322
+ label: 'per_layer_fused_projection',
1323
+ kernelPath: context.kernelPath ?? null,
1324
+ role: 'per_layer_model_projection',
1325
+ outputDtype: activationDtype,
1326
+ },
1327
+ recorder
1328
+ );
1329
+
1330
+ // One scale dispatch instead of numLayers
1331
+ const scaledFused = recorder
1332
+ ? await recordScale(recorder, fusedProjection, projectionScale, {
1333
+ count: numTokens * totalPerLayerHiddenSize,
1334
+ })
1335
+ : await runScale(fusedProjection, projectionScale, {
1336
+ count: numTokens * totalPerLayerHiddenSize,
1337
+ });
1338
+ releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
1339
+
1340
+ // Step 4: Extract per-layer slices via GPU buffer copies (one encoder, one submit).
1341
+ // Reuse cached slice buffers when available to avoid per-step pool churn.
1342
+ const device = getDevice();
1343
+ const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
1344
+ const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
1345
+ fusedProjectionSlices = new Array(numLayers);
1346
+ for (let l = 0; l < numLayers; l++) {
1347
+ const sliceBuf = pleCache?.sliceBuffers?.[l] ?? acquireBuffer(sliceBytes, undefined, `L${l}.per_layer_proj_slice`);
1348
+ encoder.copyBufferToBuffer(scaledFused.buffer, l * sliceBytes, sliceBuf, 0, sliceBytes);
1349
+ fusedProjectionSlices[l] = sliceBuf;
1350
+ }
1351
+ if (!recorder) {
1352
+ device.queue.submit([encoder.finish()]);
1353
+ }
1354
+ releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
1355
+ }
1356
+
1357
+ const embedDtypeResolved = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', {
1358
+ dtype: perLayerEmbeddingDtype,
1359
+ });
1360
+
1361
+ try {
1362
+ for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
1363
+ const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
1364
+ const layerEmbedSource = useHotVocabularyTables
1365
+ ? getEmbeddingSource(hotVocabularyRuntime.splitTables[layerIdx], `embedTokensPerLayerHot[L${layerIdx}]`)
1366
+ : hasSplitEmbeddingTables
1367
+ ? getEmbeddingSource(embedTokensPerLayerSplit[layerIdx], `embedTokensPerLayerSplit[L${layerIdx}]`)
1368
+ : embedSource;
1369
+ let gatheredTensor = null;
1370
+ let scaledProjectionTensor = null;
1371
+ let combinedTensor = null;
1372
+ try {
1373
+ gatheredTensor = await embed(perLayerTokenIds ?? tokenIds, layerEmbedSource, {
1374
+ hiddenSize: hiddenSizePerLayerInput,
1375
+ vocabSize: useHotVocabularyTables ? (hotVocabularyRuntime.sentinelIndex + 1) : vocabSizePerLayerInput,
1376
+ scaleEmbeddings: true,
1377
+ recorder,
1378
+ numTokens,
1379
+ indexOffset: perLayerTokenIds ? perLayerIndexOffset : indexOffset,
1380
+ transpose: embedTranspose,
1381
+ debugProbes: context.debugProbes,
1382
+ operatorDiagnostics: context.operatorDiagnostics,
1383
+ activationDtype,
1384
+ embeddingDtype: embedDtypeResolved,
1385
+ executionPolicies: context.executionPolicies ?? null,
1386
+ inputHiddenSize: (hasSplitEmbeddingTables || useHotVocabularyTables) ? hiddenSizePerLayerInput : totalPerLayerHiddenSize,
1387
+ hiddenOffset: (hasSplitEmbeddingTables || useHotVocabularyTables) ? 0 : hiddenOffset,
1388
+ preloadedCpuRow,
1389
+ preloadedCpuBatchedRows: prefillBatchedRows,
1390
+ outputBuffer: canFuseDecodeOps
1391
+ ? (pleCache?.gatherSliceBuffers?.[layerIdx] ?? undefined)
1392
+ : undefined,
1393
+ });
1394
+
1395
+ if (fusedProjectionSlices) {
1396
+ // Use pre-computed fused projection slice (already scaled).
1397
+ scaledProjectionTensor = createTensor(
1398
+ fusedProjectionSlices[layerIdx],
1399
+ activationDtype,
1400
+ [numTokens, hiddenSizePerLayerInput],
1401
+ `L${layerIdx}.per_layer_proj_scaled`
1402
+ );
1403
+ fusedProjectionSlices[layerIdx] = null;
1404
+ } else {
1405
+ let projectionWeightForLayer = projectionWeight;
1406
+ let projectionWeightBufferForLayer = null;
1407
+ let projectionWeightOffset = 0;
1408
+ let projectedTensor = null;
1409
+ try {
1410
+ const rangeBackedProjection = await loadRangeBackedPleProjectionSliceBytes(
1411
+ perLayerModelProjection,
1412
+ layerIdx,
1413
+ hiddenSizePerLayerInput,
1414
+ config.hiddenSize,
1415
+ `L${layerIdx}.per_layer_projection_in`
1416
+ );
1417
+ if (rangeBackedProjection) {
1418
+ projectionWeightBufferForLayer = acquireBuffer(
1419
+ rangeBackedProjection.bytes.byteLength,
1420
+ undefined,
1421
+ `L${layerIdx}.per_layer_projection_in_weight`
1422
+ );
1423
+ projectionDevice.queue.writeBuffer(
1424
+ projectionWeightBufferForLayer,
1425
+ 0,
1426
+ rangeBackedProjection.bytes,
1427
+ rangeBackedProjection.bytes.byteOffset,
1428
+ rangeBackedProjection.bytes.byteLength
1429
+ );
1430
+ projectionWeightForLayer = createWeightBuffer(
1431
+ projectionWeightBufferForLayer,
1432
+ rangeBackedProjection.dtype,
1433
+ rangeBackedProjection.layout,
1434
+ rangeBackedProjection.shape,
1435
+ `L${layerIdx}.per_layer_projection_in_weight`
1436
+ );
1437
+ projectionWeightOffset = 0;
1438
+ } else {
1439
+ projectionWeightForLayer = resolveDensePleProjectionWeight(
1440
+ projectionWeightForLayer,
1441
+ `L${layerIdx}.per_layer_projection_in_weight`
1442
+ );
1443
+ const projectionWeightDtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', {
1444
+ dtype: getPleProjectionWeightDtype(projectionWeightForLayer),
1445
+ });
1446
+ const projectionWeightBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
1447
+ dtype: projectionWeightDtype,
1448
+ });
1449
+ projectionWeightOffset = hiddenOffset * config.hiddenSize * projectionWeightBytes;
1450
+ }
1451
+ projectedTensor = await doMatmul(
1452
+ inputEmbedsTensor,
1453
+ projectionWeightForLayer,
1454
+ numTokens,
1455
+ hiddenSizePerLayerInput,
1456
+ config.hiddenSize,
1457
+ {
1458
+ transposeB: 'auto',
1459
+ bOffset: projectionWeightOffset,
1460
+ label: `L${layerIdx}.per_layer_projection_in`,
1461
+ layerIdx,
1462
+ kernelPath: context.kernelPath ?? null,
1463
+ role: 'per_layer_model_projection',
1464
+ outputDtype: activationDtype,
1465
+ },
1466
+ recorder
1467
+ );
1468
+ } finally {
1469
+ if (projectionWeightBufferForLayer) {
1470
+ releaseOrTrack(recorder, projectionWeightBufferForLayer, decodeBuffers);
1471
+ }
1472
+ }
1473
+ scaledProjectionTensor = recorder
1474
+ ? await recordScale(recorder, projectedTensor, projectionScale, {
1475
+ count: numTokens * hiddenSizePerLayerInput,
1476
+ })
1477
+ : await runScale(projectedTensor, projectionScale, {
1478
+ count: numTokens * hiddenSizePerLayerInput,
1479
+ });
1480
+ releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
1481
+ projectedTensor = null;
1482
+ }
1483
+
1484
+ // Fuse the residual add into RMSNorm so Gemma 4 PLE decode avoids an
1485
+ // extra dispatch per layer. When the model uses raw RMSNorm weights,
1486
+ // cache the fixed combine scale into the norm weights and skip the
1487
+ // legacy post-norm scale dispatch entirely.
1488
+ combinedTensor = await doRMSNorm(scaledProjectionTensor, projectionNormWeight, config.rmsNormEps, {
1489
+ batchSize: numTokens,
1490
+ hiddenSize: hiddenSizePerLayerInput,
1491
+ residual: gatheredTensor,
1492
+ label: `L${layerIdx}.per_layer_input_combine`,
1493
+ layerIdx,
1494
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
1495
+ }, recorder);
1496
+ releasePleSliceBuffer(recorder, scaledProjectionTensor.buffer, decodeBuffers, pleCache);
1497
+ scaledProjectionTensor = null;
1498
+ releasePleSliceBuffer(recorder, gatheredTensor.buffer, decodeBuffers, pleCache);
1499
+ gatheredTensor = null;
1500
+
1501
+ if (usesCachedScaledProjectionNormWeight) {
1502
+ perLayerBuffers[layerIdx] = combinedTensor.buffer;
1503
+ combinedTensor = null;
1504
+ } else {
1505
+ // Step 8: Inplace scale avoids allocating a separate output buffer.
1506
+ // The combined tensor buffer is reused as the final per-layer output.
1507
+ const scaledTensor = recorder
1508
+ ? await recordScale(recorder, combinedTensor, combineScale, {
1509
+ count: numTokens * hiddenSizePerLayerInput,
1510
+ inplace: true,
1511
+ })
1512
+ : await runScale(combinedTensor, combineScale, {
1513
+ count: numTokens * hiddenSizePerLayerInput,
1514
+ inplace: true,
1515
+ });
1516
+ perLayerBuffers[layerIdx] = scaledTensor.buffer;
1517
+ combinedTensor = null;
1518
+ }
1519
+ await runProbes('per_layer_input', perLayerBuffers[layerIdx], {
1520
+ layerIdx,
1521
+ numTokens,
1522
+ hiddenSize: hiddenSizePerLayerInput,
1523
+ probes: context.debugProbes,
1524
+ recorder,
1525
+ operatorDiagnostics: context.operatorDiagnostics,
1526
+ dtype: activationDtype,
1527
+ });
1528
+ } catch (error) {
1529
+ if (combinedTensor) {
1530
+ releaseOrTrack(recorder, combinedTensor.buffer, decodeBuffers);
1531
+ }
1532
+ if (gatheredTensor) {
1533
+ releasePleSliceBuffer(recorder, gatheredTensor.buffer, decodeBuffers, pleCache);
1534
+ }
1535
+ if (scaledProjectionTensor) {
1536
+ releasePleSliceBuffer(recorder, scaledProjectionTensor.buffer, decodeBuffers, pleCache);
1537
+ }
1538
+ throw error;
1539
+ }
1540
+ }
1541
+ } catch (error) {
1542
+ if (fusedProjectionSlices) {
1543
+ for (let i = 0; i < fusedProjectionSlices.length; i++) {
1544
+ const buf = fusedProjectionSlices[i];
1545
+ releasePleSliceBuffer(recorder, buf, decodeBuffers, pleCache);
1546
+ }
1547
+ }
1548
+ for (const buffer of perLayerBuffers) {
1549
+ if (buffer) {
1550
+ releaseOrTrack(recorder, buffer, decodeBuffers);
1551
+ }
1552
+ }
1553
+ throw error;
1554
+ } finally {
1555
+ if (!usesCachedScaledProjectionNormWeight && !isGpuBufferInstance(perLayerProjectionNorm)) {
1556
+ releaseOrTrack(recorder, projectionNormWeight, decodeBuffers);
1557
+ }
1558
+ }
1559
+
1560
+ if (decodeTokenId != null) {
1561
+ return storePreparedTokenEntry(
1562
+ pleCache,
1563
+ decodeTokenId,
1564
+ perLayerBuffers,
1565
+ context.perLayerInputsSession ?? null,
1566
+ activationDtype,
1567
+ context.stats ?? null
1568
+ );
1569
+ }
1570
+
1571
+ return perLayerBuffers;
1572
+ }
1573
+
1574
+ export function createPerLayerInputTensor(buffer, numTokens, hiddenSizePerLayerInput, activationDtype) {
1575
+ return createTensor(
1576
+ buffer,
1577
+ activationDtype,
1578
+ [numTokens, hiddenSizePerLayerInput],
1579
+ 'per_layer_input'
1580
+ );
1581
+ }