doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,6 +1,6 @@
1
1
 
2
2
  import { getDevice, initDevice, getKernelCapabilities } from '../../gpu/device.js';
3
- import { getBufferPool as getGlobalBufferPool } from '../../memory/buffer-pool.js';
3
+ import { getBufferPool as getGlobalBufferPool, releaseBuffer } from '../../memory/buffer-pool.js';
4
4
  import { log } from '../../debug/index.js';
5
5
  import { configurePerfGuards } from '../../gpu/perf-guards.js';
6
6
  import { MoERouter } from '../moe-router.js';
@@ -23,6 +23,7 @@ import {
23
23
  initEmulation,
24
24
  destroyEmulation,
25
25
  } from './text/init.js';
26
+ import { formatChatMessages } from './text/chat-format.js';
26
27
  import {
27
28
  runKernelWarmup,
28
29
  applyModelBatchingRuntimeDefaults,
@@ -46,6 +47,7 @@ import { getDopplerLoader } from '../../loader/doppler-loader.js';
46
47
  import { registerPipeline, getPipelineFactory } from './registry.js';
47
48
  import { selectRuleValue } from '../../rules/rule-registry.js';
48
49
  import { initConvLayerState } from './text/ops.js';
50
+ import { destroyPleBufferCache, destroyPleRuntimeCache } from './text/per-layer-inputs.js';
49
51
 
50
52
  function destroyMoERouter(router) {
51
53
  if (router && typeof router.destroy === 'function') {
@@ -53,6 +55,93 @@ function destroyMoERouter(router) {
53
55
  }
54
56
  }
55
57
 
58
+ function resolveSingleSpecialTokenId(tokenizer, tokenText, label) {
59
+ const rawTokenIds = tokenizer?.encode?.(tokenText);
60
+ const tokenIds = Array.isArray(rawTokenIds)
61
+ ? rawTokenIds
62
+ : (ArrayBuffer.isView(rawTokenIds) ? Array.from(rawTokenIds) : null);
63
+ if (!Array.isArray(tokenIds) || tokenIds.length !== 1) {
64
+ throw new Error(
65
+ `[Pipeline] transcribeImage: tokenizer must encode ${label} "${tokenText}" as exactly one token.`
66
+ );
67
+ }
68
+ const tokenId = Number(tokenIds[0]);
69
+ if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
70
+ throw new Error(
71
+ `[Pipeline] transcribeImage: tokenizer returned invalid ${label} token id "${tokenIds[0]}".`
72
+ );
73
+ }
74
+ return tokenId;
75
+ }
76
+
77
+ function expandImagePlaceholderTokenIds(tokenIds, imageTokenId, numImageTokens, options = {}) {
78
+ const normalizedTokenIds = Array.isArray(tokenIds)
79
+ ? Int32Array.from(tokenIds)
80
+ : (ArrayBuffer.isView(tokenIds) ? Int32Array.from(tokenIds) : null);
81
+ if (!(normalizedTokenIds instanceof Int32Array)) {
82
+ throw new Error(
83
+ '[Pipeline] transcribeImage: tokenizer.encode() must return an array or typed array of token IDs.'
84
+ );
85
+ }
86
+ if (!Number.isFinite(numImageTokens) || Math.floor(numImageTokens) !== numImageTokens || numImageTokens < 1) {
87
+ throw new Error(
88
+ `[Pipeline] transcribeImage: image token span must be a positive integer, got ${numImageTokens}.`
89
+ );
90
+ }
91
+
92
+ let placeholderIndex = -1;
93
+ let placeholderCount = 0;
94
+ for (let i = 0; i < normalizedTokenIds.length; i++) {
95
+ if (normalizedTokenIds[i] !== imageTokenId) continue;
96
+ if (placeholderIndex < 0) {
97
+ placeholderIndex = i;
98
+ }
99
+ placeholderCount++;
100
+ }
101
+
102
+ if (placeholderCount !== 1) {
103
+ throw new Error(
104
+ `[Pipeline] transcribeImage: expected exactly one image_token_id (${imageTokenId}) placeholder ` +
105
+ `from the chat template, got ${placeholderCount}.`
106
+ );
107
+ }
108
+
109
+ const boiTokenId = Number.isInteger(options.boiTokenId) ? options.boiTokenId : null;
110
+ const eoiTokenId = Number.isInteger(options.eoiTokenId) ? options.eoiTokenId : null;
111
+ const prefixExtra = boiTokenId == null ? 0 : 1;
112
+ const suffixExtra = eoiTokenId == null ? 0 : 1;
113
+ const expandedLength = normalizedTokenIds.length - 1 + prefixExtra + numImageTokens + suffixExtra;
114
+ const expanded = new Int32Array(expandedLength);
115
+ expanded.set(normalizedTokenIds.subarray(0, placeholderIndex), 0);
116
+ let writeOffset = placeholderIndex;
117
+ if (boiTokenId != null) {
118
+ expanded[writeOffset++] = boiTokenId;
119
+ }
120
+ expanded.fill(imageTokenId, writeOffset, writeOffset + numImageTokens);
121
+ const imageStartOffset = writeOffset;
122
+ writeOffset += numImageTokens;
123
+ if (eoiTokenId != null) {
124
+ expanded[writeOffset++] = eoiTokenId;
125
+ }
126
+ expanded.set(
127
+ normalizedTokenIds.subarray(placeholderIndex + 1),
128
+ writeOffset
129
+ );
130
+
131
+ return {
132
+ inputIds: expanded,
133
+ imageStartOffset,
134
+ };
135
+ }
136
+
137
+ export function buildConservativeMultimodalGenerationOptions(options = {}) {
138
+ return {
139
+ ...options,
140
+ disableCommandBatching: true,
141
+ disableMultiTokenDecode: true,
142
+ stopCheckMode: 'per-token',
143
+ };
144
+ }
56
145
 
57
146
  // ============================================================================
58
147
  // Main Inference Pipeline Class
@@ -68,6 +157,7 @@ export class InferencePipeline extends PipelineState {
68
157
 
69
158
 
70
159
  _preloadedWeights = null;
160
+ runtimeOverrides = null;
71
161
 
72
162
  constructor() {
73
163
  super();
@@ -90,6 +180,11 @@ export class InferencePipeline extends PipelineState {
90
180
  assignStorageContext: true,
91
181
  });
92
182
  this.runtimeConfig = runtimeConfig;
183
+ this.runtimeOverrides = contexts.runtimeConfig == null
184
+ ? null
185
+ : (typeof structuredClone === 'function'
186
+ ? structuredClone(contexts.runtimeConfig)
187
+ : JSON.parse(JSON.stringify(contexts.runtimeConfig)));
93
188
  applyPipelineDebugConfig(sharedDebug?.pipeline);
94
189
  configurePerfGuards(sharedDebug?.perfGuards);
95
190
 
@@ -116,6 +211,10 @@ export class InferencePipeline extends PipelineState {
116
211
  const loadStart = performance.now();
117
212
  this.manifest = manifest;
118
213
  this.decodeRing?.release();
214
+ if (this.sampleReadbackBuffer) {
215
+ this.sampleReadbackBuffer.destroy();
216
+ this.sampleReadbackBuffer = null;
217
+ }
119
218
  this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
120
219
  destroyMoERouter(this.moeRouter);
121
220
  this.moeRouter = null;
@@ -131,7 +230,7 @@ export class InferencePipeline extends PipelineState {
131
230
  // Phase 1 — applyExecutionV1RuntimeConfig
132
231
  // Reads: manifest.inference.execution, kernelCapabilities, platform
133
232
  // Mutates: runtimeConfig.inference (kernelPath, pipeline, compute,
134
- // kvcache via runtimeInferencePatch)
233
+ // session via runtimeInferencePatch)
135
234
  //
136
235
  // Phase 2 — parseModelConfig + applyModelBatchingRuntimeDefaults
137
236
  // Reads: manifest.architecture, runtimeConfig.inference.modelOverrides
@@ -141,7 +240,7 @@ export class InferencePipeline extends PipelineState {
141
240
  // Phase 3 — resolveKernelPathState
142
241
  // Reads: manifest, modelConfig.kernelPath, runtimeConfig.inference.kernelPath
143
242
  // Mutates: runtimeConfig.inference.compute.activationDtype,
144
- // runtimeConfig.inference.kvcache.kvDtype,
243
+ // runtimeConfig.inference.session.kvcache.kvDtype,
145
244
  // runtimeConfig.inference.session.compute.defaults.outputDtype
146
245
  //
147
246
  // Phase 4 — _resolveLayerPipeline
@@ -171,6 +270,7 @@ export class InferencePipeline extends PipelineState {
171
270
 
172
271
  const executionV1Runtime = applyExecutionV1RuntimeConfig({
173
272
  runtimeConfig: this.runtimeConfig,
273
+ runtimeOverrides: this.runtimeOverrides,
174
274
  manifest,
175
275
  modelId: manifest.modelId ?? 'model',
176
276
  numLayers: Number(manifest.architecture?.numLayers ?? 0),
@@ -204,12 +304,49 @@ export class InferencePipeline extends PipelineState {
204
304
  this.runtimeConfig = applyModelBatchingRuntimeDefaults(
205
305
  this.runtimeConfig,
206
306
  manifest,
207
- this.modelConfig
307
+ this.modelConfig,
308
+ this.runtimeOverrides
208
309
  );
209
310
  this.useTiedEmbeddings = this.modelConfig.useTiedEmbeddings;
210
311
  this.embeddingVocabSize = this.modelConfig.embeddingVocabSize;
211
312
  this.embeddingTranspose = this.modelConfig.embeddingTranspose;
212
313
 
314
+ // Vision capability detection — gated by manifest fields
315
+ const imageTokenId = manifest.image_token_id;
316
+ const hasVisionQuant = manifest.quantizationInfo?.vision != null;
317
+ if (Number.isInteger(imageTokenId) && imageTokenId > 0 && hasVisionQuant) {
318
+ this.visionCapable = true;
319
+ this.imageTokenId = imageTokenId;
320
+ this.visionConfig = this.modelConfig.visionConfig;
321
+ if (!this.visionConfig) {
322
+ throw new Error(
323
+ `Manifest declares image_token_id=${imageTokenId} and quantizationInfo.vision ` +
324
+ 'but no vision_config was resolved. Check conversion config.'
325
+ );
326
+ }
327
+ log.info('Pipeline', `Vision capable: imageTokenId=${imageTokenId}`);
328
+ } else {
329
+ this.visionCapable = false;
330
+ }
331
+
332
+ // Audio capability detection — gated by manifest fields
333
+ const audioTokenId = manifest.audio_token_id;
334
+ const hasAudioQuant = manifest.quantizationInfo?.audio != null;
335
+ if (Number.isInteger(audioTokenId) && audioTokenId > 0 && hasAudioQuant) {
336
+ this.audioCapable = true;
337
+ this.audioTokenId = audioTokenId;
338
+ this.audioConfig = this.modelConfig.audioConfig;
339
+ if (!this.audioConfig) {
340
+ throw new Error(
341
+ `Manifest declares audio_token_id=${audioTokenId} and quantizationInfo.audio ` +
342
+ 'but no audio_config was resolved. Check conversion config.'
343
+ );
344
+ }
345
+ log.info('Pipeline', `Audio capable: audioTokenId=${audioTokenId}`);
346
+ } else {
347
+ this.audioCapable = false;
348
+ }
349
+
213
350
  await runKernelWarmup({
214
351
  useGPU: this.useGPU,
215
352
  kernelWarmup: this.runtimeConfig.shared?.kernelWarmup,
@@ -222,6 +359,7 @@ export class InferencePipeline extends PipelineState {
222
359
  const kernelPathState = resolveKernelPathState({
223
360
  manifest,
224
361
  runtimeConfig: this.runtimeConfig,
362
+ runtimeOverrides: this.runtimeOverrides,
225
363
  modelConfig: this.modelConfig,
226
364
  });
227
365
  this.resolvedKernelPath = kernelPathState.resolvedKernelPath;
@@ -254,7 +392,7 @@ export class InferencePipeline extends PipelineState {
254
392
  // Check for execution-v1 kvDtype conflict with manifest quantization info
255
393
  if (this.executionV1State && this.resolvedKernelPath) {
256
394
  const manifestComputeHint = manifest?.quantizationInfo?.compute;
257
- const resolvedKvDtype = this.runtimeConfig.inference.kvcache?.kvDtype;
395
+ const resolvedKvDtype = this.runtimeConfig.inference.session?.kvcache?.kvDtype;
258
396
  if (
259
397
  manifestComputeHint
260
398
  && resolvedKvDtype
@@ -270,7 +408,16 @@ export class InferencePipeline extends PipelineState {
270
408
  }
271
409
 
272
410
  // Initialize KV cache
273
- this.kvCache = createKVCache(this.modelConfig, this.useGPU, this.debug, this.runtimeConfig.inference.kvcache);
411
+ if (this.modelConfig.decodeStrategy === 'replay_prefill') {
412
+ this.kvCache = null;
413
+ log.warn(
414
+ 'Pipeline',
415
+ 'Replay-prefill decode enabled for this model. Incremental KV-cache decode is disabled ' +
416
+ 'because the model config did not resolve explicit layerTypes for mixed-geometry/shared-KV decode.'
417
+ );
418
+ } else {
419
+ this.kvCache = createKVCache(this.modelConfig, this.useGPU, this.debug, this.runtimeConfig.inference);
420
+ }
274
421
  this.executionPlanState = compileExecutionPlanState({
275
422
  runtimeConfig: this.runtimeConfig,
276
423
  resolvedKernelPath: this.resolvedKernelPath,
@@ -344,6 +491,8 @@ export class InferencePipeline extends PipelineState {
344
491
  kernelPathSource: this.kernelPathSource,
345
492
  keepF32Weights: this.runtimeConfig.inference.compute.keepF32Weights === true,
346
493
  loaderDebug: this.runtimeConfig?.shared?.debug?.loader ?? null,
494
+ perLayerInputSession:
495
+ this.runtimeConfig.inference.session?.perLayerInputs ?? this.modelConfig.perLayerInputsSession,
347
496
  onProgress: (info) => {
348
497
  if (info.stage !== 'layers' && info.stage !== 'shards') {
349
498
  log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%${info.message ? ` - ${info.message}` : ''}`);
@@ -365,6 +514,7 @@ export class InferencePipeline extends PipelineState {
365
514
  this.weights.set('embed', result.embeddings);
366
515
  this.weights.set('lm_head', result.lmHead);
367
516
  this.weights.set('final_norm', result.finalNorm);
517
+ this.weights.set('per_layer_inputs', result.perLayerInputWeights);
368
518
  this.embeddingPostprocessor = result.embeddingPostprocessor;
369
519
 
370
520
  this.layerRouterWeights = result.layerRouterWeights;
@@ -380,7 +530,7 @@ export class InferencePipeline extends PipelineState {
380
530
  }
381
531
 
382
532
  if (this.useGPU && this.modelConfig) {
383
- fuseQKVWeights(result.layerWeights, this.modelConfig);
533
+ fuseQKVWeights(result.layerWeights, this.modelConfig, this.resolvedKernelPath);
384
534
  }
385
535
 
386
536
  if (this.useGPU && this.modelConfig) {
@@ -388,7 +538,7 @@ export class InferencePipeline extends PipelineState {
388
538
  try {
389
539
  this.decodeBuffers?.ensureBuffers({
390
540
  hiddenSize: this.modelConfig.hiddenSize,
391
- intermediateSize: this.modelConfig.intermediateSize,
541
+ intermediateSize: this.modelConfig.maxIntermediateSize,
392
542
  activationDtype: activeExecutionPlan.activationDtype,
393
543
  enablePingPong: true,
394
544
  });
@@ -422,14 +572,19 @@ export class InferencePipeline extends PipelineState {
422
572
  const config = (this.modelConfig);
423
573
  const maxSeqLen = config.maxSeqLen;
424
574
  const ropeBuffers = await initRoPEFrequencies({
425
- headDim: config.headDim,
575
+ headDim: config.globalHeadDim ?? config.headDim,
576
+ localHeadDim: config.headDim,
426
577
  rotaryDim: config.ropeRotaryDim,
578
+ ropeLocalRotaryDim: config.ropeLocalRotaryDim,
579
+ ropeFrequencyBaseDim: config.ropeFrequencyBaseDim,
580
+ ropeLocalFrequencyBaseDim: config.ropeLocalFrequencyBaseDim,
427
581
  maxSeqLen,
428
582
  ropeTheta: config.ropeTheta,
429
583
  ropeLocalTheta: config.ropeLocalTheta,
430
584
  mropeInterleaved: config.mropeInterleaved,
431
585
  mropeSection: config.mropeSection,
432
586
  partialRotaryFactor: config.partialRotaryFactor,
587
+ ropeLocalPartialRotaryFactor: config.ropeLocalPartialRotaryFactor,
433
588
  ropeScale: config.ropeScale,
434
589
  ropeLocalScale: config.ropeLocalScale,
435
590
  ropeScalingType: config.ropeScalingType,
@@ -489,6 +644,667 @@ export class InferencePipeline extends PipelineState {
489
644
  }
490
645
 
491
646
 
647
+ async _loadVisionWeights() {
648
+ const loader = this.dopplerLoader ?? getDopplerLoader(this.runtimeConfig.loading);
649
+ const vc = this.visionConfig;
650
+ const depth = vc.depth;
651
+
652
+ const loadRequiredTensor = async (name, toGPU = true) => {
653
+ const tensor = await loader.loadTensor(name, toGPU, true);
654
+ if (!tensor) {
655
+ throw new Error(`Vision tensor "${name}" is missing from the converted artifact.`);
656
+ }
657
+ return tensor;
658
+ };
659
+ const loadScalar = async (name) => {
660
+ const tensor = await loadRequiredTensor(name, false);
661
+ if (tensor instanceof Float32Array) {
662
+ if (tensor.length !== 1) {
663
+ throw new Error(`Vision scalar "${name}" must be a single-element tensor, got length=${tensor.length}.`);
664
+ }
665
+ return tensor[0];
666
+ }
667
+ if (ArrayBuffer.isView(tensor) && tensor.length === 1) {
668
+ return Number(tensor[0]);
669
+ }
670
+ if (typeof tensor === 'number') {
671
+ return tensor;
672
+ }
673
+ throw new Error(
674
+ `Vision scalar "${name}" must decode to a single numeric value, ` +
675
+ `got ${tensor?.constructor?.name ?? typeof tensor} length=${tensor?.length ?? 'N/A'}.`
676
+ );
677
+ };
678
+ const loadClipRange = async (prefix) => ({
679
+ inputMin: await loadScalar(`${prefix}.input_min`),
680
+ inputMax: await loadScalar(`${prefix}.input_max`),
681
+ outputMin: await loadScalar(`${prefix}.output_min`),
682
+ outputMax: await loadScalar(`${prefix}.output_max`),
683
+ });
684
+
685
+ if (vc.visionArchitecture === 'gemma4') {
686
+ const visionWeights = {
687
+ textHiddenSize: this.modelConfig.hiddenSize,
688
+ patchInputProj: await loadRequiredTensor('model.vision_tower.patch_embedder.input_proj.weight'),
689
+ patchPositionEmbeddingTable: await loadRequiredTensor('model.vision_tower.patch_embedder.position_embedding_table', false),
690
+ projector: await loadRequiredTensor('model.embed_vision.embedding_projection.weight'),
691
+ layers: [],
692
+ };
693
+
694
+ if (!(visionWeights.patchPositionEmbeddingTable instanceof Float32Array)) {
695
+ throw new Error(
696
+ 'Gemma 4 vision position_embedding_table must decode to Float32Array on CPU. ' +
697
+ 'Re-convert the artifact if this tensor was quantized incorrectly.'
698
+ );
699
+ }
700
+
701
+ for (let i = 0; i < depth; i++) {
702
+ const prefix = `model.vision_tower.encoder.layers.${i}`;
703
+ const attnPrefix = `${prefix}.self_attn`;
704
+ const mlpPrefix = `${prefix}.mlp`;
705
+ visionWeights.layers.push({
706
+ inputLayerNorm: await loadRequiredTensor(`${prefix}.input_layernorm.weight`),
707
+ postAttentionLayerNorm: await loadRequiredTensor(`${prefix}.post_attention_layernorm.weight`),
708
+ preFeedforwardLayerNorm: await loadRequiredTensor(`${prefix}.pre_feedforward_layernorm.weight`),
709
+ postFeedforwardLayerNorm: await loadRequiredTensor(`${prefix}.post_feedforward_layernorm.weight`),
710
+ qNorm: await loadRequiredTensor(`${attnPrefix}.q_norm.weight`),
711
+ kNorm: await loadRequiredTensor(`${attnPrefix}.k_norm.weight`),
712
+ qProj: await loadRequiredTensor(`${attnPrefix}.q_proj.linear.weight`),
713
+ kProj: await loadRequiredTensor(`${attnPrefix}.k_proj.linear.weight`),
714
+ vProj: await loadRequiredTensor(`${attnPrefix}.v_proj.linear.weight`),
715
+ oProj: await loadRequiredTensor(`${attnPrefix}.o_proj.linear.weight`),
716
+ qProjClip: await loadClipRange(`${attnPrefix}.q_proj`),
717
+ kProjClip: await loadClipRange(`${attnPrefix}.k_proj`),
718
+ vProjClip: await loadClipRange(`${attnPrefix}.v_proj`),
719
+ oProjClip: await loadClipRange(`${attnPrefix}.o_proj`),
720
+ gateProj: await loadRequiredTensor(`${mlpPrefix}.gate_proj.linear.weight`),
721
+ upProj: await loadRequiredTensor(`${mlpPrefix}.up_proj.linear.weight`),
722
+ downProj: await loadRequiredTensor(`${mlpPrefix}.down_proj.linear.weight`),
723
+ gateProjClip: await loadClipRange(`${mlpPrefix}.gate_proj`),
724
+ upProjClip: await loadClipRange(`${mlpPrefix}.up_proj`),
725
+ downProjClip: await loadClipRange(`${mlpPrefix}.down_proj`),
726
+ });
727
+ }
728
+
729
+ this.visionWeights = visionWeights;
730
+ log.info('Pipeline', `Vision weights loaded (${depth} Gemma 4 encoder layers)`);
731
+ return;
732
+ }
733
+
734
+ const visionWeights = {};
735
+
736
+ // Patch embedding weights
737
+ const patchProjName = 'visual.patch_embed.proj.weight';
738
+ const patchProjBiasName = 'visual.patch_embed.proj.bias';
739
+ visionWeights.patchProjWeight = await loader.loadTensor(patchProjName, true, true);
740
+ visionWeights.patchProjBias = await loader.loadTensor(patchProjBiasName, true, true);
741
+
742
+ // Vision encoder layer weights
743
+ visionWeights.layers = [];
744
+ for (let i = 0; i < depth; i++) {
745
+ const prefix = `visual.blocks.${i}`;
746
+ const layerW = {
747
+ norm1Weight: await loader.loadTensor(`${prefix}.norm1.weight`, true, true),
748
+ norm2Weight: await loader.loadTensor(`${prefix}.norm2.weight`, true, true),
749
+ qkvWeight: await loader.loadTensor(`${prefix}.attn.qkv.weight`, true, true),
750
+ qkvBias: await loader.loadTensor(`${prefix}.attn.qkv.bias`, true, true),
751
+ projWeight: await loader.loadTensor(`${prefix}.attn.proj.weight`, true, true),
752
+ projBias: await loader.loadTensor(`${prefix}.attn.proj.bias`, true, true),
753
+ fc1Weight: await loader.loadTensor(`${prefix}.mlp.fc1.weight`, true, true),
754
+ fc1Bias: await loader.loadTensor(`${prefix}.mlp.fc1.bias`, true, true),
755
+ fc2Weight: await loader.loadTensor(`${prefix}.mlp.fc2.weight`, true, true),
756
+ fc2Bias: await loader.loadTensor(`${prefix}.mlp.fc2.bias`, true, true),
757
+ };
758
+ visionWeights.layers.push(layerW);
759
+ }
760
+
761
+ // Spatial merge projection
762
+ visionWeights.mergerLnWeight = await loader.loadTensor('visual.merger.ln_q.weight', true, true);
763
+ visionWeights.mergerMlp0Weight = await loader.loadTensor('visual.merger.mlp.0.weight', true, true);
764
+ visionWeights.mergerMlp0Bias = await loader.loadTensor('visual.merger.mlp.0.bias', true, true);
765
+ visionWeights.mergerMlp2Weight = await loader.loadTensor('visual.merger.mlp.2.weight', true, true);
766
+ visionWeights.mergerMlp2Bias = await loader.loadTensor('visual.merger.mlp.2.bias', true, true);
767
+
768
+ this.visionWeights = visionWeights;
769
+ log.info('Pipeline', `Vision weights loaded (${depth} encoder layers)`);
770
+ }
771
+
772
+ async _ensureVisionWeightsLoaded() {
773
+ if (!this.visionCapable) {
774
+ throw new Error(
775
+ 'Pipeline does not support vision weights (no image_token_id in manifest).'
776
+ );
777
+ }
778
+ if (this.visionWeights) {
779
+ return;
780
+ }
781
+ log.info('Pipeline', 'Loading vision weights on demand');
782
+ await this._loadVisionWeights();
783
+ }
784
+
785
+
786
+ async _loadAudioWeights() {
787
+ const loader = this.dopplerLoader ?? getDopplerLoader(this.runtimeConfig.loading);
788
+ const ac = this.audioConfig;
789
+ const depth = ac.depth;
790
+
791
+ const loadRequiredTensor = async (name, toGPU = true) => {
792
+ const tensor = await loader.loadTensor(name, toGPU, true);
793
+ if (!tensor) {
794
+ throw new Error(`Audio tensor "${name}" is missing from the converted artifact.`);
795
+ }
796
+ return tensor;
797
+ };
798
+ const loadScalar = async (name) => {
799
+ const tensor = await loadRequiredTensor(name, false);
800
+ if (tensor instanceof Float32Array) {
801
+ if (tensor.length !== 1) {
802
+ throw new Error(`Audio scalar "${name}" must be a single-element tensor, got length=${tensor.length}.`);
803
+ }
804
+ return tensor[0];
805
+ }
806
+ if (ArrayBuffer.isView(tensor) && tensor.length === 1) {
807
+ return Number(tensor[0]);
808
+ }
809
+ if (typeof tensor === 'number') {
810
+ return tensor;
811
+ }
812
+ throw new Error(
813
+ `Audio scalar "${name}" must decode to a single numeric value, ` +
814
+ `got ${tensor?.constructor?.name ?? typeof tensor} length=${tensor?.length ?? 'N/A'}.`
815
+ );
816
+ };
817
+ const loadClipRange = async (prefix) => ({
818
+ inputMin: await loadScalar(`${prefix}.input_min`),
819
+ inputMax: await loadScalar(`${prefix}.input_max`),
820
+ outputMin: await loadScalar(`${prefix}.output_min`),
821
+ outputMax: await loadScalar(`${prefix}.output_max`),
822
+ });
823
+
824
+ const audioWeights = {
825
+ // Subsampling
826
+ subsampleConv0Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.conv.weight'),
827
+ subsampleNorm0Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.norm.weight'),
828
+ subsampleConv1Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.conv.weight'),
829
+ subsampleNorm1Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.norm.weight'),
830
+ subsampleInputProjWeight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.input_proj_linear.weight'),
831
+ // Output
832
+ outputProjWeight: await loadRequiredTensor('model.audio_tower.output_proj.weight'),
833
+ outputProjBias: await loadRequiredTensor('model.audio_tower.output_proj.bias'),
834
+ audioEmbeddingProjWeight: await loadRequiredTensor('model.embed_audio.embedding_projection.weight'),
835
+ layers: [],
836
+ };
837
+
838
+ for (let i = 0; i < depth; i++) {
839
+ const prefix = `model.audio_tower.layers.${i}`;
840
+ const layer = {
841
+ // Feed-forward 1 (Macaron half-step)
842
+ feedForward1: {
843
+ preLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward1.pre_layer_norm.weight`),
844
+ ffwLayer1Weight: await loadRequiredTensor(`${prefix}.feed_forward1.ffw_layer_1.linear.weight`),
845
+ ffwLayer1Clip: await loadClipRange(`${prefix}.feed_forward1.ffw_layer_1`),
846
+ ffwLayer2Weight: await loadRequiredTensor(`${prefix}.feed_forward1.ffw_layer_2.linear.weight`),
847
+ ffwLayer2Clip: await loadClipRange(`${prefix}.feed_forward1.ffw_layer_2`),
848
+ postLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward1.post_layer_norm.weight`),
849
+ },
850
+ // Self-attention
851
+ normPreAttn: await loadRequiredTensor(`${prefix}.norm_pre_attn.weight`),
852
+ qProj: await loadRequiredTensor(`${prefix}.self_attn.q_proj.linear.weight`),
853
+ qProjClip: await loadClipRange(`${prefix}.self_attn.q_proj`),
854
+ kProj: await loadRequiredTensor(`${prefix}.self_attn.k_proj.linear.weight`),
855
+ kProjClip: await loadClipRange(`${prefix}.self_attn.k_proj`),
856
+ vProj: await loadRequiredTensor(`${prefix}.self_attn.v_proj.linear.weight`),
857
+ vProjClip: await loadClipRange(`${prefix}.self_attn.v_proj`),
858
+ perDimScale: await loadRequiredTensor(`${prefix}.self_attn.per_dim_scale`),
859
+ relativeKProj: await loadRequiredTensor(`${prefix}.self_attn.relative_k_proj.weight`),
860
+ postProj: await loadRequiredTensor(`${prefix}.self_attn.post.linear.weight`),
861
+ postProjClip: await loadClipRange(`${prefix}.self_attn.post`),
862
+ normPostAttn: await loadRequiredTensor(`${prefix}.norm_post_attn.weight`),
863
+ // Convolution module (LConv1D)
864
+ lconvPreLayerNorm: await loadRequiredTensor(`${prefix}.lconv1d.pre_layer_norm.weight`),
865
+ lconvLinearStartWeight: await loadRequiredTensor(`${prefix}.lconv1d.linear_start.linear.weight`),
866
+ lconvLinearStartClip: await loadClipRange(`${prefix}.lconv1d.linear_start`),
867
+ lconvDepthwiseWeight: await loadRequiredTensor(`${prefix}.lconv1d.depthwise_conv1d.weight`),
868
+ lconvConvNorm: await loadRequiredTensor(`${prefix}.lconv1d.conv_norm.weight`),
869
+ lconvLinearEndWeight: await loadRequiredTensor(`${prefix}.lconv1d.linear_end.linear.weight`),
870
+ lconvLinearEndClip: await loadClipRange(`${prefix}.lconv1d.linear_end`),
871
+ // Feed-forward 2 (Macaron half-step)
872
+ feedForward2: {
873
+ preLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward2.pre_layer_norm.weight`),
874
+ ffwLayer1Weight: await loadRequiredTensor(`${prefix}.feed_forward2.ffw_layer_1.linear.weight`),
875
+ ffwLayer1Clip: await loadClipRange(`${prefix}.feed_forward2.ffw_layer_1`),
876
+ ffwLayer2Weight: await loadRequiredTensor(`${prefix}.feed_forward2.ffw_layer_2.linear.weight`),
877
+ ffwLayer2Clip: await loadClipRange(`${prefix}.feed_forward2.ffw_layer_2`),
878
+ postLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward2.post_layer_norm.weight`),
879
+ },
880
+ // Final layer norm
881
+ normOut: await loadRequiredTensor(`${prefix}.norm_out.weight`),
882
+ };
883
+ audioWeights.layers.push(layer);
884
+ }
885
+
886
+ this.audioWeights = audioWeights;
887
+ log.info('Pipeline', `Audio weights loaded (${depth} conformer layers)`);
888
+ }
889
+
890
+ async _ensureAudioWeightsLoaded() {
891
+ if (!this.audioCapable) {
892
+ throw new Error(
893
+ 'Pipeline does not support audio weights (no audio_token_id in manifest).'
894
+ );
895
+ }
896
+ if (this.audioWeights) {
897
+ return;
898
+ }
899
+ log.info('Pipeline', 'Loading audio weights on demand');
900
+ await this._loadAudioWeights();
901
+ }
902
+
903
+
904
+ // ==========================================================================
905
+ // Vision: transcribeImage
906
+ // ==========================================================================
907
+
908
+ /**
909
+ * Transcribe text from an image using the vision encoder and text decoder.
910
+ *
911
+ * @param {object} params
912
+ * @param {Uint8Array|Float32Array} params.imageBytes Raw image pixel data
913
+ * @param {number} params.width Image width
914
+ * @param {number} params.height Image height
915
+ * @param {string} [params.prompt] Custom transcription prompt
916
+ * @param {number} [params.maxTokens] Max tokens to generate
917
+ * @param {number} [params.softTokenBudget] Per-request soft token budget (Gemma 4 tiers: 70/140/280/560/1120)
918
+ * @returns {Promise<{ text: string, tokens: number[] }>}
919
+ */
920
+ async transcribeImage({ imageBytes, width, height, prompt, maxTokens, softTokenBudget }) {
921
+ if (!this.visionCapable) {
922
+ throw new Error(
923
+ 'Pipeline does not support image transcription (no image_token_id in manifest).'
924
+ );
925
+ }
926
+ await this._ensureVisionWeightsLoaded();
927
+
928
+ this.reset();
929
+
930
+ // Lazy-load vision module (avoids GPU kernel dependency for text-only pipelines)
931
+ const { encodeImage } = await import('./vision/index.js');
932
+
933
+ // Step 1: Encode image through vision pipeline
934
+ const encodeResult = await encodeImage({
935
+ pixels: imageBytes,
936
+ width,
937
+ height,
938
+ visionConfig: this.visionConfig,
939
+ weights: this.visionWeights,
940
+ softTokenBudget,
941
+ });
942
+
943
+ // Step 2: Build the multimodal prompt from the model's chat template and
944
+ // expand the single <|image|> placeholder into the exact visual-token span.
945
+ const requestedPrompt = prompt ?? 'Describe the image in one short sentence.';
946
+ const imageTokenId = this.visionConfig?.imageTokenId ?? this.modelConfig?.imageTokenId;
947
+ if (imageTokenId == null) {
948
+ throw new Error(
949
+ 'Pipeline missing image_token_id. Re-convert the model with image token metadata.'
950
+ );
951
+ }
952
+ if (this.visionConfig?.visionArchitecture !== 'gemma4') {
953
+ throw new Error(
954
+ `[Pipeline] transcribeImage: unsupported vision architecture "${this.visionConfig?.visionArchitecture ?? 'unknown'}". ` +
955
+ 'This runtime path currently requires Gemma 4 multimodal prompt expansion.'
956
+ );
957
+ }
958
+ const templateType = this.modelConfig?.chatTemplateType ?? 'gemma4';
959
+ const chatOptions = this.modelConfig?.chatTemplateThinking === true ? { thinking: true } : undefined;
960
+ const multimodalPrompt = formatChatMessages([
961
+ {
962
+ role: 'user',
963
+ content: [
964
+ { type: 'image' },
965
+ { type: 'text', text: requestedPrompt },
966
+ ],
967
+ },
968
+ ], templateType, chatOptions);
969
+ const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
970
+ const imageTokenSpanLength = encodeResult.numTokens;
971
+ const effectiveBudget = softTokenBudget ?? this.visionConfig?.defaultOutputLength;
972
+ const maxImageTokenSpanLength = Number(effectiveBudget);
973
+ if (!Number.isFinite(maxImageTokenSpanLength) || maxImageTokenSpanLength < 1 || Math.floor(maxImageTokenSpanLength) !== maxImageTokenSpanLength) {
974
+ throw new Error(
975
+ `[Pipeline] transcribeImage: invalid soft token budget ${effectiveBudget}. ` +
976
+ 'Expected a positive integer from the resolved vision config or softTokenBudget parameter.'
977
+ );
978
+ }
979
+ if (imageTokenSpanLength > maxImageTokenSpanLength) {
980
+ throw new Error(
981
+ `[Pipeline] transcribeImage: encoded Gemma 4 image produced ${imageTokenSpanLength} soft tokens, ` +
982
+ `which exceeds the effective soft token budget=${maxImageTokenSpanLength}.`
983
+ );
984
+ }
985
+ const boiTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<|image>', 'Gemma 4 BOI token');
986
+ const eoiTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<image|>', 'Gemma 4 EOI token');
987
+ const { inputIds: fullTokenIds, imageStartOffset } = expandImagePlaceholderTokenIds(
988
+ promptTokenIds,
989
+ imageTokenId,
990
+ imageTokenSpanLength,
991
+ { boiTokenId, eoiTokenId }
992
+ );
993
+ const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
994
+ if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
995
+ throw new Error(
996
+ `[Pipeline] transcribeImage: Gemma 4 multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
997
+ );
998
+ }
999
+
1000
+ // Step 3: Generate with embedding override at the image token offset.
1001
+ const tokens = [];
1002
+ const maxGen = maxTokens ?? 512;
1003
+ const stopTokenIds = this.modelConfig.stopTokenIds;
1004
+
1005
+ try {
1006
+ const generation = await this.generator.generateTokenIds('', buildConservativeMultimodalGenerationOptions({
1007
+ inputIds: fullTokenIds,
1008
+ embeddingOverrides: {
1009
+ prefixLength: encodeResult.numTokens,
1010
+ offset: imageStartOffset,
1011
+ embeddings: encodeResult.features,
1012
+ },
1013
+ __internalEmbeddingInputSpan: {
1014
+ offset: imageStartOffset,
1015
+ length: encodeResult.numTokens,
1016
+ tokenId: padTokenId,
1017
+ },
1018
+ __internalMultimodalBidirectionalSpan: {
1019
+ offset: imageStartOffset,
1020
+ length: encodeResult.numTokens,
1021
+ },
1022
+ maxTokens: maxGen,
1023
+ temperature: 0,
1024
+ topK: 1,
1025
+ topP: 1,
1026
+ repetitionPenalty: 1,
1027
+ }));
1028
+ for (const token of generation.tokenIds ?? []) {
1029
+ if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
1030
+ tokens.push(token);
1031
+ }
1032
+ } finally {
1033
+ if (encodeResult.features) {
1034
+ releaseBuffer(encodeResult.features);
1035
+ }
1036
+ }
1037
+
1038
+ const text = this.tokenizer.decode(tokens);
1039
+ return { text, tokens };
1040
+ }
1041
+
1042
+
1043
+ // ==========================================================================
1044
+ // Video: transcribeVideo
1045
+ // ==========================================================================
1046
+
1047
+ /**
1048
+ * Transcribe text from video using the vision encoder (per-frame) and text decoder.
1049
+ *
1050
+ * Video is processed as sampled frames through the existing vision encoder.
1051
+ * Each frame is encoded independently and the visual tokens are concatenated.
1052
+ *
1053
+ * @param {object} params
1054
+ * @param {Array<{ pixels: Uint8Array|Float32Array, width: number, height: number }>} params.frames Decoded video frames
1055
+ * @param {string} [params.prompt] Custom transcription prompt
1056
+ * @param {number} [params.maxTokens] Max tokens to generate
1057
+ * @param {number} [params.maxFrames=8] Maximum frames to sample
1058
+ * @param {number} [params.perFrameSoftTokenBudget] Soft token budget per frame
1059
+ * @returns {Promise<{ text: string, tokens: number[] }>}
1060
+ */
1061
+ async transcribeVideo({ frames, prompt, maxTokens, maxFrames, perFrameSoftTokenBudget }) {
1062
+ if (!this.visionCapable) {
1063
+ throw new Error(
1064
+ 'Pipeline does not support video transcription (no image_token_id in manifest for vision encoder).'
1065
+ );
1066
+ }
1067
+ await this._ensureVisionWeightsLoaded();
1068
+
1069
+ this.reset();
1070
+
1071
+ // Lazy-load video module
1072
+ const { encodeVideo } = await import('./video/index.js');
1073
+
1074
+ // Step 1: Encode video frames through vision pipeline
1075
+ const encodeResult = await encodeVideo({
1076
+ frames,
1077
+ visionConfig: this.visionConfig,
1078
+ weights: this.visionWeights,
1079
+ maxFrames: maxFrames ?? 8,
1080
+ perFrameSoftTokenBudget,
1081
+ });
1082
+
1083
+ // Step 2: Build the multimodal prompt with <|video|> placeholder
1084
+ const requestedPrompt = prompt ?? 'Describe the video in one short sentence.';
1085
+ const videoTokenId = this.tokenizer?.model?.tokenToId?.('<|video_token|>')
1086
+ ?? this.tokenizer?.encode?.('<|video|>')?.find?.((id) => id !== undefined)
1087
+ ?? null;
1088
+ // Fall back to image token ID for video placeholder expansion
1089
+ const placeholderTokenId = videoTokenId ?? this.visionConfig?.imageTokenId ?? this.imageTokenId;
1090
+ if (placeholderTokenId == null) {
1091
+ throw new Error(
1092
+ 'Pipeline missing video/image token ID for video placeholder expansion.'
1093
+ );
1094
+ }
1095
+
1096
+ const templateType = this.modelConfig?.chatTemplateType ?? 'gemma4';
1097
+ const chatOptions = this.modelConfig?.chatTemplateThinking === true ? { thinking: true } : undefined;
1098
+ const multimodalPrompt = formatChatMessages([
1099
+ {
1100
+ role: 'user',
1101
+ content: [
1102
+ { type: 'video' },
1103
+ { type: 'text', text: requestedPrompt },
1104
+ ],
1105
+ },
1106
+ ], templateType, chatOptions);
1107
+ const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
1108
+ const videoTokenSpanLength = encodeResult.numTokens;
1109
+
1110
+ // Resolve BOV/EOV tokens (reuse image BOI/EOI if video-specific ones don't exist)
1111
+ const bovTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<|video|>', 'Gemma 4 BOV token');
1112
+ const eovTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<video|>', 'Gemma 4 EOV token');
1113
+
1114
+ const { inputIds: fullTokenIds, imageStartOffset: videoStartOffset } = expandImagePlaceholderTokenIds(
1115
+ promptTokenIds,
1116
+ placeholderTokenId,
1117
+ videoTokenSpanLength,
1118
+ { boiTokenId: bovTokenId, eoiTokenId: eovTokenId }
1119
+ );
1120
+
1121
+ const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
1122
+ if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
1123
+ throw new Error(
1124
+ `[Pipeline] transcribeVideo: Gemma 4 multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
1125
+ );
1126
+ }
1127
+
1128
+ // Step 3: Generate with embedding override at the video token offset
1129
+ const tokens = [];
1130
+ const maxGen = maxTokens ?? 512;
1131
+ const stopTokenIds = this.modelConfig.stopTokenIds;
1132
+
1133
+ try {
1134
+ const generation = await this.generator.generateTokenIds('', buildConservativeMultimodalGenerationOptions({
1135
+ inputIds: fullTokenIds,
1136
+ embeddingOverrides: {
1137
+ prefixLength: encodeResult.numTokens,
1138
+ offset: videoStartOffset,
1139
+ embeddings: encodeResult.features,
1140
+ },
1141
+ __internalEmbeddingInputSpan: {
1142
+ offset: videoStartOffset,
1143
+ length: encodeResult.numTokens,
1144
+ tokenId: padTokenId,
1145
+ },
1146
+ __internalMultimodalBidirectionalSpan: {
1147
+ offset: videoStartOffset,
1148
+ length: encodeResult.numTokens,
1149
+ },
1150
+ maxTokens: maxGen,
1151
+ temperature: 0,
1152
+ topK: 1,
1153
+ topP: 1,
1154
+ repetitionPenalty: 1,
1155
+ }));
1156
+ for (const token of generation.tokenIds ?? []) {
1157
+ if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
1158
+ tokens.push(token);
1159
+ }
1160
+ } finally {
1161
+ if (encodeResult.features) {
1162
+ releaseBuffer(encodeResult.features);
1163
+ }
1164
+ }
1165
+
1166
+ const text = this.tokenizer.decode(tokens);
1167
+ return { text, tokens };
1168
+ }
1169
+
1170
+
1171
+ // ==========================================================================
1172
+ // Audio: transcribeAudio
1173
+ // ==========================================================================
1174
+
1175
+ /**
1176
+ * Transcribe text from audio using the audio encoder and text decoder.
1177
+ *
1178
+ * @param {object} params
1179
+ * @param {Float32Array} params.audio Raw audio PCM (mono, 16kHz)
1180
+ * @param {string} [params.prompt] Custom transcription prompt
1181
+ * @param {number} [params.maxTokens] Max tokens to generate
1182
+ * @returns {Promise<{ text: string, tokens: number[] }>}
1183
+ */
1184
+ async transcribeAudio({ audio, prompt, maxTokens }) {
1185
+ if (!this.audioCapable) {
1186
+ throw new Error(
1187
+ 'Pipeline does not support audio transcription (no audio_token_id in manifest).'
1188
+ );
1189
+ }
1190
+ await this._ensureAudioWeightsLoaded();
1191
+
1192
+ this.reset();
1193
+
1194
+ // Lazy-load audio modules
1195
+ const { extractLogMelSpectrogram } = await import('./audio/mel.js');
1196
+ const { encodeAudio } = await import('./audio/index.js');
1197
+
1198
+ // Step 1: Extract mel spectrogram
1199
+ const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
1200
+
1201
+ // Step 2: Encode audio through conformer pipeline
1202
+ const encodeResult = await encodeAudio({
1203
+ melFeatures,
1204
+ numFrames,
1205
+ nMels,
1206
+ audioConfig: this.audioConfig,
1207
+ weights: this.audioWeights,
1208
+ });
1209
+
1210
+ // Step 3: Build the multimodal prompt with <|audio|> placeholder
1211
+ const requestedPrompt = prompt ?? 'Transcribe the audio.';
1212
+ const audioTokenId = this.audioConfig?.audioTokenId ?? this.audioTokenId;
1213
+ if (audioTokenId == null) {
1214
+ throw new Error(
1215
+ 'Pipeline missing audio_token_id. Re-convert the model with audio token metadata.'
1216
+ );
1217
+ }
1218
+ const templateType = this.modelConfig?.chatTemplateType ?? 'gemma4';
1219
+ const chatOptions = this.modelConfig?.chatTemplateThinking === true ? { thinking: true } : undefined;
1220
+ const multimodalPrompt = formatChatMessages([
1221
+ {
1222
+ role: 'user',
1223
+ content: [
1224
+ { type: 'audio' },
1225
+ { type: 'text', text: requestedPrompt },
1226
+ ],
1227
+ },
1228
+ ], templateType, chatOptions);
1229
+ const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
1230
+ const audioTokenSpanLength = encodeResult.numTokens;
1231
+
1232
+ // Resolve BOA/EOA tokens
1233
+ const boaTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<|audio|>', 'Gemma 4 BOA token');
1234
+ const eoaTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<audio|>', 'Gemma 4 EOA token');
1235
+
1236
+ // Expand single audio placeholder token into the full audio token span
1237
+ const { inputIds: fullTokenIds, imageStartOffset: audioStartOffset } = expandImagePlaceholderTokenIds(
1238
+ promptTokenIds,
1239
+ audioTokenId,
1240
+ audioTokenSpanLength,
1241
+ { boiTokenId: boaTokenId, eoiTokenId: eoaTokenId }
1242
+ );
1243
+
1244
+ const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
1245
+ if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
1246
+ throw new Error(
1247
+ `[Pipeline] transcribeAudio: Gemma 4 multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
1248
+ );
1249
+ }
1250
+
1251
+ // Step 4: Generate with embedding override at the audio token offset
1252
+ const tokens = [];
1253
+ const maxGen = maxTokens ?? 512;
1254
+ const stopTokenIds = this.modelConfig.stopTokenIds;
1255
+
1256
+ try {
1257
+ const generation = await this.generator.generateTokenIds('', buildConservativeMultimodalGenerationOptions({
1258
+ inputIds: fullTokenIds,
1259
+ embeddingOverrides: {
1260
+ prefixLength: encodeResult.numTokens,
1261
+ offset: audioStartOffset,
1262
+ embeddings: encodeResult.features,
1263
+ },
1264
+ __internalEmbeddingInputSpan: {
1265
+ offset: audioStartOffset,
1266
+ length: encodeResult.numTokens,
1267
+ tokenId: padTokenId,
1268
+ },
1269
+ __internalMultimodalBidirectionalSpan: {
1270
+ offset: audioStartOffset,
1271
+ length: encodeResult.numTokens,
1272
+ },
1273
+ maxTokens: maxGen,
1274
+ temperature: 0,
1275
+ topK: 1,
1276
+ topP: 1,
1277
+ repetitionPenalty: 1,
1278
+ }));
1279
+ for (const token of generation.tokenIds ?? []) {
1280
+ if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
1281
+ tokens.push(token);
1282
+ }
1283
+ } finally {
1284
+ if (encodeResult.features) {
1285
+ releaseBuffer(encodeResult.features);
1286
+ }
1287
+ }
1288
+
1289
+ const text = this.tokenizer.decode(tokens);
1290
+ return { text, tokens };
1291
+ }
1292
+
1293
+
1294
+ // ==========================================================================
1295
+ // Capability Detection
1296
+ // ==========================================================================
1297
+
1298
+ get capabilities() {
1299
+ const caps = ['generation'];
1300
+ if (typeof this.prefillWithEmbedding === 'function') caps.push('embedding');
1301
+ if (this.visionCapable) caps.push('multimodal');
1302
+ if (this.audioCapable) caps.push('audio');
1303
+ if (this.visionCapable) caps.push('video');
1304
+ return Object.freeze(caps);
1305
+ }
1306
+
1307
+
492
1308
  // Layer pipeline precedence (lowest to highest):
493
1309
  // 1. execution-v1-produced pipeline (via runtimeInferencePatch.pipeline)
494
1310
  // 2. model config pipeline (manifest inference.pipeline)
@@ -580,10 +1396,11 @@ export class InferencePipeline extends PipelineState {
580
1396
  if (!Array.isArray(prompts)) {
581
1397
  throw new Error('embedBatch expects an array of prompts');
582
1398
  }
1399
+ const batchOptions = { ...options, __skipStateSnapshot: true };
583
1400
  const outputs = [];
584
1401
  for (const prompt of prompts) {
585
- outputs.push(await this.embed(prompt, options));
586
- this.reset();
1402
+ outputs.push(await this.embed(prompt, batchOptions));
1403
+ this.resetForBatch();
587
1404
  }
588
1405
  return outputs;
589
1406
  }
@@ -629,6 +1446,7 @@ export class InferencePipeline extends PipelineState {
629
1446
 
630
1447
  getStats() {
631
1448
  const stats = { ...this.stats };
1449
+ stats.batching ??= { ...this.batchingStats };
632
1450
  if (this.executionPlanState) {
633
1451
  const activeExecutionPlan = resolveActiveExecutionPlan(this);
634
1452
  stats.executionPlan ??= {
@@ -734,6 +1552,10 @@ export class InferencePipeline extends PipelineState {
734
1552
  this.emulation = null;
735
1553
  this.decodeRing?.release();
736
1554
  this.kvCache?.clear();
1555
+ destroyPleRuntimeCache(this.weights.get('per_layer_inputs'));
1556
+ destroyPleBufferCache(this.pleCache);
1557
+ this.pleCache = null;
1558
+ this.plePrefetchPending = null;
737
1559
  this.weights.clear();
738
1560
  this.expertWeights.clear();
739
1561
  this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
@@ -744,6 +1566,10 @@ export class InferencePipeline extends PipelineState {
744
1566
  this.finitenessBuffer.destroy();
745
1567
  this.finitenessBuffer = null;
746
1568
  }
1569
+ if (this.sampleReadbackBuffer) {
1570
+ this.sampleReadbackBuffer.destroy();
1571
+ this.sampleReadbackBuffer = null;
1572
+ }
747
1573
  this.isLoaded = false;
748
1574
  this.currentSeqLen = 0;
749
1575
  restorePipelineContexts(this);
@@ -776,6 +1602,7 @@ export class InferencePipeline extends PipelineState {
776
1602
  this.stats.decodeTimeMs = 0;
777
1603
  this.stats.gpuTimePrefillMs = undefined;
778
1604
  this.stats.gpuTimeDecodeMs = undefined;
1605
+ this.stats.prefillProfileSteps = [];
779
1606
  this.stats.decodeProfileSteps = [];
780
1607
  this.stats.executionPlan = null;
781
1608
  this.stats.kernelPathId = null;
@@ -783,16 +1610,37 @@ export class InferencePipeline extends PipelineState {
783
1610
  this.stats.attentionInputs = [];
784
1611
  }
785
1612
 
1613
+ /**
1614
+ * Lightweight reset for batch embedding: clears KV cache and sequence state
1615
+ * without resetting stats or debug flags (they are overwritten on each prefill).
1616
+ */
1617
+ resetForBatch() {
1618
+ this.kvCache?.clear();
1619
+ this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
1620
+ this.currentSeqLen = 0;
1621
+ this.decodeStepCount = 0;
1622
+ this.decodeBuffers?.resetPingPong();
1623
+ this.decodeRing?.reset();
1624
+ }
1625
+
786
1626
 
787
1627
  releaseGPUResources() {
788
1628
  this.decodeBuffers?.release();
789
1629
  this.decodeRing?.release();
790
1630
  destroyMoERouter(this.moeRouter);
791
1631
  this.moeRouter = null;
1632
+ destroyPleRuntimeCache(this.weights.get('per_layer_inputs'));
1633
+ destroyPleBufferCache(this.pleCache);
1634
+ this.pleCache = null;
1635
+ this.plePrefetchPending = null;
792
1636
  if (this.finitenessBuffer) {
793
1637
  this.finitenessBuffer.destroy();
794
1638
  this.finitenessBuffer = null;
795
1639
  }
1640
+ if (this.sampleReadbackBuffer) {
1641
+ this.sampleReadbackBuffer.destroy();
1642
+ this.sampleReadbackBuffer = null;
1643
+ }
796
1644
  }
797
1645
  }
798
1646
 
@@ -806,6 +1654,7 @@ async function createTransformerPipeline(manifest, contexts = {}) {
806
1654
  }
807
1655
 
808
1656
  registerPipeline('transformer', createTransformerPipeline);
1657
+ registerPipeline('gemma4', createTransformerPipeline);
809
1658
 
810
1659
  export class EmbeddingPipeline extends InferencePipeline {
811
1660
  async *generate() {