doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -0,0 +1,243 @@
1
+
2
+ import { log } from '../../../debug/index.js';
3
+ import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
4
+ import { resolveActiveExecutionPlan } from './execution-plan.js';
5
+ import { resolvePerLayerInputsSession } from './generator-helpers.js';
6
+ import {
7
+ ensurePleGpuHotVocabularyRuntime,
8
+ ensurePleGpuSplitTablesRuntime,
9
+ ensurePleScaledProjectionNormWeight,
10
+ } from './per-layer-inputs.js';
11
+
12
+ export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
13
+ return hitStop !== true
14
+ && Number.isInteger(actualCount)
15
+ && Number.isInteger(requestedCount)
16
+ && actualCount > 0
17
+ && actualCount < requestedCount;
18
+ }
19
+
20
+ export function resolveHotVocabularyBatchDecodeAvailability({
21
+ hasRangeBackedPerLayerInputs,
22
+ pleHotVocabularyRuntime,
23
+ tokenId,
24
+ }) {
25
+ if (hasRangeBackedPerLayerInputs !== true) {
26
+ return false;
27
+ }
28
+ if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
29
+ return false;
30
+ }
31
+ const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
32
+ if (!Number.isInteger(sentinelIndex)) {
33
+ return false;
34
+ }
35
+ if (!Number.isInteger(tokenId)) {
36
+ return false;
37
+ }
38
+ const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
39
+ return hotTokenIndex !== sentinelIndex;
40
+ }
41
+
42
+ export async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
43
+ const perLayerInputsSession = resolvePerLayerInputsSession(
44
+ state.modelConfig?.perLayerInputsSession ?? null,
45
+ state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
46
+ );
47
+ const materialization = perLayerInputsSession?.materialization;
48
+ if (state.debug) {
49
+ log.debug(
50
+ 'Pipeline',
51
+ `PLE materialization session=${materialization ?? 'null'} ` +
52
+ `modelId=${state.modelConfig?.modelId ?? 'unknown'} ` +
53
+ `runtimePerLayerInputs=${Boolean(state.runtimeConfig.inference.session?.perLayerInputs)} ` +
54
+ `manifestPerLayerInputs=${Boolean(state.modelConfig?.perLayerInputsSession)}`
55
+ );
56
+ }
57
+ await ensurePleGpuHotVocabularyRuntime({
58
+ config: state.modelConfig,
59
+ weights: state.weights,
60
+ perLayerInputsSession,
61
+ debugFlags: state.debugFlags,
62
+ tokenizer: state.tokenizer ?? null,
63
+ seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
64
+ });
65
+ await ensurePleGpuSplitTablesRuntime({
66
+ config: state.modelConfig,
67
+ weights: state.weights,
68
+ perLayerInputsSession,
69
+ debugFlags: state.debugFlags,
70
+ });
71
+ await ensurePleScaledProjectionNormWeight({
72
+ config: state.modelConfig,
73
+ weights: state.weights,
74
+ weightConfig: {
75
+ rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
76
+ },
77
+ debugFlags: state.debugFlags,
78
+ });
79
+ }
80
+
81
+ export function recordPrefillProfileStep(state, entry) {
82
+ if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
83
+ if (!state.stats.prefillProfileSteps) {
84
+ state.stats.prefillProfileSteps = [];
85
+ }
86
+ state.stats.prefillProfileSteps.push(entry);
87
+ }
88
+
89
+ /**
90
+ * Resolve display text for a token ID sequence.
91
+ *
92
+ * Preference order:
93
+ * 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
94
+ * 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
95
+ * 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
96
+ * unless primary returned empty string and fallback looks like a special token (<...>),
97
+ * in which case empty string is preserved to keep skip-special behavior deterministic.
98
+ * 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
99
+ * 5. fallbackText — static fallback string (default '?').
100
+ */
101
+ export function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
102
+ const renderPrimary = typeof renderTokenText === 'function'
103
+ ? renderTokenText
104
+ : (ids) => tokenizer?.decode?.(ids);
105
+ const renderFallback = typeof renderFallbackTokenText === 'function'
106
+ ? renderFallbackTokenText
107
+ : (ids) => tokenizer?.decode?.(ids, false);
108
+
109
+ const primaryText = renderPrimary(tokenIds);
110
+ if (typeof primaryText === 'string' && primaryText.length > 0) {
111
+ return primaryText;
112
+ }
113
+
114
+ const fallback = renderFallback(tokenIds);
115
+ if (typeof fallback === 'string' && fallback.length > 0) {
116
+ // Keep skip-special behavior deterministic: if primary decoding filtered this
117
+ // token to empty, do not reintroduce obvious special-token text via fallback.
118
+ if (
119
+ primaryText === ''
120
+ && /^<[^>\n]{1,80}>$/.test(fallback.trim())
121
+ ) {
122
+ return '';
123
+ }
124
+ return fallback;
125
+ }
126
+
127
+ return fallbackText;
128
+ }
129
+
130
+ export function usesReplayPrefillDecode(state) {
131
+ return state?.modelConfig?.decodeStrategy === 'replay_prefill';
132
+ }
133
+
134
+ export function assertIncrementalDecodeSupport(state, operation) {
135
+ if (!usesReplayPrefillDecode(state)) {
136
+ return;
137
+ }
138
+ throw new Error(
139
+ `[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
140
+ 'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
141
+ 'explicit layerTypes for mixed-geometry/shared-KV decode.'
142
+ );
143
+ }
144
+
145
+ export function summarizeExecutionPlan(plan) {
146
+ if (!plan) {
147
+ return null;
148
+ }
149
+ if (typeof plan !== 'object') {
150
+ log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
151
+ return null;
152
+ }
153
+ if (typeof plan.id !== 'string') {
154
+ log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
155
+ }
156
+ if (typeof plan.activationDtype !== 'string') {
157
+ log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
158
+ }
159
+ return {
160
+ id: plan.id,
161
+ kernelPathId: plan.kernelPathId ?? null,
162
+ kernelPathSource: plan.kernelPathSource ?? 'none',
163
+ activationDtype: plan.activationDtype,
164
+ readbackInterval: plan.readbackInterval ?? null,
165
+ batchSize: plan.defaultBatchSize,
166
+ stopCheckMode: plan.defaultStopCheckMode,
167
+ ringTokens: plan.ringTokens ?? null,
168
+ ringStop: plan.ringStop ?? null,
169
+ ringStaging: plan.ringStaging ?? null,
170
+ };
171
+ }
172
+
173
+ export function shouldRetryWithFinitenessFallback(error) {
174
+ if (error?.name === 'FinitenessError') {
175
+ return true;
176
+ }
177
+ const message = typeof error?.message === 'string'
178
+ ? error.message
179
+ : (typeof error === 'string' ? error : '');
180
+ if (!message.startsWith('[Sampling]')) {
181
+ return false;
182
+ }
183
+ return message.includes('no finite candidate logits after masking the pad token')
184
+ || message.includes('Softmax produced no finite candidate probabilities');
185
+ }
186
+
187
+ export function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
188
+ const activePlan = resolveActiveExecutionPlan(state);
189
+ const wrapped = new Error(
190
+ `[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
191
+ `"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
192
+ 'Resolve the unstable path with an explicit capability-aware execution override, ' +
193
+ 'or opt into alternate-plan recovery with ' +
194
+ 'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
195
+ error instanceof Error ? { cause: error } : undefined
196
+ );
197
+ wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
198
+ return wrapped;
199
+ }
200
+
201
+ function assertResolvedKVDtype(kvDtype, contextLabel) {
202
+ if (kvDtype === 'f16' || kvDtype === 'f32') {
203
+ return kvDtype;
204
+ }
205
+ throw new Error(
206
+ `[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
207
+ `got ${JSON.stringify(kvDtype)}.`
208
+ );
209
+ }
210
+
211
+ export function resolveTargetPlanKVDtype(plan, contextLabel) {
212
+ return assertResolvedKVDtype(
213
+ plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
214
+ contextLabel
215
+ );
216
+ }
217
+
218
+ export function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
219
+ return assertResolvedKVDtype(
220
+ state?.kvCache?.kvDtype
221
+ ?? plan?.kernelPath?.kvDtype
222
+ ?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
223
+ ?? plan?.activationDtype
224
+ ?? null,
225
+ contextLabel
226
+ );
227
+ }
228
+
229
+ export function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
230
+ const runtimeInference = state?.runtimeConfig?.inference;
231
+ if (!runtimeInference?.session?.kvcache) {
232
+ throw new Error(
233
+ '[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
234
+ );
235
+ }
236
+ return mergeRuntimeValues(runtimeInference, {
237
+ session: {
238
+ kvcache: {
239
+ kvDtype,
240
+ },
241
+ },
242
+ });
243
+ }
@@ -9,6 +9,7 @@ import type { LayerContext } from './types.js';
9
9
  import type { LogitsConfig, LogitsWeights } from './logits/index.js';
10
10
  import type { WeightBufferConfig } from './weights.js';
11
11
  import type { ExecutionSessionPlan } from './execution-plan.js';
12
+ import type { ExecutionV1PerLayerInputsSessionSchema } from '../../../config/schema/execution-v1.schema.js';
12
13
 
13
14
  export declare function debugCheckBuffer(
14
15
  state: PipelineState,
@@ -27,6 +28,11 @@ export declare function buildLayerContext(
27
28
  executionPlan?: ExecutionSessionPlan | null
28
29
  ): LayerContext;
29
30
 
31
+ export declare function resolvePerLayerInputsSession(
32
+ manifestSession: ExecutionV1PerLayerInputsSessionSchema | null | undefined,
33
+ runtimeSession: ExecutionV1PerLayerInputsSessionSchema | Record<string, unknown> | null | undefined
34
+ ): ExecutionV1PerLayerInputsSessionSchema | null;
35
+
30
36
  export declare function releaseSharedAttentionState(
31
37
  sharedAttentionState: LayerContext['sharedAttentionState'],
32
38
  recorder?: {
@@ -34,8 +40,6 @@ export declare function releaseSharedAttentionState(
34
40
  } | null
35
41
  ): void;
36
42
 
37
- export declare function getWeightBufferConfig(state: PipelineState): WeightBufferConfig;
38
-
39
43
  export declare function getLogitsWeights(state: PipelineState): LogitsWeights;
40
44
 
41
45
  export declare function getLogitsConfig(state: PipelineState): LogitsConfig;
@@ -2,11 +2,23 @@
2
2
  import { readBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
3
3
  import { allowReadback } from '../../../gpu/perf-guards.js';
4
4
  import { log } from '../../../debug/index.js';
5
+ import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
5
6
  import { selectRuleValue } from '../../../rules/rule-registry.js';
6
7
  import { decodeReadback } from './debug-utils/index.js';
7
8
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
8
9
  import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
9
10
  import { resolveActiveExecutionPlan } from './execution-plan.js';
11
+ import { isPlainObject } from '../../../utils/plain-object.js';
12
+
13
+ export function resolvePerLayerInputsSession(manifestSession, runtimeSession) {
14
+ if (!isPlainObject(runtimeSession)) {
15
+ return runtimeSession ?? (manifestSession ?? null);
16
+ }
17
+ if (!isPlainObject(manifestSession)) {
18
+ return runtimeSession;
19
+ }
20
+ return mergeRuntimeValues(manifestSession, runtimeSession);
21
+ }
10
22
  export async function debugCheckBuffer(state, buffer, label, numTokens, expectedDim) {
11
23
  if (!allowReadback(`pipeline.debug.${label}`)) return;
12
24
 
@@ -83,6 +95,10 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
83
95
  const activeExecutionPlan = executionPlan ?? resolveActiveExecutionPlan(state);
84
96
  const activeKernelPath = activeExecutionPlan.kernelPath ?? state.resolvedKernelPath ?? null;
85
97
  const effectiveActivationDtype = activeExecutionPlan.activationDtype;
98
+ const perLayerInputsSession = resolvePerLayerInputsSession(
99
+ config.perLayerInputsSession ?? null,
100
+ state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
101
+ );
86
102
  const effectiveComputeConfig = {
87
103
  ...computeConfig,
88
104
  activationDtype: effectiveActivationDtype,
@@ -122,7 +138,7 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
122
138
  debugProbes: state.runtimeConfig.shared.debug.probes,
123
139
  debugCheckBuffer: debugCheckBufferFn,
124
140
  perLayerInputBuffer: null,
125
- perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? config.perLayerInputsSession,
141
+ perLayerInputsSession,
126
142
  pipelinePlan: state.layerPipelinePlan,
127
143
  expertWeights: state.expertWeights,
128
144
  expertLoader: state.dopplerLoader,
@@ -142,6 +158,7 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
142
158
  step: state.decodeStepCount,
143
159
  phase: isDecodeMode ? 'decode' : 'prefill',
144
160
  operatorDiagnostics: state.operatorDiagnostics,
161
+ useFusedGateUpGelu: state.runtimeConfig?.inference?.session?.useFusedGateUpGelu === true,
145
162
  };
146
163
  }
147
164
 
@@ -172,7 +189,7 @@ export function releaseSharedAttentionState(sharedAttentionState, recorder = nul
172
189
  }
173
190
 
174
191
 
175
- export function getWeightBufferConfig(state) {
192
+ function getWeightBufferConfig(state) {
176
193
  return {
177
194
  rmsNormWeightOffset: state.modelConfig.rmsNormWeightOffset,
178
195
  };
@@ -209,5 +226,6 @@ export function getLogitsConfig(state) {
209
226
  activationDtype: effectiveActivationDtype,
210
227
  kernelPath: activeKernelPath,
211
228
  executionPolicies: state.executionV1State?.policies ?? null,
229
+ debugProbes: state.runtimeConfig.shared.debug.probes,
212
230
  };
213
231
  }
@@ -0,0 +1,293 @@
1
+
2
+ import { getDevice } from '../../../gpu/device.js';
3
+ import { acquireBuffer, releaseBuffer, uploadData } from '../../../memory/buffer-pool.js';
4
+ import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
5
+ import { f32ToF16Array } from '../../kv-cache/types.js';
6
+ import { isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
7
+ import { createTensor } from '../../../gpu/tensor.js';
8
+ import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
9
+ import { applyChatTemplate } from './init.js';
10
+ import { formatChatMessages } from './chat-format.js';
11
+
12
+ function isStructuredChatRequest(prompt) {
13
+ return prompt != null
14
+ && typeof prompt === 'object'
15
+ && !Array.isArray(prompt)
16
+ && Array.isArray(prompt.messages);
17
+ }
18
+
19
+ export function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
20
+ const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
21
+ if (typeof prompt === 'string') {
22
+ if (useChatTemplate && state.modelConfig.chatTemplateType) {
23
+ if (state.modelConfig.chatTemplateType === 'translategemma') {
24
+ throw new Error(
25
+ `[Pipeline] ${contextLabel}: translategemma chat template requires structured messages. ` +
26
+ 'Pass { messages: [...] } instead of a plain string prompt.'
27
+ );
28
+ }
29
+ return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
30
+ }
31
+ return prompt;
32
+ }
33
+
34
+ if (prompt != null && typeof prompt === 'object' && !Array.isArray(prompt) && 'messages' in prompt && !Array.isArray(prompt.messages)) {
35
+ throw new Error(
36
+ `[Pipeline] ${contextLabel}: prompt.messages must be an array of chat messages, got ${typeof prompt.messages}. ` +
37
+ 'Pass { messages: [{ role: "user", content: "..." }, ...] }.'
38
+ );
39
+ }
40
+ const messages = isStructuredChatRequest(prompt)
41
+ ? prompt.messages
42
+ : (Array.isArray(prompt) ? prompt : null);
43
+ if (!messages) {
44
+ throw new Error(
45
+ `[Pipeline] ${contextLabel}: prompt must be a string, chat message array, or { messages: [...] }.`
46
+ );
47
+ }
48
+ const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
49
+ return formatChatMessages(messages, templateType, chatOptions);
50
+ }
51
+
52
+ export function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
53
+ if (!buffer) {
54
+ return;
55
+ }
56
+ const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
57
+ if (ownsBuffer) {
58
+ return;
59
+ }
60
+ const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
61
+ if (cachedPleBuffer) {
62
+ return;
63
+ }
64
+ if (recorder) {
65
+ recorder.trackTemporaryBuffer(buffer);
66
+ return;
67
+ }
68
+ releaseBuffer(buffer);
69
+ }
70
+
71
+ export function normalizePrefixEmbeddingOverride(override, hiddenSize, numTokens, contextLabel) {
72
+ if (override == null) {
73
+ return null;
74
+ }
75
+ if (typeof override !== 'object') {
76
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
77
+ }
78
+
79
+ const prefixLength = Number(override.prefixLength);
80
+ if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
81
+ throw new Error(
82
+ `[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
83
+ );
84
+ }
85
+ if (prefixLength === 0) {
86
+ return null;
87
+ }
88
+
89
+ const offset = Number(override.offset ?? 0);
90
+ if (!Number.isFinite(offset) || offset < 0 || Math.floor(offset) !== offset) {
91
+ throw new Error(
92
+ `[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
93
+ );
94
+ }
95
+ if (offset + prefixLength > numTokens) {
96
+ throw new Error(
97
+ `[Pipeline] ${contextLabel}: embedding override offset=${offset} + prefixLength=${prefixLength} exceeds numTokens=${numTokens}.`
98
+ );
99
+ }
100
+
101
+ const embeddings = override.embeddings ?? null;
102
+ if (!embeddings) {
103
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings is required when prefixLength > 0.`);
104
+ }
105
+
106
+ const expectedLength = prefixLength * hiddenSize;
107
+ if (embeddings instanceof Float32Array) {
108
+ if (embeddings.length !== expectedLength) {
109
+ throw new Error(
110
+ `[Pipeline] ${contextLabel}: embedding override length mismatch ` +
111
+ `(expected=${expectedLength}, got=${embeddings.length}).`
112
+ );
113
+ }
114
+ } else if (isGpuBufferInstance(embeddings)) {
115
+ const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
116
+ if (embeddings.size < expectedBytes) {
117
+ throw new Error(
118
+ `[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
119
+ `(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
120
+ );
121
+ }
122
+ } else {
123
+ throw new Error(
124
+ `[Pipeline] ${contextLabel}: embeddingOverrides.embeddings must be a Float32Array or GPUBuffer.`
125
+ );
126
+ }
127
+
128
+ return {
129
+ prefixLength,
130
+ offset,
131
+ embeddings,
132
+ expectedLength,
133
+ byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
134
+ byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
135
+ };
136
+ }
137
+
138
+ export function resolvePrefillEmbeddingInputIds(inputIds, embeddingInputSpan, contextLabel) {
139
+ if (embeddingInputSpan == null) {
140
+ return inputIds;
141
+ }
142
+ if (typeof embeddingInputSpan !== 'object') {
143
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
144
+ }
145
+
146
+ const offset = Number(embeddingInputSpan.offset);
147
+ const length = Number(embeddingInputSpan.length);
148
+ const tokenId = Number(embeddingInputSpan.tokenId);
149
+ if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
150
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.offset must be a non-negative integer.`);
151
+ }
152
+ if (!Number.isFinite(length) || Math.floor(length) !== length || length < 0) {
153
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.length must be a non-negative integer.`);
154
+ }
155
+ if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
156
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.tokenId must be a non-negative integer.`);
157
+ }
158
+ if (offset + length > inputIds.length) {
159
+ throw new Error(
160
+ `[Pipeline] ${contextLabel}: embeddingInputSpan offset=${offset} + length=${length} ` +
161
+ `exceeds inputIds length ${inputIds.length}.`
162
+ );
163
+ }
164
+ const replacedInputIds = Array.from(inputIds);
165
+ replacedInputIds.fill(tokenId, offset, offset + length);
166
+ return replacedInputIds;
167
+ }
168
+
169
+ export function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectionalSpan, contextLabel) {
170
+ if (bidirectionalSpan == null) {
171
+ return null;
172
+ }
173
+ if (typeof bidirectionalSpan !== 'object') {
174
+ throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
175
+ }
176
+
177
+ const offset = Number(bidirectionalSpan.offset);
178
+ const length = Number(bidirectionalSpan.length);
179
+ if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
180
+ throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
181
+ }
182
+ if (!Number.isFinite(length) || Math.floor(length) !== length || length < 1) {
183
+ throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.length must be a positive integer.`);
184
+ }
185
+ if ((offset + length) > inputIds.length) {
186
+ throw new Error(
187
+ `[Pipeline] ${contextLabel}: multimodalBidirectionalSpan offset=${offset} + length=${length} ` +
188
+ `exceeds inputIds length ${inputIds.length}.`
189
+ );
190
+ }
191
+ return { offset, length };
192
+ }
193
+
194
+ export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
195
+ if (!override) {
196
+ return baseTensor;
197
+ }
198
+ if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
199
+ throw new Error(
200
+ `[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
201
+ );
202
+ }
203
+
204
+ const device = getDevice();
205
+ if (!device) {
206
+ throw new Error(`[Pipeline] ${contextLabel}: GPU device is required for embedding overrides.`);
207
+ }
208
+
209
+ const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
210
+ const targetElementOffset = override.offset * hiddenSize;
211
+ const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
212
+ try {
213
+ const encoder = device.createCommandEncoder();
214
+ encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
215
+ if (baseTensor.dtype === 'f32') {
216
+ if (isGpuBufferInstance(override.embeddings)) {
217
+ encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
218
+ device.queue.submit([encoder.finish()]);
219
+ } else {
220
+ device.queue.submit([encoder.finish()]);
221
+ uploadData(outputBuffer, override.embeddings, override.byteOffset);
222
+ }
223
+ return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
224
+ }
225
+
226
+ if (isGpuBufferInstance(override.embeddings)) {
227
+ const overrideTensor = createTensor(
228
+ override.embeddings,
229
+ 'f32',
230
+ [override.prefixLength, hiddenSize],
231
+ 'prefill_embedding_override_f32'
232
+ );
233
+ assertImplicitDtypeTransitionAllowed({
234
+ executionPolicies,
235
+ fromDtype: 'f32',
236
+ toDtype: 'f16',
237
+ op: 'prefill_embedding_override',
238
+ detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
239
+ });
240
+ const castedOverride = await castF32ToF16(overrideTensor);
241
+ try {
242
+ encoder.copyBufferToBuffer(
243
+ castedOverride.buffer,
244
+ 0,
245
+ outputBuffer,
246
+ targetByteOffset,
247
+ castedOverride.buffer.size
248
+ );
249
+ device.queue.submit([encoder.finish()]);
250
+ } finally {
251
+ releaseBuffer(castedOverride.buffer);
252
+ }
253
+ } else {
254
+ assertImplicitDtypeTransitionAllowed({
255
+ executionPolicies,
256
+ fromDtype: 'f32',
257
+ toDtype: 'f16',
258
+ op: 'prefill_embedding_override',
259
+ detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
260
+ });
261
+ const packedOverride = f32ToF16Array(override.embeddings);
262
+ device.queue.submit([encoder.finish()]);
263
+ uploadData(outputBuffer, packedOverride, targetByteOffset);
264
+ }
265
+ return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
266
+ } catch (error) {
267
+ releaseBuffer(outputBuffer);
268
+ throw error;
269
+ }
270
+ }
271
+
272
+ export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
273
+ if (
274
+ opts?.disableCommandBatching === true
275
+ || opts?.debug === true
276
+ || (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
277
+ ) {
278
+ return true;
279
+ }
280
+ if (state?.kvCache?.layout === 'bdpa_paged') {
281
+ return true;
282
+ }
283
+ if (multimodalBidirectionalSpan == null) {
284
+ return false;
285
+ }
286
+ if (state?.kvCache?.hasGPUCache?.() !== true) {
287
+ return false;
288
+ }
289
+ // WORKAROUND: recorded prefill with live KV-cache writes regresses Gemma 4
290
+ // multimodal logits on the first generated token. Keep this lane fail-closed
291
+ // until the recorder/cache interaction is numerically matched to the direct path.
292
+ return true;
293
+ }
@@ -75,13 +75,6 @@ export function resolveFloatDtypeFromByteSize(
75
75
  fallback?: 'f16' | 'f32'
76
76
  ): 'f16' | 'f32';
77
77
 
78
- export function decodeFloatWeights(
79
- data: ArrayBuffer,
80
- dtype: 'f16' | 'f32' | 'bf16',
81
- expectedLength: number,
82
- label: string
83
- ): Float32Array;
84
-
85
78
  export function getFinalNormWeights(state: PipelineState): Promise<Float32Array>;
86
79
 
87
80
  export function extractEmbeddingFromHidden(
@@ -297,11 +297,13 @@ export function resolvePrefillEmbeddingOptions(state, options = {}) {
297
297
  ? state.manifest.modelType.toLowerCase()
298
298
  : '';
299
299
  const generationDefaults = state.runtimeConfig.inference.generation;
300
- // Embedding models default to 'mean' pooling — this is a model-category behavior,
301
- // not a model-family identity check. Ideally embedding conversion configs would set
302
- // generation.embeddingMode='mean' in their runtime config; the modelType fallback
303
- // provides this default for manifests that predate runtime-profile embedding mode.
304
- const defaultEmbeddingMode = modelType === 'embedding'
300
+ // Models that expose embedding extraction default to 'mean' pooling. This covers
301
+ // dedicated embedding models (modelType="embedding") and text-generation models
302
+ // that opt in via inference.supportsEmbedding=true. Conversion configs can still
303
+ // override via generation.embeddingMode in their runtime profile.
304
+ const supportsEmbeddingExtraction = modelType === 'embedding'
305
+ || state.manifest?.inference?.supportsEmbedding === true;
306
+ const defaultEmbeddingMode = supportsEmbeddingExtraction
305
307
  ? 'mean'
306
308
  : generationDefaults.embeddingMode;
307
309
  return {
@@ -349,7 +351,7 @@ export function resolveFloatDtypeFromByteSize(totalBytes, expectedLength) {
349
351
  });
350
352
  }
351
353
 
352
- export function decodeFloatWeights(data, dtype, expectedLength, label) {
354
+ function decodeFloatWeights(data, dtype, expectedLength, label) {
353
355
  const decodeDtype = resolveFloatDtypeFromAlias(dtype);
354
356
  const decoded = decodeReadback(data, decodeDtype);
355
357
  if (decoded.length !== expectedLength) {