doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -0,0 +1,171 @@
1
+ /**
2
+ * Mel spectrogram extraction for audio encoder preprocessing.
3
+ *
4
+ * Converts raw PCM audio (Float32Array, mono, 16kHz) into a log-mel spectrogram
5
+ * suitable for Gemma 4's conformer audio tower.
6
+ *
7
+ * All operations are CPU-only (no GPU dependency).
8
+ */
9
+
10
+ const DEFAULT_SAMPLE_RATE = 16000;
11
+ const DEFAULT_N_FFT = 512;
12
+ const DEFAULT_HOP_LENGTH = 160;
13
+ const DEFAULT_N_MELS = 80;
14
+ const DEFAULT_WINDOW_LENGTH = 400;
15
+
16
+ /**
17
+ * Hann window of given length.
18
+ * @param {number} length
19
+ * @returns {Float32Array}
20
+ */
21
+ function hannWindow(length) {
22
+ const window = new Float32Array(length);
23
+ for (let i = 0; i < length; i++) {
24
+ window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (length - 1)));
25
+ }
26
+ return window;
27
+ }
28
+
29
+ /**
30
+ * Compute power spectrum via real FFT (Cooley-Tukey radix-2).
31
+ * @param {Float32Array} frame Windowed audio frame of length nFft
32
+ * @param {number} nFft
33
+ * @returns {Float32Array} Power spectrum of length nFft/2 + 1
34
+ */
35
+ function powerSpectrum(frame, nFft) {
36
+ const real = new Float32Array(nFft);
37
+ const imag = new Float32Array(nFft);
38
+ real.set(frame.subarray(0, Math.min(frame.length, nFft)));
39
+
40
+ // In-place Cooley-Tukey FFT
41
+ const bits = Math.log2(nFft);
42
+ for (let i = 0; i < nFft; i++) {
43
+ let j = 0;
44
+ for (let b = 0; b < bits; b++) {
45
+ j = (j << 1) | ((i >> b) & 1);
46
+ }
47
+ if (j > i) {
48
+ let tmp = real[i]; real[i] = real[j]; real[j] = tmp;
49
+ tmp = imag[i]; imag[i] = imag[j]; imag[j] = tmp;
50
+ }
51
+ }
52
+
53
+ for (let size = 2; size <= nFft; size *= 2) {
54
+ const halfSize = size / 2;
55
+ const angle = -2 * Math.PI / size;
56
+ for (let i = 0; i < nFft; i += size) {
57
+ for (let j = 0; j < halfSize; j++) {
58
+ const cos = Math.cos(angle * j);
59
+ const sin = Math.sin(angle * j);
60
+ const tpRe = real[i + j + halfSize] * cos - imag[i + j + halfSize] * sin;
61
+ const tpIm = real[i + j + halfSize] * sin + imag[i + j + halfSize] * cos;
62
+ real[i + j + halfSize] = real[i + j] - tpRe;
63
+ imag[i + j + halfSize] = imag[i + j] - tpIm;
64
+ real[i + j] += tpRe;
65
+ imag[i + j] += tpIm;
66
+ }
67
+ }
68
+ }
69
+
70
+ const numBins = nFft / 2 + 1;
71
+ const power = new Float32Array(numBins);
72
+ for (let i = 0; i < numBins; i++) {
73
+ power[i] = real[i] * real[i] + imag[i] * imag[i];
74
+ }
75
+ return power;
76
+ }
77
+
78
+ /**
79
+ * Build mel filter bank matrix.
80
+ * @param {number} nMels Number of mel bands
81
+ * @param {number} nFft FFT size
82
+ * @param {number} sampleRate
83
+ * @returns {Float32Array} [nMels * (nFft/2 + 1)] row-major filter bank
84
+ */
85
+ function melFilterBank(nMels, nFft, sampleRate) {
86
+ const numBins = nFft / 2 + 1;
87
+ const hzToMel = (hz) => 2595 * Math.log10(1 + hz / 700);
88
+ const melToHz = (mel) => 700 * (10 ** (mel / 2595) - 1);
89
+
90
+ const melLow = hzToMel(0);
91
+ const melHigh = hzToMel(sampleRate / 2);
92
+ const melPoints = new Float32Array(nMels + 2);
93
+ for (let i = 0; i < nMels + 2; i++) {
94
+ melPoints[i] = melToHz(melLow + (melHigh - melLow) * i / (nMels + 1));
95
+ }
96
+
97
+ const binFreqs = new Float32Array(numBins);
98
+ for (let i = 0; i < numBins; i++) {
99
+ binFreqs[i] = (sampleRate / nFft) * i;
100
+ }
101
+
102
+ const filters = new Float32Array(nMels * numBins);
103
+ for (let m = 0; m < nMels; m++) {
104
+ const fLow = melPoints[m];
105
+ const fCenter = melPoints[m + 1];
106
+ const fHigh = melPoints[m + 2];
107
+ for (let k = 0; k < numBins; k++) {
108
+ const freq = binFreqs[k];
109
+ if (freq >= fLow && freq <= fCenter && fCenter > fLow) {
110
+ filters[m * numBins + k] = (freq - fLow) / (fCenter - fLow);
111
+ } else if (freq > fCenter && freq <= fHigh && fHigh > fCenter) {
112
+ filters[m * numBins + k] = (fHigh - freq) / (fHigh - fCenter);
113
+ }
114
+ }
115
+ }
116
+ return filters;
117
+ }
118
+
119
+ /**
120
+ * Extract log-mel spectrogram from raw audio PCM.
121
+ *
122
+ * @param {Float32Array} audio Mono audio samples (16kHz expected)
123
+ * @param {object} [opts]
124
+ * @param {number} [opts.sampleRate=16000]
125
+ * @param {number} [opts.nFft=512]
126
+ * @param {number} [opts.hopLength=160]
127
+ * @param {number} [opts.nMels=80]
128
+ * @param {number} [opts.windowLength=400]
129
+ * @returns {{ features: Float32Array, numFrames: number, nMels: number }}
130
+ */
131
+ export function extractLogMelSpectrogram(audio, opts = {}) {
132
+ const sampleRate = opts.sampleRate ?? DEFAULT_SAMPLE_RATE;
133
+ const nFft = opts.nFft ?? DEFAULT_N_FFT;
134
+ const hopLength = opts.hopLength ?? DEFAULT_HOP_LENGTH;
135
+ const nMels = opts.nMels ?? DEFAULT_N_MELS;
136
+ const windowLength = opts.windowLength ?? DEFAULT_WINDOW_LENGTH;
137
+
138
+ if (!(audio instanceof Float32Array) || audio.length === 0) {
139
+ throw new Error('[Audio] extractLogMelSpectrogram requires a non-empty Float32Array of audio samples.');
140
+ }
141
+
142
+ const window = hannWindow(windowLength);
143
+ const numBins = nFft / 2 + 1;
144
+ const filters = melFilterBank(nMels, nFft, sampleRate);
145
+
146
+ const numFrames = Math.max(1, Math.floor((audio.length - windowLength) / hopLength) + 1);
147
+ const features = new Float32Array(numFrames * nMels);
148
+ const paddedFrame = new Float32Array(nFft);
149
+
150
+ for (let t = 0; t < numFrames; t++) {
151
+ const offset = t * hopLength;
152
+ paddedFrame.fill(0);
153
+ for (let i = 0; i < windowLength; i++) {
154
+ const sampleIdx = offset + i;
155
+ paddedFrame[i] = sampleIdx < audio.length ? audio[sampleIdx] * window[i] : 0;
156
+ }
157
+
158
+ const power = powerSpectrum(paddedFrame, nFft);
159
+
160
+ for (let m = 0; m < nMels; m++) {
161
+ let melEnergy = 0;
162
+ const filterOffset = m * numBins;
163
+ for (let k = 0; k < numBins; k++) {
164
+ melEnergy += filters[filterOffset + k] * power[k];
165
+ }
166
+ features[t * nMels + m] = Math.log(Math.max(melEnergy, 1e-10));
167
+ }
168
+ }
169
+
170
+ return { features, numFrames, nMels };
171
+ }
@@ -121,7 +121,15 @@ export function applyPipelineContexts(target, contexts = {}, options = {}) {
121
121
 
122
122
  if (contexts.gpu?.device) {
123
123
  const device = contexts.gpu.device;
124
- setDevice(device);
124
+ const isRebindingCurrentDevice = previousDevice != null && device === previousDevice;
125
+ const nextAdapterInfo = contexts.gpu.adapterInfo
126
+ ?? (isRebindingCurrentDevice ? previousAdapterInfo : undefined);
127
+ const nextPlatformConfig = contexts.gpu.platformConfig
128
+ ?? (isRebindingCurrentDevice ? previousPlatformConfig : undefined);
129
+ setDevice(device, {
130
+ ...(nextAdapterInfo ? { adapterInfo: nextAdapterInfo } : {}),
131
+ ...(nextPlatformConfig !== undefined ? { platformConfig: nextPlatformConfig } : {}),
132
+ });
125
133
  setGPUDevice(device);
126
134
  if (options.assignGpuContext) {
127
135
  target.gpuContext = { device };
@@ -871,6 +871,8 @@ async function runGemma2TextEncoder(tokens, weightsEntry, config, runtime, optio
871
871
 
872
872
  const ropeFreqs = await initRoPEFrequencies({
873
873
  headDim: resolved.headDim,
874
+ ropeFrequencyBaseDim: resolved.headDim,
875
+ ropeLocalFrequencyBaseDim: resolved.headDim,
874
876
  maxSeqLen: resolved.maxPositionEmbeddings,
875
877
  ropeTheta: resolved.ropeTheta,
876
878
  ropeLocalTheta: null,
@@ -0,0 +1,46 @@
1
+ import { getDevice } from '../../../gpu/device.js';
2
+ import { createTensor } from '../../../gpu/tensor.js';
3
+ import { runClamp } from '../../../gpu/kernels/clamp.js';
4
+ import { runMatmul } from '../../../gpu/kernel-selector.js';
5
+ import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
6
+
7
+ export function shouldClamp(minValue, maxValue) {
8
+ return Number.isFinite(minValue) || Number.isFinite(maxValue);
9
+ }
10
+
11
+ async function cloneTensorBuffer(tensor, label) {
12
+ const device = getDevice();
13
+ const byteLength = tensor.buffer.size;
14
+ const buffer = acquireBuffer(byteLength, GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC, label);
15
+ try {
16
+ const encoder = device.createCommandEncoder();
17
+ encoder.copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
18
+ device.queue.submit([encoder.finish()]);
19
+ return createTensor(buffer, tensor.dtype, [...tensor.shape], label);
20
+ } catch (error) {
21
+ releaseBuffer(buffer);
22
+ throw error;
23
+ }
24
+ }
25
+
26
+ export async function runClippableLinear(inputTensor, weight, M, N, K, clip, label) {
27
+ let matmulInput = inputTensor;
28
+ try {
29
+ if (shouldClamp(clip?.inputMin, clip?.inputMax)) {
30
+ matmulInput = await cloneTensorBuffer(inputTensor, `${label}_input`);
31
+ await runClamp(matmulInput, clip.inputMin, clip.inputMax, { count: M * K });
32
+ }
33
+ const output = await runMatmul(matmulInput, weight, M, N, K, {
34
+ outputDtype: 'f32',
35
+ transposeB: 'auto',
36
+ });
37
+ if (shouldClamp(clip?.outputMin, clip?.outputMax)) {
38
+ await runClamp(output, clip.outputMin, clip.outputMax, { count: M * N });
39
+ }
40
+ return output;
41
+ } finally {
42
+ if (matmulInput !== inputTensor) {
43
+ releaseBuffer(matmulInput.buffer);
44
+ }
45
+ }
46
+ }
@@ -22,7 +22,7 @@ export interface KVCacheDispatchState {
22
22
  causalForAttention: boolean;
23
23
  startPosForMask: number;
24
24
  kvStart: number;
25
- kvLayout: 'contiguous' | 'ring' | 'paged' | 'tiered' | 'bdpa';
25
+ kvLayout: 'contiguous' | 'contiguous_quantized' | 'ring' | 'paged' | 'tiered' | 'bdpa';
26
26
  kvPageTable: GPUBuffer | null;
27
27
  kvPageSize: number;
28
28
  cachedKHot: GPUBuffer | undefined;
@@ -81,6 +81,7 @@ export interface AttentionDispatchParams {
81
81
  cachedKTensor: Tensor | null;
82
82
  cachedVTensor: Tensor | null;
83
83
  isTieredKernel: boolean;
84
+ prefillFallbackNeedsCast: boolean;
84
85
  causalForAttention: boolean;
85
86
  }
86
87
 
@@ -87,6 +87,35 @@ export function resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSe
87
87
  kvState.coldPageSize = gpuBuffers.coldPageSize ?? state.kvCache.coldPageSize ?? 0;
88
88
  kvState.kvLenForAttention = kvState.coldLen + kvState.hotLen;
89
89
  kvState.kvLayout = 'tiered';
90
+ // TurboQuant shared buffers
91
+ kvState.rotationMatrixBuffer = gpuBuffers.rotationMatrixBuffer ?? null;
92
+ kvState.codebookCentroidsBuffer = gpuBuffers.codebookCentroidsBuffer ?? null;
93
+ // TurboQuant prod buffers
94
+ kvState.residualKGPU = gpuBuffers.residualKGPU ?? null;
95
+ kvState.residualVGPU = gpuBuffers.residualVGPU ?? null;
96
+ kvState.residualNormsKGPU = gpuBuffers.residualNormsKGPU ?? null;
97
+ kvState.residualNormsVGPU = gpuBuffers.residualNormsVGPU ?? null;
98
+ kvState.qjlMatrixBuffer = gpuBuffers.qjlMatrixBuffer ?? null;
99
+ kvState.residualPackedStride = gpuBuffers.residualPackedStride ?? 0;
100
+ } else if (gpuBuffers?.layout === 'contiguous_quantized') {
101
+ kvState.kvLayout = 'contiguous_quantized';
102
+ kvState.kvLenForAttention = gpuBuffers.seqLen;
103
+ kvState.cachedKCold = gpuBuffers.keysPackedGPU;
104
+ kvState.cachedVCold = gpuBuffers.valuesPackedGPU;
105
+ kvState.coldScalesK = gpuBuffers.scalesKGPU ?? null;
106
+ kvState.coldScalesV = gpuBuffers.scalesVGPU ?? null;
107
+ kvState.coldPackedStride = gpuBuffers.packedStride ?? 0;
108
+ kvState.coldQuantMode = gpuBuffers.quantMode ?? 'turboquant';
109
+ kvState.rotationMatrixBuffer = gpuBuffers.rotationMatrixBuffer ?? null;
110
+ kvState.codebookCentroidsBuffer = gpuBuffers.codebookCentroidsBuffer ?? null;
111
+ // Prod-mode buffers
112
+ kvState.residualKGPU = gpuBuffers.residualKGPU ?? null;
113
+ kvState.residualVGPU = gpuBuffers.residualVGPU ?? null;
114
+ kvState.residualNormsKGPU = gpuBuffers.residualNormsKGPU ?? null;
115
+ kvState.residualNormsVGPU = gpuBuffers.residualNormsVGPU ?? null;
116
+ kvState.qjlMatrixBuffer = gpuBuffers.qjlMatrixBuffer ?? null;
117
+ kvState.residualPackedStride = gpuBuffers.residualPackedStride ?? 0;
118
+ kvState.prodMode = gpuBuffers.prodMode === true;
90
119
  } else if (gpuBuffers?.layout === 'bdpa') {
91
120
  kvState.kvLayout = 'bdpa';
92
121
  kvState.kvLenForAttention = gpuBuffers.seqLen;
@@ -102,7 +131,7 @@ export function resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSe
102
131
  kvState.kvLenForAttention = gpuBuffers.seqLen;
103
132
  kvState.kvPageTable = gpuBuffers.pageTableGPU ?? null;
104
133
  kvState.kvPageSize = gpuBuffers.pageSize ?? state.kvCache.pageSize ?? 0;
105
- if (state.kvCache instanceof SlidingWindowKVCache) {
134
+ if (gpuBuffers?.layout === 'ring' || state.kvCache instanceof SlidingWindowKVCache) {
106
135
  kvState.kvLayout = 'ring';
107
136
  } else if (state.kvCache.layout === 'paged') {
108
137
  kvState.kvLayout = 'paged';
@@ -120,12 +149,12 @@ export function buildAttentionDispatchParams(config, state, kTensor, vTensor, kv
120
149
  const {
121
150
  numTokens, slidingWindow, layerType, headDim, queryPreAttnScalar, numKVHeads,
122
151
  } = config;
152
+ const resolvedKvCacheDtype = config.kvCacheDtype ?? state.kvCache?.kvDtype ?? null;
123
153
 
124
154
  // Tiered prefill fallback: tiered layout does not support prefill (numTokens > 1)
155
+ let prefillFallbackNeedsCast = false;
125
156
  if (kvState.kvLayout === 'tiered' && numTokens > 1) {
126
157
  kvState.kvLayout = 'contiguous';
127
- kvState.cachedK = kTensor.buffer;
128
- kvState.cachedV = vTensor.buffer;
129
158
  kvState.kvLenForAttention = numTokens;
130
159
  kvState.startPosForMask = 0;
131
160
  kvState.cachedKHot = null;
@@ -133,6 +162,18 @@ export function buildAttentionDispatchParams(config, state, kTensor, vTensor, kv
133
162
  kvState.cachedKCold = null;
134
163
  kvState.cachedVCold = null;
135
164
  kvState.coldQuantMode = 'none';
165
+ prefillFallbackNeedsCast = true;
166
+ }
167
+
168
+ // Contiguous quantized prefill fallback: decode-only kernel, use raw K/V for prefill
169
+ if (kvState.kvLayout === 'contiguous_quantized' && numTokens > 1) {
170
+ kvState.kvLayout = 'contiguous';
171
+ kvState.kvLenForAttention = numTokens;
172
+ kvState.startPosForMask = 0;
173
+ kvState.cachedKCold = null;
174
+ kvState.cachedVCold = null;
175
+ kvState.coldQuantMode = 'none';
176
+ prefillFallbackNeedsCast = true;
136
177
  }
137
178
 
138
179
  // Sliding window
@@ -181,20 +222,22 @@ export function buildAttentionDispatchParams(config, state, kTensor, vTensor, kv
181
222
 
182
223
  // Cached K/V dtypes
183
224
  const cachedKDtype = selectRuleValue('inference', 'dtype', 'f16OrFallback', {
184
- kvDtype: state.kvCache?.kvDtype,
225
+ kvDtype: resolvedKvCacheDtype,
185
226
  fallback: kTensor.dtype,
186
227
  });
187
228
  const cachedVDtype = selectRuleValue('inference', 'dtype', 'f16OrFallback', {
188
- kvDtype: state.kvCache?.kvDtype,
229
+ kvDtype: resolvedKvCacheDtype,
189
230
  fallback: vTensor.dtype,
190
231
  });
191
232
 
192
- // Cached K/V tensors (null for tiered kernels)
233
+ // Cached K/V tensors (null for tiered, contiguousQuant, and prefill-fallback paths)
193
234
  const isTieredKernel = attentionKernelVariant === 'tiered' || attentionKernelVariant === 'tieredQuant';
194
- const cachedKTensor = isTieredKernel
235
+ const isContiguousQuantKernel = attentionKernelVariant === 'contiguousQuant';
236
+ const skipCachedKVTensors = isTieredKernel || isContiguousQuantKernel || prefillFallbackNeedsCast;
237
+ const cachedKTensor = skipCachedKVTensors
195
238
  ? null
196
239
  : createTensor(kvState.cachedK, cachedKDtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
197
- const cachedVTensor = isTieredKernel
240
+ const cachedVTensor = skipCachedKVTensors
198
241
  ? null
199
242
  : createTensor(kvState.cachedV, cachedVDtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
200
243
 
@@ -207,6 +250,7 @@ export function buildAttentionDispatchParams(config, state, kTensor, vTensor, kv
207
250
  cachedKTensor,
208
251
  cachedVTensor,
209
252
  isTieredKernel,
253
+ prefillFallbackNeedsCast,
210
254
  causalForAttention: config.causalAttention !== false,
211
255
  };
212
256
  }
@@ -1,6 +1,6 @@
1
- export async function prepareAttentionProjectionInput(attnForProjection, matmulOutputDtype, castToF16) {
2
- if (matmulOutputDtype === 'f16' && attnForProjection.dtype !== 'f16') {
3
- const casted = await castToF16(attnForProjection);
1
+ export async function prepareAttentionProjectionInput(attnForProjection, matmulOutputDtype, castTensor) {
2
+ if (matmulOutputDtype && attnForProjection.dtype !== matmulOutputDtype) {
3
+ const casted = await castTensor(attnForProjection);
4
4
  return { oProjInput: casted, oProjInputTemp: casted };
5
5
  }
6
6
 
@@ -0,0 +1,36 @@
1
+ export interface AttentionPrecisionContract {
2
+ precision: {
3
+ activationDtype?: 'f16' | 'f32';
4
+ kvDtype?: 'f16' | 'f32';
5
+ inputDtype?: 'f16' | 'f32';
6
+ outputDtype?: 'f16' | 'f32';
7
+ } | null;
8
+ explicitInputDtype: 'f16' | 'f32' | null;
9
+ explicitOutputDtype: 'f16' | 'f32' | null;
10
+ explicitKvDtype: 'f16' | 'f32' | null;
11
+ resolvedActivationDtype: 'f16' | 'f32' | null;
12
+ resolvedOutputDtype: 'f16' | 'f32' | null;
13
+ resolvedKvCacheDtype: 'f16' | 'f32' | null;
14
+ }
15
+
16
+ export function resolveAttentionPrecisionContract(
17
+ config: {
18
+ isPrefill?: boolean;
19
+ layerIdx?: number;
20
+ kernelPath?: Record<string, unknown> | null;
21
+ activationDtype?: 'f16' | 'f32' | null;
22
+ inputDtype?: 'f16' | 'f32' | null;
23
+ outputDtype?: 'f16' | 'f32' | null;
24
+ kvDtype?: 'f16' | 'f32' | null;
25
+ } | null | undefined,
26
+ state: {
27
+ kvCache?: {
28
+ kvDtype?: 'f16' | 'f32' | null;
29
+ } | null;
30
+ } | null | undefined
31
+ ): AttentionPrecisionContract;
32
+
33
+ export function isAttentionKvDtypeExplicit(
34
+ contract: AttentionPrecisionContract | null | undefined,
35
+ targetDtype: 'f16' | 'f32'
36
+ ): boolean;
@@ -0,0 +1,115 @@
1
+ import { getKernelPathAttentionPrecision } from '../../../../config/kernel-path-loader.js';
2
+
3
+ const VALID_DTYPES = new Set(['f16', 'f32']);
4
+
5
+ function normalizeOptionalAttentionDtype(value, label) {
6
+ if (value == null || value === '') {
7
+ return null;
8
+ }
9
+ const normalized = String(value).trim().toLowerCase();
10
+ if (!VALID_DTYPES.has(normalized)) {
11
+ throw new Error(`[ExecutionV1] ${label} must be "f16" or "f32"; got "${value}".`);
12
+ }
13
+ return normalized;
14
+ }
15
+
16
+ export function resolveAttentionPrecisionContract(config, state) {
17
+ const phase = config?.isPrefill === true ? 'prefill' : 'decode';
18
+ const layerIdx = Number.isInteger(config?.layerIdx) ? config.layerIdx : 0;
19
+ const kernelPathPrecision = getKernelPathAttentionPrecision(phase, layerIdx, config?.kernelPath ?? undefined);
20
+ const configActivationDtype = normalizeOptionalAttentionDtype(
21
+ config?.activationDtype ?? null,
22
+ `attention config activationDtype at layer ${layerIdx}`
23
+ );
24
+ const configInputDtype = normalizeOptionalAttentionDtype(
25
+ config?.inputDtype ?? null,
26
+ `attention config inputDtype at layer ${layerIdx}`
27
+ );
28
+ const configOutputDtype = normalizeOptionalAttentionDtype(
29
+ config?.outputDtype ?? null,
30
+ `attention config outputDtype at layer ${layerIdx}`
31
+ );
32
+ const kernelPathActivationDtype = normalizeOptionalAttentionDtype(
33
+ kernelPathPrecision?.activationDtype ?? null,
34
+ `attention precision activationDtype at layer ${layerIdx}`
35
+ );
36
+ const kernelPathInputDtype = normalizeOptionalAttentionDtype(
37
+ kernelPathPrecision?.inputDtype ?? null,
38
+ `attention precision inputDtype at layer ${layerIdx}`
39
+ );
40
+ const kernelPathOutputDtype = normalizeOptionalAttentionDtype(
41
+ kernelPathPrecision?.outputDtype ?? null,
42
+ `attention precision outputDtype at layer ${layerIdx}`
43
+ );
44
+ const configKvDtype = normalizeOptionalAttentionDtype(
45
+ config?.kvDtype ?? null,
46
+ `attention config kvDtype at layer ${layerIdx}`
47
+ );
48
+ const kernelPathKvDtype = normalizeOptionalAttentionDtype(
49
+ kernelPathPrecision?.kvDtype ?? null,
50
+ `attention precision kvDtype at layer ${layerIdx}`
51
+ );
52
+
53
+ const kernelPathResolvedInputDtype = kernelPathInputDtype ?? kernelPathActivationDtype ?? null;
54
+ if (
55
+ kernelPathActivationDtype
56
+ && kernelPathInputDtype
57
+ && kernelPathActivationDtype !== kernelPathInputDtype
58
+ ) {
59
+ throw new Error(
60
+ `[ExecutionV1] attention precision declares activationDtype="${kernelPathActivationDtype}" ` +
61
+ `but inputDtype="${kernelPathInputDtype}" at layer ${layerIdx}.`
62
+ );
63
+ }
64
+
65
+ if (configInputDtype && kernelPathResolvedInputDtype && configInputDtype !== kernelPathResolvedInputDtype) {
66
+ throw new Error(
67
+ `[ExecutionV1] attention config declares inputDtype="${configInputDtype}" ` +
68
+ `but kernelPath attention precision declares inputDtype="${kernelPathResolvedInputDtype}" at layer ${layerIdx}.`
69
+ );
70
+ }
71
+
72
+ if (configOutputDtype && kernelPathOutputDtype && configOutputDtype !== kernelPathOutputDtype) {
73
+ throw new Error(
74
+ `[ExecutionV1] attention config declares outputDtype="${configOutputDtype}" ` +
75
+ `but kernelPath attention precision declares outputDtype="${kernelPathOutputDtype}" at layer ${layerIdx}.`
76
+ );
77
+ }
78
+
79
+ if (configKvDtype && kernelPathKvDtype && configKvDtype !== kernelPathKvDtype) {
80
+ throw new Error(
81
+ `[ExecutionV1] attention config declares kvDtype="${configKvDtype}" ` +
82
+ `but kernelPath attention precision declares kvDtype="${kernelPathKvDtype}" at layer ${layerIdx}.`
83
+ );
84
+ }
85
+
86
+ const explicitInputDtype = configInputDtype ?? kernelPathResolvedInputDtype ?? null;
87
+ const explicitOutputDtype = configOutputDtype ?? kernelPathOutputDtype ?? null;
88
+ const explicitKvDtype = configKvDtype ?? kernelPathKvDtype ?? null;
89
+ const liveKvDtype = normalizeOptionalAttentionDtype(
90
+ state?.kvCache?.kvDtype ?? null,
91
+ `KV cache dtype at layer ${layerIdx}`
92
+ );
93
+
94
+ if (explicitKvDtype && liveKvDtype && explicitKvDtype !== liveKvDtype) {
95
+ throw new Error(
96
+ `[ExecutionV1] attention precision declares kvDtype="${explicitKvDtype}" ` +
97
+ `but KV cache resolved "${liveKvDtype}" at layer ${layerIdx}.`
98
+ );
99
+ }
100
+
101
+ return {
102
+ precision: kernelPathPrecision,
103
+ explicitInputDtype,
104
+ explicitOutputDtype,
105
+ explicitKvDtype,
106
+ resolvedActivationDtype: explicitInputDtype ?? configActivationDtype ?? null,
107
+ resolvedOutputDtype: explicitOutputDtype ?? configActivationDtype ?? null,
108
+ resolvedKvCacheDtype: explicitKvDtype ?? liveKvDtype ?? null,
109
+ };
110
+ }
111
+
112
+ export function isAttentionKvDtypeExplicit(contract, targetDtype) {
113
+ const normalizedTarget = normalizeOptionalAttentionDtype(targetDtype, 'attention target kvDtype');
114
+ return normalizedTarget != null && contract?.explicitKvDtype === normalizedTarget;
115
+ }
@@ -58,6 +58,14 @@ export function resolveAttentionProjectionOutputDtype(
58
58
  attentionInputDtype: string,
59
59
  options?: { forceF32?: boolean }
60
60
  ): 'f16' | 'f32' | string;
61
+ export function resolveProjectionMatmulDtype(options: {
62
+ useFusedQKV: boolean;
63
+ phase: 'prefill' | 'decode';
64
+ layerIdx: number;
65
+ kernelPath: Record<string, unknown> | null | undefined;
66
+ precisionField: 'inputDtype' | 'outputDtype';
67
+ fallbackDtype: 'f16' | 'f32' | string | null | undefined;
68
+ }): 'f16' | 'f32' | string | null | undefined;
61
69
  export function resolveProjectionSliceOffsetBytes(
62
70
  weightBuffer: WeightBuffer | Tensor | GPUBuffer | null | undefined,
63
71
  outputRows: number,
@@ -112,3 +120,18 @@ export interface ApplyAttentionQKNormOptions {
112
120
  export function applyAttentionQKNorm(
113
121
  options: ApplyAttentionQKNormOptions
114
122
  ): Promise<{ qTensor: Tensor; kTensor: Tensor }>;
123
+
124
+ export interface ApplyAttentionValueNormOptions {
125
+ recorder?: CommandRecorder | null;
126
+ vTensor: Tensor;
127
+ rmsNormEps: number;
128
+ numTokens: number;
129
+ numKVHeads: number;
130
+ headDim: number;
131
+ releaseTemporary: (buffer: GPUBuffer) => void;
132
+ onVNormApplied?: ((tensor: Tensor) => Promise<void> | void) | null;
133
+ }
134
+
135
+ export function applyAttentionValueNorm(
136
+ options: ApplyAttentionValueNormOptions
137
+ ): Promise<Tensor>;