doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -0,0 +1,655 @@
1
+ /**
2
+ * Gemma 4 Conformer Audio Encoder
3
+ *
4
+ * Architecture:
5
+ * mel spectrogram -> 2-stage conv subsampling -> linear projection
6
+ * -> 12 conformer layers (half-FFN + self-attn + depthwise conv1d + half-FFN + norm)
7
+ * -> output projection -> audio embedding projection
8
+ *
9
+ * All linears use clipped ranges (input_min/max/output_min/max).
10
+ * Self-attention uses relative position encoding with per_dim_scale and logit capping.
11
+ */
12
+
13
+ import { log } from '../../../debug/index.js';
14
+ import { getDevice } from '../../../gpu/device.js';
15
+ import { createTensor } from '../../../gpu/tensor.js';
16
+ import {
17
+ runMatmul,
18
+ runRMSNorm,
19
+ runResidualAdd,
20
+ runSiLU,
21
+ } from '../../../gpu/kernel-selector.js';
22
+ import { runConv2D } from '../../../gpu/kernels/conv2d.js';
23
+ import { runDepthwiseConv1D } from '../../../gpu/kernels/depthwise_conv1d.js';
24
+ import { acquireBuffer, readBuffer, releaseBuffer, uploadData } from '../../../memory/buffer-pool.js';
25
+ import { runClippableLinear } from '../shared/clipped-linear.js';
26
+
27
+ function reshapeTensor(tensor, shape, label) {
28
+ return createTensor(tensor.buffer, tensor.dtype, shape, label ?? tensor.label);
29
+ }
30
+
31
+ // ---------------------------------------------------------------------------
32
+ // Subsampling: 2-stage Conv2D + LayerNorm + Linear projection
33
+ // ---------------------------------------------------------------------------
34
+
35
+ async function runSubsampling(melFeatures, numFrames, nMels, weights, audioConfig) {
36
+ const hiddenSize = audioConfig.hiddenSize;
37
+ const convChannels = audioConfig.subsamplingConvChannels;
38
+
39
+ // Reshape mel [numFrames, nMels] -> [1, numFrames, nMels] for conv2d (treat as single-channel image)
40
+ const inputBuffer = await uploadData(melFeatures, 'audio_mel_input');
41
+ let currentTensor = createTensor(inputBuffer, 'f32', [1, numFrames, nMels], 'audio_mel');
42
+
43
+ let currentHeight = numFrames;
44
+ let currentWidth = nMels;
45
+ let currentChannels = 1;
46
+
47
+ // Conv layer 0: [1, H, W] -> [convChannels[0], H/2, W/2] with stride=2, kernel=3, pad=1
48
+ const conv0Result = await runConv2D(
49
+ currentTensor,
50
+ weights.subsampleConv0Weight,
51
+ null,
52
+ {
53
+ inChannels: currentChannels,
54
+ outChannels: convChannels[0],
55
+ height: currentHeight,
56
+ width: currentWidth,
57
+ kernelH: 3,
58
+ kernelW: 3,
59
+ stride: 2,
60
+ pad: 1,
61
+ }
62
+ );
63
+ releaseBuffer(currentTensor.buffer);
64
+ currentHeight = Math.floor((currentHeight + 2 - 3) / 2) + 1;
65
+ currentWidth = Math.floor((currentWidth + 2 - 3) / 2) + 1;
66
+ currentChannels = convChannels[0];
67
+
68
+ // Norm after conv0
69
+ const norm0Elements = currentChannels * currentHeight * currentWidth;
70
+ const norm0Result = await runRMSNorm(
71
+ reshapeTensor(conv0Result, [norm0Elements, 1], 'audio_conv0_flat'),
72
+ weights.subsampleNorm0Weight,
73
+ audioConfig.rmsNormEps,
74
+ { batchSize: currentHeight * currentWidth, hiddenSize: currentChannels }
75
+ );
76
+ releaseBuffer(conv0Result.buffer);
77
+
78
+ // Conv layer 1: [convChannels[0], H/2, W/2] -> [convChannels[1], H/4, W/4]
79
+ const conv1Input = reshapeTensor(norm0Result, [currentChannels, currentHeight, currentWidth], 'audio_conv1_in');
80
+ const conv1Result = await runConv2D(
81
+ conv1Input,
82
+ weights.subsampleConv1Weight,
83
+ null,
84
+ {
85
+ inChannels: currentChannels,
86
+ outChannels: convChannels[1],
87
+ height: currentHeight,
88
+ width: currentWidth,
89
+ kernelH: 3,
90
+ kernelW: 3,
91
+ stride: 2,
92
+ pad: 1,
93
+ }
94
+ );
95
+ releaseBuffer(norm0Result.buffer);
96
+ currentHeight = Math.floor((currentHeight + 2 - 3) / 2) + 1;
97
+ currentWidth = Math.floor((currentWidth + 2 - 3) / 2) + 1;
98
+ currentChannels = convChannels[1];
99
+
100
+ // Norm after conv1
101
+ const norm1Result = await runRMSNorm(
102
+ reshapeTensor(conv1Result, [currentHeight * currentWidth, currentChannels], 'audio_conv1_flat'),
103
+ weights.subsampleNorm1Weight,
104
+ audioConfig.rmsNormEps,
105
+ { batchSize: currentHeight * currentWidth, hiddenSize: currentChannels }
106
+ );
107
+ releaseBuffer(conv1Result.buffer);
108
+
109
+ // Flatten to [seqLen, flatDim] where seqLen = currentHeight, flatDim = currentChannels * currentWidth
110
+ const seqLen = currentHeight;
111
+ const flatDim = currentChannels * currentWidth;
112
+
113
+ // Linear projection: [seqLen, flatDim] -> [seqLen, hiddenSize]
114
+ const projInput = reshapeTensor(norm1Result, [seqLen, flatDim], 'audio_subsample_flat');
115
+ const projected = await runMatmul(
116
+ projInput,
117
+ weights.subsampleInputProjWeight,
118
+ seqLen,
119
+ hiddenSize,
120
+ flatDim,
121
+ { outputDtype: 'f32', transposeB: 'auto' }
122
+ );
123
+ releaseBuffer(norm1Result.buffer);
124
+
125
+ return { tensor: projected, seqLen };
126
+ }
127
+
128
+ // ---------------------------------------------------------------------------
129
+ // Conformer Half-Step FFN (Macaron style)
130
+ // ---------------------------------------------------------------------------
131
+
132
+ async function runConformerFFN(hiddenTensor, layerWeights, audioConfig, seqLen, prefix) {
133
+ const hiddenSize = audioConfig.hiddenSize;
134
+ const intermediateSize = hiddenSize * 4; // Conformer FFN is typically 4x
135
+ const residualWeight = audioConfig.residualWeight;
136
+
137
+ // Pre layer norm
138
+ const normed = await runRMSNorm(
139
+ hiddenTensor,
140
+ layerWeights.preLayerNorm,
141
+ audioConfig.rmsNormEps,
142
+ { batchSize: seqLen, hiddenSize }
143
+ );
144
+
145
+ // FFW layer 1: [seqLen, hiddenSize] -> [seqLen, intermediateSize]
146
+ const ffn1 = await runClippableLinear(
147
+ normed,
148
+ layerWeights.ffwLayer1Weight,
149
+ seqLen,
150
+ intermediateSize,
151
+ hiddenSize,
152
+ layerWeights.ffwLayer1Clip,
153
+ `${prefix}_ffw1`
154
+ );
155
+ releaseBuffer(normed.buffer);
156
+
157
+ // SiLU activation
158
+ const activated = await runSiLU(ffn1, { size: seqLen * intermediateSize });
159
+ releaseBuffer(ffn1.buffer);
160
+
161
+ // FFW layer 2: [seqLen, intermediateSize] -> [seqLen, hiddenSize]
162
+ const ffn2 = await runClippableLinear(
163
+ activated,
164
+ layerWeights.ffwLayer2Weight,
165
+ seqLen,
166
+ hiddenSize,
167
+ intermediateSize,
168
+ layerWeights.ffwLayer2Clip,
169
+ `${prefix}_ffw2`
170
+ );
171
+ releaseBuffer(activated.buffer);
172
+
173
+ // Post layer norm
174
+ const postNormed = await runRMSNorm(
175
+ ffn2,
176
+ layerWeights.postLayerNorm,
177
+ audioConfig.rmsNormEps,
178
+ { batchSize: seqLen, hiddenSize }
179
+ );
180
+ releaseBuffer(ffn2.buffer);
181
+
182
+ // Scale by residualWeight (0.5 for Macaron half-step)
183
+ if (residualWeight !== 1.0) {
184
+ const data = await readBuffer(postNormed.buffer, seqLen * hiddenSize * 4);
185
+ const f32 = new Float32Array(data instanceof ArrayBuffer ? data : data.buffer);
186
+ for (let i = 0; i < f32.length; i++) {
187
+ f32[i] *= residualWeight;
188
+ }
189
+ const device = getDevice();
190
+ device.queue.writeBuffer(postNormed.buffer, 0, f32);
191
+ }
192
+
193
+ // Residual add
194
+ const result = await runResidualAdd(hiddenTensor, postNormed, seqLen * hiddenSize);
195
+ releaseBuffer(postNormed.buffer);
196
+
197
+ return result;
198
+ }
199
+
200
+ // ---------------------------------------------------------------------------
201
+ // Conformer Self-Attention with Relative Position Encoding
202
+ // ---------------------------------------------------------------------------
203
+
204
+ async function runConformerAttention(hiddenTensor, layerWeights, audioConfig, seqLen) {
205
+ const hiddenSize = audioConfig.hiddenSize;
206
+ const numHeads = audioConfig.numAttentionHeads;
207
+ const headDim = audioConfig.headDim;
208
+ const logitCap = audioConfig.attentionLogitCap;
209
+ const contextLeft = audioConfig.attentionContextLeft;
210
+ const contextRight = audioConfig.attentionContextRight;
211
+
212
+ // Pre-attention norm
213
+ const normed = await runRMSNorm(
214
+ hiddenTensor,
215
+ layerWeights.normPreAttn,
216
+ audioConfig.rmsNormEps,
217
+ { batchSize: seqLen, hiddenSize }
218
+ );
219
+
220
+ // Q/K/V projections with clipping
221
+ let qTensor = await runClippableLinear(
222
+ normed, layerWeights.qProj, seqLen, hiddenSize, hiddenSize,
223
+ layerWeights.qProjClip, 'audio_q_proj'
224
+ );
225
+ let kTensor = await runClippableLinear(
226
+ normed, layerWeights.kProj, seqLen, hiddenSize, hiddenSize,
227
+ layerWeights.kProjClip, 'audio_k_proj'
228
+ );
229
+ let vTensor = await runClippableLinear(
230
+ normed, layerWeights.vProj, seqLen, hiddenSize, hiddenSize,
231
+ layerWeights.vProjClip, 'audio_v_proj'
232
+ );
233
+ releaseBuffer(normed.buffer);
234
+
235
+ // Apply per_dim_scale to queries: Q *= softplus(per_dim_scale)
236
+ // per_dim_scale is [headDim], applied per-head
237
+ const perDimScaleData = await readBuffer(layerWeights.perDimScale, headDim * 4);
238
+ const perDimScaleF32 = new Float32Array(
239
+ perDimScaleData instanceof ArrayBuffer ? perDimScaleData : perDimScaleData.buffer
240
+ );
241
+ const qData = await readBuffer(qTensor.buffer, seqLen * hiddenSize * 4);
242
+ const qF32 = new Float32Array(qData instanceof ArrayBuffer ? qData : qData.buffer);
243
+ for (let s = 0; s < seqLen; s++) {
244
+ for (let h = 0; h < numHeads; h++) {
245
+ for (let d = 0; d < headDim; d++) {
246
+ const idx = s * hiddenSize + h * headDim + d;
247
+ const scale = Math.log(1 + Math.exp(perDimScaleF32[d])); // softplus
248
+ qF32[idx] *= scale;
249
+ }
250
+ }
251
+ }
252
+
253
+ // Compute attention scores: [seqLen, numHeads, seqLen]
254
+ // Score = Q @ K^T / sqrt(headDim)
255
+ const scaleFactor = 1.0 / Math.sqrt(headDim);
256
+ const scores = new Float32Array(numHeads * seqLen * seqLen);
257
+
258
+ const kData = await readBuffer(kTensor.buffer, seqLen * hiddenSize * 4);
259
+ const kF32 = new Float32Array(kData instanceof ArrayBuffer ? kData : kData.buffer);
260
+
261
+ for (let h = 0; h < numHeads; h++) {
262
+ for (let qi = 0; qi < seqLen; qi++) {
263
+ for (let ki = 0; ki < seqLen; ki++) {
264
+ let dot = 0;
265
+ for (let d = 0; d < headDim; d++) {
266
+ dot += qF32[qi * hiddenSize + h * headDim + d] * kF32[ki * hiddenSize + h * headDim + d];
267
+ }
268
+ scores[h * seqLen * seqLen + qi * seqLen + ki] = dot * scaleFactor;
269
+ }
270
+ }
271
+ }
272
+
273
+ // Add relative position bias via relative_k_proj
274
+ // This is a simplified approach: compute relative position embeddings and add to scores
275
+ const relKProjData = await readBuffer(layerWeights.relativeKProj, hiddenSize * hiddenSize * 4);
276
+ const relKProjF32 = new Float32Array(
277
+ relKProjData instanceof ArrayBuffer ? relKProjData : relKProjData.buffer
278
+ );
279
+
280
+ // Build relative position vectors and project
281
+ for (let h = 0; h < numHeads; h++) {
282
+ for (let qi = 0; qi < seqLen; qi++) {
283
+ for (let ki = 0; ki < seqLen; ki++) {
284
+ const relPos = ki - qi;
285
+ // Compute relative bias from q and relative_k_proj
286
+ let relBias = 0;
287
+ for (let d = 0; d < headDim; d++) {
288
+ const qVal = qF32[qi * hiddenSize + h * headDim + d];
289
+ // Use distance-based relative encoding
290
+ relBias += qVal * relKProjF32[(h * headDim + d) * hiddenSize + (Math.abs(relPos) % hiddenSize)];
291
+ }
292
+ scores[h * seqLen * seqLen + qi * seqLen + ki] += relBias * scaleFactor;
293
+ }
294
+ }
295
+ }
296
+
297
+ // Apply logit capping: scores = cap * tanh(scores / cap)
298
+ for (let i = 0; i < scores.length; i++) {
299
+ scores[i] = logitCap * Math.tanh(scores[i] / logitCap);
300
+ }
301
+
302
+ // Apply chunked attention mask (contextLeft, contextRight)
303
+ const invalidValue = audioConfig.attentionInvalidLogitsValue;
304
+ for (let h = 0; h < numHeads; h++) {
305
+ for (let qi = 0; qi < seqLen; qi++) {
306
+ for (let ki = 0; ki < seqLen; ki++) {
307
+ const dist = ki - qi;
308
+ if (dist < -contextLeft || dist > contextRight) {
309
+ scores[h * seqLen * seqLen + qi * seqLen + ki] = invalidValue;
310
+ }
311
+ }
312
+ }
313
+ }
314
+
315
+ // Softmax per query position per head
316
+ for (let h = 0; h < numHeads; h++) {
317
+ for (let qi = 0; qi < seqLen; qi++) {
318
+ const offset = h * seqLen * seqLen + qi * seqLen;
319
+ let maxVal = -Infinity;
320
+ for (let ki = 0; ki < seqLen; ki++) {
321
+ if (scores[offset + ki] > maxVal) maxVal = scores[offset + ki];
322
+ }
323
+ let sumExp = 0;
324
+ for (let ki = 0; ki < seqLen; ki++) {
325
+ scores[offset + ki] = Math.exp(scores[offset + ki] - maxVal);
326
+ sumExp += scores[offset + ki];
327
+ }
328
+ for (let ki = 0; ki < seqLen; ki++) {
329
+ scores[offset + ki] /= sumExp;
330
+ }
331
+ }
332
+ }
333
+
334
+ // Apply attention: output = scores @ V
335
+ const vData = await readBuffer(vTensor.buffer, seqLen * hiddenSize * 4);
336
+ const vF32 = new Float32Array(vData instanceof ArrayBuffer ? vData : vData.buffer);
337
+ const attnOutput = new Float32Array(seqLen * hiddenSize);
338
+
339
+ for (let h = 0; h < numHeads; h++) {
340
+ for (let qi = 0; qi < seqLen; qi++) {
341
+ for (let d = 0; d < headDim; d++) {
342
+ let sum = 0;
343
+ for (let ki = 0; ki < seqLen; ki++) {
344
+ sum += scores[h * seqLen * seqLen + qi * seqLen + ki] * vF32[ki * hiddenSize + h * headDim + d];
345
+ }
346
+ attnOutput[qi * hiddenSize + h * headDim + d] = sum;
347
+ }
348
+ }
349
+ }
350
+
351
+ releaseBuffer(qTensor.buffer);
352
+ releaseBuffer(kTensor.buffer);
353
+ releaseBuffer(vTensor.buffer);
354
+
355
+ // Upload attention output
356
+ const attnBuffer = await uploadData(attnOutput, 'audio_attn_output');
357
+ const attnTensor = createTensor(attnBuffer, 'f32', [seqLen, hiddenSize], 'audio_attn_output');
358
+
359
+ // Output projection with clipping
360
+ const projected = await runClippableLinear(
361
+ attnTensor,
362
+ layerWeights.postProj,
363
+ seqLen,
364
+ hiddenSize,
365
+ hiddenSize,
366
+ layerWeights.postProjClip,
367
+ 'audio_attn_post'
368
+ );
369
+ releaseBuffer(attnTensor.buffer);
370
+
371
+ // Post-attention norm
372
+ const postNormed = await runRMSNorm(
373
+ projected,
374
+ layerWeights.normPostAttn,
375
+ audioConfig.rmsNormEps,
376
+ { batchSize: seqLen, hiddenSize }
377
+ );
378
+ releaseBuffer(projected.buffer);
379
+
380
+ // Residual add
381
+ const result = await runResidualAdd(hiddenTensor, postNormed, seqLen * hiddenSize);
382
+ releaseBuffer(postNormed.buffer);
383
+
384
+ return result;
385
+ }
386
+
387
+ // ---------------------------------------------------------------------------
388
+ // Conformer Convolution Module (LConv1D)
389
+ // ---------------------------------------------------------------------------
390
+
391
+ async function runConformerConvModule(hiddenTensor, layerWeights, audioConfig, seqLen) {
392
+ const hiddenSize = audioConfig.hiddenSize;
393
+ const kernelSize = audioConfig.convKernelSize;
394
+
395
+ // Pre layer norm
396
+ const normed = await runRMSNorm(
397
+ hiddenTensor,
398
+ layerWeights.lconvPreLayerNorm,
399
+ audioConfig.rmsNormEps,
400
+ { batchSize: seqLen, hiddenSize }
401
+ );
402
+
403
+ // Linear start (pointwise): [seqLen, hiddenSize] -> [seqLen, hiddenSize*2]
404
+ // The GLU split: first half goes through sigmoid gate, second half is the value
405
+ const linearStart = await runClippableLinear(
406
+ normed,
407
+ layerWeights.lconvLinearStartWeight,
408
+ seqLen,
409
+ hiddenSize * 2,
410
+ hiddenSize,
411
+ layerWeights.lconvLinearStartClip,
412
+ 'audio_lconv_start'
413
+ );
414
+ releaseBuffer(normed.buffer);
415
+
416
+ // GLU: split into gate and value, gate = sigmoid(gate_half), output = gate * value_half
417
+ const startData = await readBuffer(linearStart.buffer, seqLen * hiddenSize * 2 * 4);
418
+ const startF32 = new Float32Array(startData instanceof ArrayBuffer ? startData : startData.buffer);
419
+ const gluOutput = new Float32Array(seqLen * hiddenSize);
420
+ for (let s = 0; s < seqLen; s++) {
421
+ for (let d = 0; d < hiddenSize; d++) {
422
+ const gateVal = startF32[s * hiddenSize * 2 + d];
423
+ const valueVal = startF32[s * hiddenSize * 2 + hiddenSize + d];
424
+ const sigmoid = 1.0 / (1.0 + Math.exp(-gateVal));
425
+ gluOutput[s * hiddenSize + d] = sigmoid * valueVal;
426
+ }
427
+ }
428
+ releaseBuffer(linearStart.buffer);
429
+
430
+ // Depthwise Conv1D: [channels=hiddenSize, length=seqLen]
431
+ // Transpose from [seqLen, hiddenSize] to [hiddenSize, seqLen]
432
+ const transposed = new Float32Array(hiddenSize * seqLen);
433
+ for (let s = 0; s < seqLen; s++) {
434
+ for (let d = 0; d < hiddenSize; d++) {
435
+ transposed[d * seqLen + s] = gluOutput[s * hiddenSize + d];
436
+ }
437
+ }
438
+ const transposedBuffer = await uploadData(transposed, 'audio_lconv_transposed');
439
+ const transposedTensor = createTensor(transposedBuffer, 'f32', [hiddenSize, seqLen], 'audio_lconv_transposed');
440
+
441
+ const convResult = await runDepthwiseConv1D(
442
+ transposedTensor,
443
+ layerWeights.lconvDepthwiseWeight,
444
+ { channels: hiddenSize, length: seqLen, kernelSize }
445
+ );
446
+ releaseBuffer(transposedTensor.buffer);
447
+
448
+ // Transpose back: [hiddenSize, seqLen] -> [seqLen, hiddenSize]
449
+ const convData = await readBuffer(convResult.buffer, hiddenSize * seqLen * 4);
450
+ const convF32 = new Float32Array(convData instanceof ArrayBuffer ? convData : convData.buffer);
451
+ const convTransposed = new Float32Array(seqLen * hiddenSize);
452
+ for (let s = 0; s < seqLen; s++) {
453
+ for (let d = 0; d < hiddenSize; d++) {
454
+ convTransposed[s * hiddenSize + d] = convF32[d * seqLen + s];
455
+ }
456
+ }
457
+ releaseBuffer(convResult.buffer);
458
+
459
+ // Conv norm (RMSNorm on the convolution output)
460
+ const convNormBuffer = await uploadData(convTransposed, 'audio_lconv_normed');
461
+ const convNormTensor = createTensor(convNormBuffer, 'f32', [seqLen, hiddenSize], 'audio_lconv_normed');
462
+ const convNormed = await runRMSNorm(
463
+ convNormTensor,
464
+ layerWeights.lconvConvNorm,
465
+ audioConfig.rmsNormEps,
466
+ { batchSize: seqLen, hiddenSize }
467
+ );
468
+ releaseBuffer(convNormTensor.buffer);
469
+
470
+ // SiLU activation
471
+ const activated = await runSiLU(convNormed, { size: seqLen * hiddenSize });
472
+ releaseBuffer(convNormed.buffer);
473
+
474
+ // Linear end (pointwise): [seqLen, hiddenSize] -> [seqLen, hiddenSize]
475
+ const linearEnd = await runClippableLinear(
476
+ activated,
477
+ layerWeights.lconvLinearEndWeight,
478
+ seqLen,
479
+ hiddenSize,
480
+ hiddenSize,
481
+ layerWeights.lconvLinearEndClip,
482
+ 'audio_lconv_end'
483
+ );
484
+ releaseBuffer(activated.buffer);
485
+
486
+ // Residual add
487
+ const result = await runResidualAdd(hiddenTensor, linearEnd, seqLen * hiddenSize);
488
+ releaseBuffer(linearEnd.buffer);
489
+
490
+ return result;
491
+ }
492
+
493
+ // ---------------------------------------------------------------------------
494
+ // Full Conformer Layer
495
+ // ---------------------------------------------------------------------------
496
+
497
+ async function runConformerLayer(hiddenTensor, layerWeights, audioConfig, seqLen, layerIdx) {
498
+ // 1. Half-step FFN1 (Macaron)
499
+ let current = await runConformerFFN(
500
+ hiddenTensor,
501
+ layerWeights.feedForward1,
502
+ audioConfig,
503
+ seqLen,
504
+ `audio_layer${layerIdx}_ff1`
505
+ );
506
+
507
+ // 2. Self-attention with relative position encoding
508
+ const afterAttn = await runConformerAttention(
509
+ current,
510
+ layerWeights,
511
+ audioConfig,
512
+ seqLen
513
+ );
514
+ releaseBuffer(current.buffer);
515
+ current = afterAttn;
516
+
517
+ // 3. Convolution module
518
+ const afterConv = await runConformerConvModule(
519
+ current,
520
+ layerWeights,
521
+ audioConfig,
522
+ seqLen
523
+ );
524
+ releaseBuffer(current.buffer);
525
+ current = afterConv;
526
+
527
+ // 4. Half-step FFN2
528
+ const afterFFN2 = await runConformerFFN(
529
+ current,
530
+ layerWeights.feedForward2,
531
+ audioConfig,
532
+ seqLen,
533
+ `audio_layer${layerIdx}_ff2`
534
+ );
535
+ releaseBuffer(current.buffer);
536
+ current = afterFFN2;
537
+
538
+ // 5. Final norm_out
539
+ const hiddenSize = audioConfig.hiddenSize;
540
+ const normed = await runRMSNorm(
541
+ current,
542
+ layerWeights.normOut,
543
+ audioConfig.rmsNormEps,
544
+ { batchSize: seqLen, hiddenSize }
545
+ );
546
+ releaseBuffer(current.buffer);
547
+
548
+ return normed;
549
+ }
550
+
551
+ // ---------------------------------------------------------------------------
552
+ // Main Encoder Entry Point
553
+ // ---------------------------------------------------------------------------
554
+
555
+ /**
556
+ * Encode audio through the Gemma 4 conformer audio tower.
557
+ *
558
+ * @param {object} params
559
+ * @param {Float32Array} params.melFeatures Log-mel spectrogram [numFrames * nMels]
560
+ * @param {number} params.numFrames Number of mel frames
561
+ * @param {number} params.nMels Number of mel bands
562
+ * @param {object} params.audioConfig Resolved audio encoder config
563
+ * @param {object} params.weights Audio encoder weight buffers
564
+ * @returns {Promise<{ features: GPUBuffer, numTokens: number }>}
565
+ */
566
+ export async function encodeGemma4Audio(params) {
567
+ const { melFeatures, numFrames, nMels, audioConfig, weights } = params;
568
+
569
+ if (audioConfig.audioArchitecture !== 'gemma4') {
570
+ throw new Error(
571
+ `[Audio] Unsupported audio architecture "${audioConfig.audioArchitecture}". Expected "gemma4".`
572
+ );
573
+ }
574
+ if (!audioConfig.useClippedLinears) {
575
+ throw new Error('[Audio] Gemma 4 audio encoder requires useClippedLinears=true.');
576
+ }
577
+
578
+ const hiddenSize = audioConfig.hiddenSize;
579
+ const depth = audioConfig.depth;
580
+
581
+ log.debug('Audio', `encodeGemma4Audio: ${numFrames} frames, ${nMels} mels, ${depth} layers`);
582
+
583
+ // Step 1: Subsampling — Conv -> Norm -> Linear projection
584
+ const { tensor: subsampledTensor, seqLen } = await runSubsampling(
585
+ melFeatures, numFrames, nMels, weights, audioConfig
586
+ );
587
+
588
+ log.debug('Audio', `Subsampling complete: seqLen=${seqLen}, hiddenSize=${hiddenSize}`);
589
+
590
+ // Step 2: Conformer layers
591
+ let currentTensor = subsampledTensor;
592
+ for (let i = 0; i < depth; i++) {
593
+ const layerResult = await runConformerLayer(
594
+ currentTensor,
595
+ weights.layers[i],
596
+ audioConfig,
597
+ seqLen,
598
+ i
599
+ );
600
+ if (currentTensor !== subsampledTensor || i > 0) {
601
+ releaseBuffer(currentTensor.buffer);
602
+ }
603
+ currentTensor = layerResult;
604
+ }
605
+ if (depth > 0) {
606
+ // subsampledTensor was released in the loop
607
+ } else {
608
+ releaseBuffer(subsampledTensor.buffer);
609
+ }
610
+
611
+ // Step 3: Output projection [seqLen, hiddenSize] -> [seqLen, outputProjDims]
612
+ const outputProjDims = audioConfig.outputProjDims;
613
+ const outputProj = await runMatmul(
614
+ currentTensor,
615
+ weights.outputProjWeight,
616
+ seqLen,
617
+ outputProjDims,
618
+ hiddenSize,
619
+ { outputDtype: 'f32', transposeB: 'auto' }
620
+ );
621
+ releaseBuffer(currentTensor.buffer);
622
+
623
+ // Add output projection bias
624
+ if (weights.outputProjBias) {
625
+ const biasData = await readBuffer(weights.outputProjBias, outputProjDims * 4);
626
+ const biasF32 = new Float32Array(biasData instanceof ArrayBuffer ? biasData : biasData.buffer);
627
+ const projData = await readBuffer(outputProj.buffer, seqLen * outputProjDims * 4);
628
+ const projF32 = new Float32Array(projData instanceof ArrayBuffer ? projData : projData.buffer);
629
+ for (let s = 0; s < seqLen; s++) {
630
+ for (let d = 0; d < outputProjDims; d++) {
631
+ projF32[s * outputProjDims + d] += biasF32[d];
632
+ }
633
+ }
634
+ const device = getDevice();
635
+ device.queue.writeBuffer(outputProj.buffer, 0, projF32);
636
+ }
637
+
638
+ // Step 4: Audio embedding projection [seqLen, outputProjDims] -> [seqLen, outputProjDims]
639
+ const embedProj = await runMatmul(
640
+ outputProj,
641
+ weights.audioEmbeddingProjWeight,
642
+ seqLen,
643
+ outputProjDims,
644
+ outputProjDims,
645
+ { outputDtype: 'f32', transposeB: 'auto' }
646
+ );
647
+ releaseBuffer(outputProj.buffer);
648
+
649
+ log.info('Audio', `Gemma 4 audio encoding complete: ${seqLen} tokens, ${outputProjDims} dims`);
650
+
651
+ return {
652
+ features: embedProj.buffer,
653
+ numTokens: seqLen,
654
+ };
655
+ }
@@ -0,0 +1,39 @@
1
+
2
+ import { log } from '../../../debug/index.js';
3
+ import { encodeGemma4Audio } from './gemma4.js';
4
+
5
+ /**
6
+ * Encode audio through the audio pipeline.
7
+ *
8
+ * Routes to architecture-specific encoder based on audioConfig.audioArchitecture.
9
+ *
10
+ * @param {object} params
11
+ * @param {Float32Array} params.melFeatures Log-mel spectrogram [numFrames * nMels]
12
+ * @param {number} params.numFrames Number of mel frames
13
+ * @param {number} params.nMels Number of mel bands
14
+ * @param {object} params.audioConfig Audio encoder config from manifest
15
+ * @param {object} params.weights Audio encoder weight buffers
16
+ * @returns {Promise<AudioEncodeResult>}
17
+ */
18
+ export async function encodeAudio(params) {
19
+ const { audioConfig } = params;
20
+ const arch = audioConfig?.audioArchitecture ?? null;
21
+
22
+ log.debug('Audio', `encodeAudio: arch=${arch}`);
23
+
24
+ switch (arch) {
25
+ case 'gemma4':
26
+ return encodeGemma4Audio(params);
27
+ default:
28
+ throw new Error(
29
+ `Unsupported audio architecture "${arch}". ` +
30
+ 'Supported: gemma4. Check audio_config.audio_architecture.'
31
+ );
32
+ }
33
+ }
34
+
35
+ /**
36
+ * @typedef {object} AudioEncodeResult
37
+ * @property {GPUBuffer} features Encoded audio tokens [numTokens, outputDims]
38
+ * @property {number} numTokens Number of audio tokens after subsampling
39
+ */