doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,6 +1,13 @@
1
1
  import { log } from '../../../debug/index.js';
2
2
  import { mergeConfig, dumpConfigSources } from '../../../config/merge.js';
3
3
  import { selectRuleValue } from '../../../rules/rule-registry.js';
4
+ import {
5
+ PER_LAYER_INPUT_MATERIALIZATION_MODES,
6
+ PER_LAYER_INPUT_ROW_CACHE_MODES,
7
+ PER_LAYER_INPUT_PREFETCH_MODES,
8
+ PER_LAYER_INPUT_GPU_UPLOAD_MODES,
9
+ PER_LAYER_INPUT_HOT_CACHE_MODES,
10
+ } from '../../../config/schema/execution-v1.schema.js';
4
11
 
5
12
  const UNSUPPORTED_RUNTIME_MODEL_TYPES = new Set(['mamba', 'rwkv']);
6
13
 
@@ -11,6 +18,7 @@ const UNSUPPORTED_RUNTIME_MODEL_TYPES = new Set(['mamba', 'rwkv']);
11
18
  */
12
19
  const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
13
20
  'gemma',
21
+ 'gemma4',
14
22
  'llama3',
15
23
  'gpt-oss',
16
24
  'chatml',
@@ -20,8 +28,7 @@ const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
20
28
 
21
29
  /**
22
30
  * Validate that a chatTemplate.type value is either null (disabled) or a known
23
- * formatter type. Logs a warning for unknown types so callers get early feedback
24
- * without a breaking throw.
31
+ * formatter type.
25
32
  *
26
33
  * @param {string | null} type - The chatTemplate.type from manifest inference.
27
34
  * @param {string} modelId - Model identifier for diagnostic messages.
@@ -30,13 +37,163 @@ const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
30
37
  export function validateChatTemplateType(type, modelId) {
31
38
  if (type === null || type === undefined) return true;
32
39
  if (KNOWN_CHAT_TEMPLATE_TYPES.has(type)) return true;
33
- log.warn(
34
- 'Config',
35
- `Manifest "${modelId}" declares chatTemplate.type="${type}" which is not a known ` +
36
- `formatter type. Known types: ${[...KNOWN_CHAT_TEMPLATE_TYPES].join(', ')}. ` +
37
- `The formatter will fall back to plaintext at generation time.`
40
+ throw new Error(
41
+ `Manifest "${modelId}" declares chatTemplate.type="${type}" which is not a known formatter type. ` +
42
+ `Known types: ${[...KNOWN_CHAT_TEMPLATE_TYPES].join(', ')}. Re-convert the model or fix the manifest.`
38
43
  );
39
- return false;
44
+ }
45
+
46
+ function resolvePerLayerInputsSession(inferenceConfig, modelId) {
47
+ const sessionConfig = inferenceConfig?.session?.perLayerInputs;
48
+ if (sessionConfig === undefined) {
49
+ throw new Error(
50
+ `Manifest "${modelId}" is missing inference.session.perLayerInputs. ` +
51
+ 'Re-convert the model so per-layer input materialization policy is explicit.'
52
+ );
53
+ }
54
+ if (!sessionConfig || typeof sessionConfig !== 'object' || Array.isArray(sessionConfig)) {
55
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.perLayerInputs.`);
56
+ }
57
+
58
+ const materialization = sessionConfig.materialization;
59
+ if (!PER_LAYER_INPUT_MATERIALIZATION_MODES.includes(materialization)) {
60
+ throw new Error(
61
+ `Manifest "${modelId}" has invalid inference.session.perLayerInputs.materialization ` +
62
+ `"${String(materialization)}".`
63
+ );
64
+ }
65
+
66
+ const rowCache = sessionConfig.rowCache;
67
+ if (!rowCache || typeof rowCache !== 'object' || Array.isArray(rowCache)) {
68
+ throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.rowCache.`);
69
+ }
70
+ if (!PER_LAYER_INPUT_ROW_CACHE_MODES.includes(rowCache.mode)) {
71
+ throw new Error(
72
+ `Manifest "${modelId}" has invalid inference.session.perLayerInputs.rowCache.mode ` +
73
+ `"${String(rowCache.mode)}".`
74
+ );
75
+ }
76
+
77
+ const maxRows = Math.trunc(Number(rowCache.maxRows));
78
+ if (!Number.isFinite(maxRows) || maxRows <= 0) {
79
+ throw new Error(
80
+ `Manifest "${modelId}" requires inference.session.perLayerInputs.rowCache.maxRows ` +
81
+ `to be a positive integer; got ${String(rowCache.maxRows)}.`
82
+ );
83
+ }
84
+
85
+ const maxBytes = Math.trunc(Number(rowCache.maxBytes));
86
+ if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
87
+ throw new Error(
88
+ `Manifest "${modelId}" requires inference.session.perLayerInputs.rowCache.maxBytes ` +
89
+ `to be a positive integer; got ${String(rowCache.maxBytes)}.`
90
+ );
91
+ }
92
+
93
+ const decodedDtype = String(rowCache.decodedDtype ?? '').toLowerCase();
94
+ if (decodedDtype !== 'f16' && decodedDtype !== 'f32') {
95
+ throw new Error(
96
+ `Manifest "${modelId}" has invalid inference.session.perLayerInputs.rowCache.decodedDtype ` +
97
+ `"${String(rowCache.decodedDtype)}".`
98
+ );
99
+ }
100
+
101
+ const prefetch = sessionConfig.prefetch;
102
+ if (!prefetch || typeof prefetch !== 'object' || Array.isArray(prefetch)) {
103
+ throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.prefetch.`);
104
+ }
105
+ if (!PER_LAYER_INPUT_PREFETCH_MODES.includes(prefetch.mode)) {
106
+ throw new Error(
107
+ `Manifest "${modelId}" has invalid inference.session.perLayerInputs.prefetch.mode ` +
108
+ `"${String(prefetch.mode)}".`
109
+ );
110
+ }
111
+
112
+ const rowsAhead = Math.trunc(Number(prefetch.rowsAhead));
113
+ if (!Number.isFinite(rowsAhead) || rowsAhead <= 0) {
114
+ throw new Error(
115
+ `Manifest "${modelId}" requires inference.session.perLayerInputs.prefetch.rowsAhead ` +
116
+ `to be a positive integer; got ${String(prefetch.rowsAhead)}.`
117
+ );
118
+ }
119
+
120
+ const gpuUpload = sessionConfig.gpuUpload;
121
+ if (!gpuUpload || typeof gpuUpload !== 'object' || Array.isArray(gpuUpload)) {
122
+ throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.gpuUpload.`);
123
+ }
124
+ if (!PER_LAYER_INPUT_GPU_UPLOAD_MODES.includes(gpuUpload.mode)) {
125
+ throw new Error(
126
+ `Manifest "${modelId}" has invalid inference.session.perLayerInputs.gpuUpload.mode ` +
127
+ `"${String(gpuUpload.mode)}".`
128
+ );
129
+ }
130
+
131
+ const stagingRows = Math.trunc(Number(gpuUpload.stagingRows));
132
+ if (!Number.isFinite(stagingRows) || stagingRows <= 0) {
133
+ throw new Error(
134
+ `Manifest "${modelId}" requires inference.session.perLayerInputs.gpuUpload.stagingRows ` +
135
+ `to be a positive integer; got ${String(gpuUpload.stagingRows)}.`
136
+ );
137
+ }
138
+
139
+ const hotCache = sessionConfig.hotCache;
140
+ if (!hotCache || typeof hotCache !== 'object' || Array.isArray(hotCache)) {
141
+ throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.hotCache.`);
142
+ }
143
+ if (!PER_LAYER_INPUT_HOT_CACHE_MODES.includes(hotCache.mode)) {
144
+ throw new Error(
145
+ `Manifest "${modelId}" has invalid inference.session.perLayerInputs.hotCache.mode ` +
146
+ `"${String(hotCache.mode)}".`
147
+ );
148
+ }
149
+
150
+ const hotMaxTokens = Math.trunc(Number(hotCache.maxTokens));
151
+ if (!Number.isFinite(hotMaxTokens) || hotMaxTokens <= 0) {
152
+ throw new Error(
153
+ `Manifest "${modelId}" requires inference.session.perLayerInputs.hotCache.maxTokens ` +
154
+ `to be a positive integer; got ${String(hotCache.maxTokens)}.`
155
+ );
156
+ }
157
+
158
+ const hotMaxBytes = Math.trunc(Number(hotCache.maxBytes));
159
+ if (!Number.isFinite(hotMaxBytes) || hotMaxBytes <= 0) {
160
+ throw new Error(
161
+ `Manifest "${modelId}" requires inference.session.perLayerInputs.hotCache.maxBytes ` +
162
+ `to be a positive integer; got ${String(hotCache.maxBytes)}.`
163
+ );
164
+ }
165
+
166
+ const hotOutputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
167
+ if (hotOutputDtype !== 'f16' && hotOutputDtype !== 'f32') {
168
+ throw new Error(
169
+ `Manifest "${modelId}" has invalid inference.session.perLayerInputs.hotCache.outputDtype ` +
170
+ `"${String(hotCache.outputDtype)}".`
171
+ );
172
+ }
173
+
174
+ return {
175
+ materialization,
176
+ rowCache: {
177
+ mode: rowCache.mode,
178
+ maxRows,
179
+ maxBytes,
180
+ decodedDtype,
181
+ },
182
+ prefetch: {
183
+ mode: prefetch.mode,
184
+ rowsAhead,
185
+ },
186
+ gpuUpload: {
187
+ mode: gpuUpload.mode,
188
+ stagingRows,
189
+ },
190
+ hotCache: {
191
+ mode: hotCache.mode,
192
+ maxTokens: hotMaxTokens,
193
+ maxBytes: hotMaxBytes,
194
+ outputDtype: hotOutputDtype,
195
+ },
196
+ };
40
197
  }
41
198
 
42
199
  // =============================================================================
@@ -78,6 +235,32 @@ function resolveRotaryDim(headDim, partialRotaryFactor, modelId) {
78
235
  return rotaryDim;
79
236
  }
80
237
 
238
+ function resolveFrequencyBaseDim(headDim, rotaryDim, frequencyBaseDim, modelId, fieldName) {
239
+ if (frequencyBaseDim == null) {
240
+ return rotaryDim;
241
+ }
242
+ if (typeof frequencyBaseDim !== 'number' || Number.isNaN(frequencyBaseDim)) {
243
+ throw new Error(`Manifest "${modelId}" has invalid ${fieldName}.`);
244
+ }
245
+ const resolved = Math.trunc(frequencyBaseDim);
246
+ if (resolved <= 0 || (resolved % 2) !== 0) {
247
+ throw new Error(
248
+ `Manifest "${modelId}" requires ${fieldName} to be a positive even integer; got ${frequencyBaseDim}.`
249
+ );
250
+ }
251
+ if (resolved < rotaryDim) {
252
+ throw new Error(
253
+ `Manifest "${modelId}" requires ${fieldName} (${resolved}) to be >= rotary dim (${rotaryDim}).`
254
+ );
255
+ }
256
+ if (resolved > headDim) {
257
+ throw new Error(
258
+ `Manifest "${modelId}" requires ${fieldName} (${resolved}) to be <= attention head dim (${headDim}).`
259
+ );
260
+ }
261
+ return resolved;
262
+ }
263
+
81
264
  export function getStopTokenIds(manifest) {
82
265
  const eosTokenId = manifest?.eos_token_id;
83
266
  if (Array.isArray(eosTokenId)) return eosTokenId;
@@ -189,6 +372,170 @@ function resolveIntermediateSizeForRuntime(manifest, inf, arch, modelId) {
189
372
  );
190
373
  }
191
374
 
375
+ function buildPerLayerIntermediateSizes({
376
+ baseIntermediateSize,
377
+ numLayers,
378
+ numKvSharedLayers,
379
+ useDoubleWideMlp,
380
+ modelId,
381
+ }) {
382
+ if (!Number.isFinite(baseIntermediateSize) || baseIntermediateSize <= 0) {
383
+ throw new Error(
384
+ `Manifest "${modelId}" has invalid architecture.intermediateSize (${String(baseIntermediateSize)}).`
385
+ );
386
+ }
387
+ if (!Number.isFinite(numLayers) || numLayers <= 0) {
388
+ throw new Error(
389
+ `Manifest "${modelId}" has invalid architecture.numLayers (${String(numLayers)}).`
390
+ );
391
+ }
392
+
393
+ const resolvedBaseIntermediateSize = Math.trunc(baseIntermediateSize);
394
+ const resolvedNumLayers = Math.trunc(numLayers);
395
+ const intermediateSizes = new Array(resolvedNumLayers).fill(resolvedBaseIntermediateSize);
396
+
397
+ if (!useDoubleWideMlp) {
398
+ return intermediateSizes;
399
+ }
400
+
401
+ if (!Number.isFinite(numKvSharedLayers) || numKvSharedLayers <= 0 || numKvSharedLayers >= resolvedNumLayers) {
402
+ throw new Error(
403
+ `Manifest "${modelId}" enables ffn.useDoubleWideMlp, but architecture.numKvSharedLayers=${String(numKvSharedLayers)} ` +
404
+ `must be a positive integer smaller than numLayers=${resolvedNumLayers}.`
405
+ );
406
+ }
407
+
408
+ const firstKvSharedLayerIdx = resolvedNumLayers - Math.trunc(numKvSharedLayers);
409
+ if (firstKvSharedLayerIdx <= 0 || firstKvSharedLayerIdx >= resolvedNumLayers) {
410
+ throw new Error(
411
+ `Manifest "${modelId}" enables ffn.useDoubleWideMlp, but the derived first KV-shared layer index ` +
412
+ `(${firstKvSharedLayerIdx}) is invalid for numLayers=${resolvedNumLayers}.`
413
+ );
414
+ }
415
+
416
+ const widenedIntermediateSize = resolvedBaseIntermediateSize * 2;
417
+ for (let layerIdx = firstKvSharedLayerIdx; layerIdx < resolvedNumLayers; layerIdx += 1) {
418
+ intermediateSizes[layerIdx] = widenedIntermediateSize;
419
+ }
420
+
421
+ return intermediateSizes;
422
+ }
423
+
424
+ function getDenseFfnTensorShape(tensors, names) {
425
+ if (!tensors || typeof tensors !== 'object') return null;
426
+ for (const name of names) {
427
+ const shape = normalizeFfnTensorShape(tensors[name]?.shape);
428
+ if (shape) return shape;
429
+ }
430
+ return null;
431
+ }
432
+
433
+ function assertDenseFfnTensorShape(modelId, layerIdx, label, actualShape, expectedShape) {
434
+ if (!actualShape) return;
435
+ if (actualShape[0] === expectedShape[0] && actualShape[1] === expectedShape[1]) {
436
+ return;
437
+ }
438
+ throw new Error(
439
+ `Manifest "${modelId}" layer ${layerIdx} ${label} shape [${actualShape.join(', ')}] does not match ` +
440
+ `the resolved FFN contract [${expectedShape.join(', ')}]. Re-convert the model so manifest inference ` +
441
+ 'and FFN tensor shapes agree.'
442
+ );
443
+ }
444
+
445
+ function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, layerIntermediateSizes, modelId) {
446
+ const tensors = manifest?.tensors;
447
+ if (!tensors || typeof tensors !== 'object' || !Array.isArray(layerIntermediateSizes)) {
448
+ return;
449
+ }
450
+
451
+ for (let layerIdx = 0; layerIdx < layerIntermediateSizes.length; layerIdx += 1) {
452
+ const intermediateSize = Number(layerIntermediateSizes[layerIdx]);
453
+ if (!Number.isFinite(intermediateSize) || intermediateSize <= 0) {
454
+ throw new Error(
455
+ `Manifest "${modelId}" resolved an invalid FFN intermediate size (${String(layerIntermediateSizes[layerIdx])}) ` +
456
+ `for layer ${layerIdx}.`
457
+ );
458
+ }
459
+
460
+ const expectedIntermediateSize = Math.trunc(intermediateSize);
461
+ const languagePrefix = `model.language_model.layers.${layerIdx}`;
462
+ const genericPrefix = `model.layers.${layerIdx}`;
463
+ const gateShape = getDenseFfnTensorShape(tensors, [
464
+ `${languagePrefix}.mlp.gate_proj.weight`,
465
+ `${genericPrefix}.mlp.gate_proj.weight`,
466
+ `${languagePrefix}.ffn.gate_proj.weight`,
467
+ `${languagePrefix}.ffn_gate.weight`,
468
+ `layers.${layerIdx}.feed_forward.w1.weight`,
469
+ ]);
470
+ const upShape = getDenseFfnTensorShape(tensors, [
471
+ `${languagePrefix}.mlp.up_proj.weight`,
472
+ `${genericPrefix}.mlp.up_proj.weight`,
473
+ `${languagePrefix}.ffn.up_proj.weight`,
474
+ `${languagePrefix}.ffn_up.weight`,
475
+ `layers.${layerIdx}.feed_forward.w3.weight`,
476
+ ]);
477
+ const downShape = getDenseFfnTensorShape(tensors, [
478
+ `${languagePrefix}.mlp.down_proj.weight`,
479
+ `${genericPrefix}.mlp.down_proj.weight`,
480
+ `${languagePrefix}.ffn.down_proj.weight`,
481
+ `${languagePrefix}.ffn_down.weight`,
482
+ `layers.${layerIdx}.feed_forward.w2.weight`,
483
+ ]);
484
+ const gateUpShape = getDenseFfnTensorShape(tensors, [
485
+ `${languagePrefix}.mlp.gate_up_proj.weight`,
486
+ `${genericPrefix}.mlp.gate_up_proj.weight`,
487
+ `${languagePrefix}.ffn.gate_up_proj.weight`,
488
+ `${languagePrefix}.ffn_gate_up.weight`,
489
+ `layers.${layerIdx}.feed_forward.w1_w3.weight`,
490
+ ]);
491
+
492
+ assertDenseFfnTensorShape(
493
+ modelId,
494
+ layerIdx,
495
+ 'gate weight',
496
+ gateShape,
497
+ [expectedIntermediateSize, hiddenSize]
498
+ );
499
+ assertDenseFfnTensorShape(
500
+ modelId,
501
+ layerIdx,
502
+ 'up weight',
503
+ upShape,
504
+ [expectedIntermediateSize, hiddenSize]
505
+ );
506
+ assertDenseFfnTensorShape(
507
+ modelId,
508
+ layerIdx,
509
+ 'down weight',
510
+ downShape,
511
+ [hiddenSize, expectedIntermediateSize]
512
+ );
513
+ assertDenseFfnTensorShape(
514
+ modelId,
515
+ layerIdx,
516
+ 'gate_up weight',
517
+ gateUpShape,
518
+ [expectedIntermediateSize * 2, hiddenSize]
519
+ );
520
+ }
521
+ }
522
+
523
+ export function resolveLayerIntermediateSize(config, layerIdx) {
524
+ const intermediateSizes = Array.isArray(config?.intermediateSizes) ? config.intermediateSizes : null;
525
+ if (Number.isFinite(layerIdx) && intermediateSizes) {
526
+ const resolved = intermediateSizes[Math.trunc(layerIdx)];
527
+ if (Number.isFinite(resolved) && resolved > 0) {
528
+ return Math.trunc(resolved);
529
+ }
530
+ }
531
+
532
+ const fallback = Number(config?.intermediateSize);
533
+ if (!Number.isFinite(fallback) || fallback <= 0) {
534
+ throw new Error(`Invalid modelConfig.intermediateSize: ${String(config?.intermediateSize)}`);
535
+ }
536
+ return Math.trunc(fallback);
537
+ }
538
+
192
539
  // =============================================================================
193
540
  // Manifest-First Config Resolution (NEW)
194
541
  // =============================================================================
@@ -217,6 +564,9 @@ export function validateRequiredInferenceFields(inf, modelId) {
217
564
  if (inf.attention.queryKeyNorm == null) {
218
565
  errors.push('attention.queryKeyNorm is required');
219
566
  }
567
+ if (inf.attention.valueNorm == null) {
568
+ errors.push('attention.valueNorm is required');
569
+ }
220
570
  if (inf.attention.attentionBias == null) {
221
571
  errors.push('attention.attentionBias is required');
222
572
  }
@@ -255,6 +605,9 @@ export function validateRequiredInferenceFields(inf, modelId) {
255
605
  if (inf.ffn.gatedActivation == null) {
256
606
  errors.push('ffn.gatedActivation is required');
257
607
  }
608
+ if (inf.ffn.useDoubleWideMlp == null) {
609
+ errors.push('ffn.useDoubleWideMlp is required');
610
+ }
258
611
  if (inf.ffn.swigluLimit === undefined) {
259
612
  errors.push('ffn.swigluLimit must be explicitly set (null for no limit, or number)');
260
613
  } else {
@@ -278,6 +631,13 @@ export function validateRequiredInferenceFields(inf, modelId) {
278
631
  if (inf.rope.ropeLocalTheta === undefined) {
279
632
  errors.push('rope.ropeLocalTheta must be explicitly set (null for no local theta, or number)');
280
633
  }
634
+ if (
635
+ inf.rope.ropeInterleaved !== undefined
636
+ && inf.rope.ropeInterleaved != null
637
+ && typeof inf.rope.ropeInterleaved !== 'boolean'
638
+ ) {
639
+ errors.push('rope.ropeInterleaved must be boolean when provided');
640
+ }
281
641
  if (inf.rope.mropeInterleaved == null) {
282
642
  errors.push('rope.mropeInterleaved is required');
283
643
  }
@@ -292,6 +652,30 @@ export function validateRequiredInferenceFields(inf, modelId) {
292
652
  errors.push('rope.partialRotaryFactor must be a number in (0, 1] or null');
293
653
  }
294
654
  }
655
+ if (inf.rope.ropeLocalPartialRotaryFactor === undefined) {
656
+ errors.push('rope.ropeLocalPartialRotaryFactor must be explicitly set (null when unused, or a number in (0, 1])');
657
+ } else {
658
+ const factor = inf.rope.ropeLocalPartialRotaryFactor;
659
+ if (factor !== null && (typeof factor !== 'number' || Number.isNaN(factor) || factor <= 0 || factor > 1)) {
660
+ errors.push('rope.ropeLocalPartialRotaryFactor must be a number in (0, 1] or null');
661
+ }
662
+ }
663
+ if (inf.rope.ropeFrequencyBaseDim === undefined) {
664
+ errors.push('rope.ropeFrequencyBaseDim must be explicitly set (null when using rotary dim, or a positive even integer)');
665
+ } else {
666
+ const dim = inf.rope.ropeFrequencyBaseDim;
667
+ if (dim !== null && (typeof dim !== 'number' || Number.isNaN(dim) || dim <= 0 || (Math.trunc(dim) % 2) !== 0)) {
668
+ errors.push('rope.ropeFrequencyBaseDim must be a positive even integer or null');
669
+ }
670
+ }
671
+ if (inf.rope.ropeLocalFrequencyBaseDim === undefined) {
672
+ errors.push('rope.ropeLocalFrequencyBaseDim must be explicitly set (null when using local rotary dim, or a positive even integer)');
673
+ } else {
674
+ const dim = inf.rope.ropeLocalFrequencyBaseDim;
675
+ if (dim !== null && (typeof dim !== 'number' || Number.isNaN(dim) || dim <= 0 || (Math.trunc(dim) % 2) !== 0)) {
676
+ errors.push('rope.ropeLocalFrequencyBaseDim must be a positive even integer or null');
677
+ }
678
+ }
295
679
 
296
680
  // Output fields - non-nullable required
297
681
  if (inf.output.tieWordEmbeddings == null) {
@@ -391,6 +775,15 @@ export function validateRequiredInferenceFields(inf, modelId) {
391
775
  if (inf.rope.yarnOriginalMaxPos === undefined) {
392
776
  errors.push('rope.yarnOriginalMaxPos must be explicitly set (null if not YARN)');
393
777
  }
778
+ if (inf.rope.ropeLocalYarnBetaFast === undefined) {
779
+ errors.push('rope.ropeLocalYarnBetaFast must be explicitly set (null if not local YARN)');
780
+ }
781
+ if (inf.rope.ropeLocalYarnBetaSlow === undefined) {
782
+ errors.push('rope.ropeLocalYarnBetaSlow must be explicitly set (null if not local YARN)');
783
+ }
784
+ if (inf.rope.ropeLocalYarnOriginalMaxPos === undefined) {
785
+ errors.push('rope.ropeLocalYarnOriginalMaxPos must be explicitly set (null if not local YARN)');
786
+ }
394
787
 
395
788
  if (errors.length > 0) {
396
789
  throw new Error(
@@ -452,18 +845,234 @@ function resolveVisionConfig(rawConfig, manifest) {
452
845
  );
453
846
  return null;
454
847
  }
848
+ const modelId = manifest?.modelId ?? 'unknown';
849
+ const resolveRequiredVisionField = (keys, label) => {
850
+ for (const key of keys) {
851
+ if (vc[key] !== undefined) {
852
+ return vc[key];
853
+ }
854
+ }
855
+ throw new Error(
856
+ `Manifest "${modelId}" is missing vision_config.${label}. ` +
857
+ 'Re-convert the model with explicit vision config metadata.'
858
+ );
859
+ };
860
+ const resolveRequiredPositiveInteger = (keys, label) => {
861
+ const value = resolveRequiredVisionField(keys, label);
862
+ const number = Number(value);
863
+ if (!Number.isFinite(number) || number <= 0 || Math.floor(number) !== number) {
864
+ throw new Error(
865
+ `Manifest "${modelId}" has invalid vision_config.${label}=${JSON.stringify(value)}. ` +
866
+ 'Expected a positive integer.'
867
+ );
868
+ }
869
+ return Math.trunc(number);
870
+ };
871
+ const resolveRequiredPositiveNumber = (value, label) => {
872
+ const number = Number(value);
873
+ if (!Number.isFinite(number) || number <= 0) {
874
+ throw new Error(
875
+ `Manifest "${modelId}" has invalid ${label}=${JSON.stringify(value)}. ` +
876
+ 'Expected a positive number.'
877
+ );
878
+ }
879
+ return number;
880
+ };
881
+ const visionArchitecture = String(resolveRequiredVisionField(['vision_architecture'], 'vision_architecture')).trim();
882
+ if (visionArchitecture !== 'gemma4' && visionArchitecture !== 'qwen3vl') {
883
+ throw new Error(
884
+ `Manifest "${modelId}" has unsupported vision_config.vision_architecture="${visionArchitecture}". ` +
885
+ 'Supported: "gemma4", "qwen3vl".'
886
+ );
887
+ }
888
+
889
+ if (visionArchitecture === 'gemma4') {
890
+ const hiddenSize = resolveRequiredPositiveInteger(['hidden_size'], 'hidden_size');
891
+ const numHeads = resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_attention_heads');
892
+ const ropeParameters = vc.rope_parameters;
893
+ if (!ropeParameters || typeof ropeParameters !== 'object') {
894
+ throw new Error(
895
+ `Manifest "${modelId}" is missing vision_config.rope_parameters. ` +
896
+ 'Re-convert the model with explicit Gemma 4 vision RoPE metadata.'
897
+ );
898
+ }
899
+ const hiddenActivation = String(resolveRequiredVisionField(['hidden_activation'], 'hidden_activation')).trim();
900
+ if (hiddenActivation !== 'gelu' && hiddenActivation !== 'gelu_pytorch_tanh') {
901
+ throw new Error(
902
+ `Manifest "${modelId}" has unsupported Gemma 4 vision hidden_activation="${hiddenActivation}". ` +
903
+ 'Supported values: "gelu", "gelu_pytorch_tanh".'
904
+ );
905
+ }
906
+ if (vc.standardize === true) {
907
+ throw new Error(
908
+ `Manifest "${modelId}" enables vision_config.standardize, but Gemma 4 runtime preprocessing does not support it yet.`
909
+ );
910
+ }
911
+ if (vc.use_clipped_linears !== true) {
912
+ throw new Error(
913
+ `Manifest "${modelId}" requires vision_config.use_clipped_linears=true for Gemma 4 vision weights.`
914
+ );
915
+ }
916
+
917
+ return {
918
+ depth: resolveRequiredPositiveInteger(['depth', 'num_hidden_layers'], 'num_hidden_layers'),
919
+ hiddenSize,
920
+ intermediateSize: resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size'),
921
+ numHeads,
922
+ numKeyValueHeads: resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads'),
923
+ headDim: resolveRequiredPositiveInteger(['head_dim', 'global_head_dim'], 'head_dim'),
924
+ outHiddenSize: vc.out_hidden_size ?? vc.output_proj_dims ?? null,
925
+ patchSize: resolveRequiredPositiveInteger(['patch_size'], 'patch_size'),
926
+ poolingKernelSize: resolveRequiredPositiveInteger(['pooling_kernel_size'], 'pooling_kernel_size'),
927
+ spatialMergeSize: vc.spatial_merge_size ?? null,
928
+ temporalPatchSize: vc.temporal_patch_size ?? null,
929
+ positionEmbeddingSize: resolveRequiredPositiveInteger(['position_embedding_size'], 'position_embedding_size'),
930
+ defaultOutputLength: resolveRequiredPositiveInteger(['default_output_length'], 'default_output_length'),
931
+ ropeTheta: resolveRequiredPositiveNumber(ropeParameters.rope_theta, 'vision_config.rope_parameters.rope_theta'),
932
+ eps: resolveRequiredPositiveNumber(
933
+ resolveRequiredVisionField(['eps', 'rms_norm_eps'], 'rms_norm_eps'),
934
+ 'vision_config.rms_norm_eps'
935
+ ),
936
+ hiddenActivation,
937
+ standardize: false,
938
+ useClippedLinears: true,
939
+ deepstackVisualIndexes: [],
940
+ imageTokenId: rawConfig?.image_token_id ?? manifest?.image_token_id ?? null,
941
+ visionArchitecture,
942
+ softTokenBudgetTiers: Array.isArray(vc.soft_token_budget_tiers)
943
+ ? vc.soft_token_budget_tiers.map(Number).filter((n) => Number.isFinite(n) && n > 0)
944
+ : [70, 140, 280, 560, 1120],
945
+ };
946
+ }
947
+
948
+ const hiddenSize = resolveRequiredPositiveInteger(['hidden_size'], 'hidden_size');
949
+ const intermediateSize = resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size');
950
+ const numHeads = resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_heads');
951
+ const outHiddenSize = resolveRequiredPositiveInteger(['out_hidden_size', 'output_proj_dims'], 'out_hidden_size');
952
+ const patchSize = resolveRequiredPositiveInteger(['patch_size'], 'patch_size');
953
+ const spatialMergeSize = resolveRequiredPositiveInteger(['spatial_merge_size', 'merge_size'], 'spatial_merge_size');
954
+ const temporalPatchSize = resolveRequiredPositiveInteger(['temporal_patch_size'], 'temporal_patch_size');
955
+ const eps = resolveRequiredPositiveNumber(
956
+ resolveRequiredVisionField(['eps', 'rms_norm_eps'], 'eps'),
957
+ 'vision_config.eps'
958
+ );
959
+ const hiddenActivation = String(
960
+ resolveRequiredVisionField(['hidden_activation', 'hidden_act'], 'hidden_activation')
961
+ ).trim();
962
+ const minPixels = resolveRequiredPositiveInteger(['min_pixels'], 'min_pixels');
963
+ const maxPixels = resolveRequiredPositiveInteger(['max_pixels'], 'max_pixels');
964
+ const normalization = vc.normalization;
965
+ if (!normalization || typeof normalization !== 'object') {
966
+ throw new Error(
967
+ `Manifest "${modelId}" is missing vision_config.normalization. ` +
968
+ 'Re-convert the model with explicit normalization metadata.'
969
+ );
970
+ }
971
+ if (!Array.isArray(normalization.mean) || normalization.mean.length !== 3) {
972
+ throw new Error(
973
+ `Manifest "${modelId}" requires vision_config.normalization.mean to contain exactly 3 values.`
974
+ );
975
+ }
976
+ if (!Array.isArray(normalization.std) || normalization.std.length !== 3) {
977
+ throw new Error(
978
+ `Manifest "${modelId}" requires vision_config.normalization.std to contain exactly 3 values.`
979
+ );
980
+ }
455
981
  return {
456
- depth: vc.depth ?? 24,
457
- hiddenSize: vc.hidden_size ?? 1024,
458
- intermediateSize: vc.intermediate_size ?? 4096,
459
- numHeads: vc.num_heads ?? 16,
460
- outHiddenSize: vc.out_hidden_size ?? vc.hidden_size ?? 1024,
461
- patchSize: vc.patch_size ?? 16,
462
- spatialMergeSize: vc.spatial_merge_size ?? 2,
463
- temporalPatchSize: vc.temporal_patch_size ?? 2,
464
- eps: vc.eps ?? 1e-6,
982
+ depth: resolveRequiredPositiveInteger(['depth', 'num_hidden_layers'], 'depth'),
983
+ hiddenSize,
984
+ intermediateSize,
985
+ numHeads,
986
+ numKeyValueHeads: resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads'),
987
+ headDim: resolveRequiredPositiveInteger(['head_dim'], 'head_dim'),
988
+ outHiddenSize,
989
+ patchSize,
990
+ poolingKernelSize: resolveRequiredPositiveInteger(['pooling_kernel_size'], 'pooling_kernel_size'),
991
+ spatialMergeSize,
992
+ temporalPatchSize,
993
+ positionEmbeddingSize: vc.position_embedding_size ?? null,
994
+ defaultOutputLength: vc.default_output_length ?? null,
995
+ ropeTheta: vc.rope_parameters?.rope_theta ?? null,
996
+ eps,
997
+ hiddenActivation,
998
+ standardize: vc.standardize === true,
999
+ useClippedLinears: vc.use_clipped_linears === true,
1000
+ minPixels,
1001
+ maxPixels,
1002
+ normalization,
465
1003
  deepstackVisualIndexes: Array.isArray(vc.deepstack_visual_indexes) ? vc.deepstack_visual_indexes : [],
466
1004
  imageTokenId: rawConfig?.image_token_id ?? manifest?.image_token_id ?? null,
1005
+ visionArchitecture,
1006
+ };
1007
+ }
1008
+
1009
+ function resolveAudioConfig(rawConfig, manifest) {
1010
+ const ac = rawConfig?.audio_config ?? manifest?.config?.audio_config;
1011
+ if (!ac || typeof ac !== 'object') {
1012
+ log.debug(
1013
+ 'Config',
1014
+ `Audio config not present for model "${manifest?.modelId ?? 'unknown'}"; audio pipeline disabled.`
1015
+ );
1016
+ return null;
1017
+ }
1018
+ const modelId = manifest?.modelId ?? 'unknown';
1019
+ const resolveRequiredPositiveInteger = (value, label) => {
1020
+ const number = Number(value);
1021
+ if (!Number.isFinite(number) || number <= 0 || Math.floor(number) !== number) {
1022
+ throw new Error(
1023
+ `Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
1024
+ 'Expected a positive integer.'
1025
+ );
1026
+ }
1027
+ return Math.trunc(number);
1028
+ };
1029
+ const resolveRequiredPositiveNumber = (value, label) => {
1030
+ const number = Number(value);
1031
+ if (!Number.isFinite(number) || number <= 0) {
1032
+ throw new Error(
1033
+ `Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
1034
+ 'Expected a positive number.'
1035
+ );
1036
+ }
1037
+ return number;
1038
+ };
1039
+ const audioArchitecture = String(ac.audio_architecture ?? '').trim();
1040
+ if (audioArchitecture !== 'gemma4') {
1041
+ throw new Error(
1042
+ `Manifest "${modelId}" has unsupported audio_config.audio_architecture="${audioArchitecture}". ` +
1043
+ 'Supported: "gemma4".'
1044
+ );
1045
+ }
1046
+
1047
+ const hiddenSize = resolveRequiredPositiveInteger(ac.hidden_size, 'hidden_size');
1048
+ const numAttentionHeads = resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads');
1049
+ const headDim = Math.trunc(hiddenSize / numAttentionHeads);
1050
+
1051
+ if (!Array.isArray(ac.subsampling_conv_channels) || ac.subsampling_conv_channels.length < 1) {
1052
+ throw new Error(
1053
+ `Manifest "${modelId}" is missing audio_config.subsampling_conv_channels array.`
1054
+ );
1055
+ }
1056
+
1057
+ return {
1058
+ audioArchitecture,
1059
+ depth: resolveRequiredPositiveInteger(ac.num_hidden_layers, 'num_hidden_layers'),
1060
+ hiddenSize,
1061
+ numAttentionHeads,
1062
+ headDim,
1063
+ convKernelSize: resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size'),
1064
+ subsamplingConvChannels: ac.subsampling_conv_channels.map(Number),
1065
+ outputProjDims: resolveRequiredPositiveInteger(ac.output_proj_dims, 'output_proj_dims'),
1066
+ attentionContextLeft: resolveRequiredPositiveInteger(ac.attention_context_left, 'attention_context_left'),
1067
+ attentionContextRight: Number(ac.attention_context_right ?? 0),
1068
+ attentionChunkSize: resolveRequiredPositiveInteger(ac.attention_chunk_size, 'attention_chunk_size'),
1069
+ attentionLogitCap: resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap'),
1070
+ attentionInvalidLogitsValue: Number(ac.attention_invalid_logits_value ?? -1e9),
1071
+ residualWeight: resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight'),
1072
+ rmsNormEps: resolveRequiredPositiveNumber(ac.rms_norm_eps ?? 1e-6, 'rms_norm_eps'),
1073
+ hiddenAct: String(ac.hidden_act ?? 'silu').trim(),
1074
+ useClippedLinears: ac.use_clipped_linears === true,
1075
+ audioTokenId: rawConfig?.audio_token_id ?? manifest?.audio_token_id ?? null,
467
1076
  };
468
1077
  }
469
1078
 
@@ -512,8 +1121,20 @@ function parseLinearNormMode(value, sharedFlag = null, modelId = 'unknown') {
512
1121
 
513
1122
 
514
1123
  export function toParsedConfigFromMerged(merged, manifest) {
515
- const rawConfig = manifest.config ?? {};
516
- const config = rawConfig.text_config ?? rawConfig;
1124
+ const mergedConfig = merged?.config ?? manifest.config ?? {};
1125
+ const rawConfig = mergedConfig.text_config ?? mergedConfig;
1126
+ const config = {
1127
+ ...rawConfig,
1128
+ ...(merged?.vision_config !== null && merged?.vision_config !== undefined
1129
+ ? { vision_config: merged.vision_config }
1130
+ : {}),
1131
+ ...(merged?.audio_config !== null && merged?.audio_config !== undefined
1132
+ ? { audio_config: merged.audio_config }
1133
+ : {}),
1134
+ ...(merged?.quantization_config !== null && merged?.quantization_config !== undefined
1135
+ ? { quantization_config: merged.quantization_config }
1136
+ : {}),
1137
+ };
517
1138
  const inf = merged.inference;
518
1139
 
519
1140
  // Validate required fields are present (fail fast on incomplete manifests)
@@ -533,6 +1154,37 @@ export function toParsedConfigFromMerged(merged, manifest) {
533
1154
  );
534
1155
  }
535
1156
  const resolvedIntermediateSize = resolveIntermediateSizeForRuntime(manifest, inf, arch, merged.modelId);
1157
+ const archNumHeads = Number(arch.numAttentionHeads ?? arch.numHeads);
1158
+ const archNumKVHeads = Number(arch.numKeyValueHeads ?? arch.numKVHeads);
1159
+ const archHeadDim = Number(arch.headDim);
1160
+ const archGlobalHeadDimRaw = arch.globalHeadDim ?? null;
1161
+ const archGlobalHeadDim = (
1162
+ typeof archGlobalHeadDimRaw === 'number' && Number.isFinite(archGlobalHeadDimRaw) && archGlobalHeadDimRaw > 0
1163
+ )
1164
+ ? Math.trunc(archGlobalHeadDimRaw)
1165
+ : null;
1166
+ const archNumKvSharedLayersRaw = arch.numKvSharedLayers ?? 0;
1167
+ const archNumKvSharedLayers = (
1168
+ typeof archNumKvSharedLayersRaw === 'number'
1169
+ && Number.isFinite(archNumKvSharedLayersRaw)
1170
+ && archNumKvSharedLayersRaw >= 0
1171
+ )
1172
+ ? Math.trunc(archNumKvSharedLayersRaw)
1173
+ : 0;
1174
+ const intermediateSizes = buildPerLayerIntermediateSizes({
1175
+ baseIntermediateSize: resolvedIntermediateSize,
1176
+ numLayers: arch.numLayers,
1177
+ numKvSharedLayers: archNumKvSharedLayers,
1178
+ useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
1179
+ modelId: merged.modelId,
1180
+ });
1181
+ const maxIntermediateSize = Math.max(...intermediateSizes);
1182
+ validateLayerIntermediateSizesAgainstManifest(
1183
+ manifest,
1184
+ arch.hiddenSize,
1185
+ intermediateSizes,
1186
+ merged.modelId
1187
+ );
536
1188
 
537
1189
  // Compute layer types from layerPattern
538
1190
 
@@ -596,13 +1248,13 @@ export function toParsedConfigFromMerged(merged, manifest) {
596
1248
 
597
1249
  // Compute queryPreAttnScalar from manifest inference (NOT from family detection)
598
1250
  // Manifest-first: queryPreAttnScalar is required in ManifestAttentionSchema
599
- const headDim = arch.headDim;
1251
+ const headDim = archHeadDim;
600
1252
  const queryPreAttnScalar = inf.attention.queryPreAttnScalar;
601
1253
  const causalAttention = inf.attention.causal;
602
1254
 
603
- // Cross-field sanity: queryPreAttnScalar should typically equal headDim.
604
- // A value of sqrt(headDim) indicates a known converter bug that produces
605
- // attnScale = 1/sqrt(sqrt(headDim)) instead of the correct 1/sqrt(headDim).
1255
+ // Preserve the manifest scalar exactly. Gemma-family models legitimately use
1256
+ // queryPreAttnScalar=1, but sqrt(headDim) is still a known converter bug that
1257
+ // produces attnScale = 1/sqrt(sqrt(headDim)) instead of the intended value.
606
1258
  if (queryPreAttnScalar != null && headDim != null
607
1259
  && queryPreAttnScalar !== headDim
608
1260
  && Math.abs(queryPreAttnScalar - Math.sqrt(headDim)) < 0.01) {
@@ -653,8 +1305,9 @@ export function toParsedConfigFromMerged(merged, manifest) {
653
1305
  const ropeLocalScale = inf.rope.ropeLocalScalingFactor;
654
1306
  const ropeLocalScalingType = inf.rope.ropeLocalScalingType;
655
1307
  const partialRotaryFactor = inf.rope.partialRotaryFactor;
1308
+ const ropeLocalPartialRotaryFactor = inf.rope.ropeLocalPartialRotaryFactor;
656
1309
  const mropeInterleaved = inf.rope.mropeInterleaved === true;
657
- const ropeInterleaved = false;
1310
+ const ropeInterleaved = inf.rope.ropeInterleaved === true;
658
1311
 
659
1312
  if (ropeLocalScale == null && (inf.rope.ropeLocalTheta != null || inf.rope.mropeSection != null)) {
660
1313
  throw new Error(
@@ -665,7 +1318,22 @@ export function toParsedConfigFromMerged(merged, manifest) {
665
1318
  const mropeSection = Array.isArray(inf.rope.mropeSection)
666
1319
  ? inf.rope.mropeSection.map((entry) => Math.trunc(Number(entry)))
667
1320
  : null;
668
- const ropeRotaryDim = resolveRotaryDim(arch.headDim, partialRotaryFactor, merged.modelId);
1321
+ const ropeRotaryDim = resolveRotaryDim(archGlobalHeadDim ?? archHeadDim, partialRotaryFactor, merged.modelId);
1322
+ const ropeLocalRotaryDim = resolveRotaryDim(archHeadDim, ropeLocalPartialRotaryFactor, merged.modelId);
1323
+ const ropeFrequencyBaseDim = resolveFrequencyBaseDim(
1324
+ archGlobalHeadDim ?? archHeadDim,
1325
+ ropeRotaryDim,
1326
+ inf.rope.ropeFrequencyBaseDim,
1327
+ merged.modelId,
1328
+ 'rope.ropeFrequencyBaseDim'
1329
+ );
1330
+ const ropeLocalFrequencyBaseDim = resolveFrequencyBaseDim(
1331
+ archHeadDim,
1332
+ ropeLocalRotaryDim,
1333
+ inf.rope.ropeLocalFrequencyBaseDim,
1334
+ merged.modelId,
1335
+ 'rope.ropeLocalFrequencyBaseDim'
1336
+ );
669
1337
  if (mropeSection && mropeSection.some((entry) => !Number.isFinite(entry) || entry <= 0)) {
670
1338
  throw new Error(
671
1339
  `Manifest "${merged.modelId}" has invalid rope.mropeSection; expected positive integers.`
@@ -722,6 +1390,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
722
1390
  const chatTemplateType = inf.chatTemplate.type;
723
1391
  validateChatTemplateType(chatTemplateType, merged.modelId);
724
1392
  const chatTemplateEnabled = inf.chatTemplate.enabled;
1393
+ const chatTemplateThinking = inf.chatTemplate.thinking ?? null;
725
1394
  const parsePositiveInt = (value) => {
726
1395
  const num = Number(value);
727
1396
  if (!Number.isFinite(num) || num <= 0) return null;
@@ -738,15 +1407,28 @@ export function toParsedConfigFromMerged(merged, manifest) {
738
1407
  config.linear_norm_shared,
739
1408
  merged.modelId
740
1409
  );
1410
+ const hasMixedAttentionGeometry = archGlobalHeadDim != null && archGlobalHeadDim !== archHeadDim;
1411
+ const hasSharedKvLayers = archNumKvSharedLayers > 0;
1412
+ const hasExplicitLayerTypes = Array.isArray(layerTypes) && layerTypes.length === arch.numLayers;
1413
+ const decodeStrategy = (hasMixedAttentionGeometry || hasSharedKvLayers) && !hasExplicitLayerTypes
1414
+ ? 'replay_prefill'
1415
+ : 'incremental';
1416
+ const perLayerInputsSession = resolvePerLayerInputsSession(inf, merged.modelId);
741
1417
 
742
1418
  return {
743
1419
  numLayers: arch.numLayers,
744
1420
  hiddenSize: arch.hiddenSize,
745
1421
  intermediateSize: resolvedIntermediateSize,
746
- numHeads: arch.numAttentionHeads,
747
- numKVHeads: arch.numKeyValueHeads,
748
- headDim: arch.headDim,
1422
+ intermediateSizes,
1423
+ maxIntermediateSize,
1424
+ numHeads: archNumHeads,
1425
+ numKVHeads: archNumKVHeads,
1426
+ headDim: archHeadDim,
1427
+ globalHeadDim: archGlobalHeadDim,
749
1428
  vocabSize: arch.vocabSize,
1429
+ hiddenSizePerLayerInput: arch.hiddenSizePerLayerInput ?? null,
1430
+ vocabSizePerLayerInput: arch.vocabSizePerLayerInput ?? null,
1431
+ numKvSharedLayers: archNumKvSharedLayers,
750
1432
  maxSeqLen: arch.maxSeqLen,
751
1433
  useMoE,
752
1434
  numExperts,
@@ -756,10 +1438,14 @@ export function toParsedConfigFromMerged(merged, manifest) {
756
1438
  ropeTheta: inf.rope.ropeTheta,
757
1439
  ropeLocalTheta: inf.rope.ropeLocalTheta,
758
1440
  ropeRotaryDim,
1441
+ ropeLocalRotaryDim,
1442
+ ropeFrequencyBaseDim,
1443
+ ropeLocalFrequencyBaseDim,
759
1444
  ropeInterleaved,
760
1445
  mropeInterleaved,
761
1446
  mropeSection,
762
1447
  partialRotaryFactor,
1448
+ ropeLocalPartialRotaryFactor,
763
1449
  ropeScale,
764
1450
  ropeLocalScale,
765
1451
  ropeScalingType,
@@ -779,6 +1465,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
779
1465
  embeddingVocabSize: inf.output.embeddingVocabSize,
780
1466
  embeddingPostprocessor,
781
1467
  hiddenActivation,
1468
+ useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
782
1469
  swigluLimit: inf.ffn.swigluLimit,
783
1470
  stopTokenIds,
784
1471
  layerTypes,
@@ -793,13 +1480,18 @@ export function toParsedConfigFromMerged(merged, manifest) {
793
1480
  finalLogitSoftcapping: inf.output.finalLogitSoftcapping,
794
1481
  attnLogitSoftcapping: inf.attention.attnLogitSoftcapping,
795
1482
  queryKeyNorm: inf.attention.queryKeyNorm,
1483
+ valueNorm: inf.attention.valueNorm,
796
1484
  attentionOutputGate: inf.attention.attentionOutputGate === true,
797
1485
  queryPreAttnScalar,
798
1486
  layerPipeline: inf.pipeline ?? null,
799
1487
  chatTemplateType,
800
1488
  chatTemplateEnabled,
1489
+ chatTemplateThinking,
1490
+ decodeStrategy,
1491
+ perLayerInputsSession,
801
1492
  kernelPath: null,
802
1493
  visionConfig: resolveVisionConfig(config, manifest),
1494
+ audioConfig: resolveAudioConfig(config, manifest),
803
1495
  };
804
1496
  }
805
1497
 
@@ -813,6 +1505,8 @@ export function parseModelConfigFromManifest(manifest, runtimeOverrides) {
813
1505
  modelId: manifest.modelId ?? 'unknown',
814
1506
  inference: manifest.inference,
815
1507
  architecture: manifest.architecture,
1508
+ config: manifest.config ?? null,
1509
+ quantization_config: manifest.quantization_config ?? null,
816
1510
  },
817
1511
  runtimeOverrides
818
1512
  );