doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -17,6 +17,17 @@ export interface DtypeConsistencyResult {
17
17
  };
18
18
  }
19
19
 
20
+ export interface ImplicitDtypeTransitionOptions {
21
+ executionPolicies?: {
22
+ dtypeTransition?: 'require_cast_step' | null;
23
+ } | null;
24
+ fromDtype?: 'f16' | 'f32' | null;
25
+ toDtype?: 'f16' | 'f32' | null;
26
+ op?: string | null;
27
+ detail?: string | null;
28
+ transitionDeclaredBy?: 'step_precision' | null;
29
+ }
30
+
20
31
  /**
21
32
  * Resolve activation dtype from all available sources.
22
33
  * Returns { activationDtype, source, allSources } for diagnostics.
@@ -36,3 +47,7 @@ export declare function assertDtypeConsistency(
36
47
  runtimeConfig: Record<string, unknown> | null,
37
48
  layerContext: Record<string, unknown> | null
38
49
  ): DtypeConsistencyResult;
50
+
51
+ export declare function assertImplicitDtypeTransitionAllowed(
52
+ options?: ImplicitDtypeTransitionOptions
53
+ ): void;
@@ -1,6 +1,17 @@
1
1
 
2
2
  import { log } from '../../../debug/index.js';
3
3
 
4
+ function normalizeDtype(value, label) {
5
+ if (value == null) {
6
+ return null;
7
+ }
8
+ const normalized = String(value).trim().toLowerCase();
9
+ if (normalized !== 'f16' && normalized !== 'f32') {
10
+ throw new Error(`[DtypeContract] ${label} must be "f16" or "f32"; got "${value}".`);
11
+ }
12
+ return normalized;
13
+ }
14
+
4
15
  /**
5
16
  * Resolve activation dtype from all available sources.
6
17
  *
@@ -85,3 +96,30 @@ export function assertDtypeConsistency(executionPlanState, runtimeConfig, layerC
85
96
 
86
97
  return { consistent, values };
87
98
  }
99
+
100
+ export function assertImplicitDtypeTransitionAllowed(options = {}) {
101
+ const policy = options.executionPolicies?.dtypeTransition ?? null;
102
+ if (policy !== 'require_cast_step') {
103
+ return;
104
+ }
105
+ if (options.transitionDeclaredBy === 'step_precision') {
106
+ return;
107
+ }
108
+
109
+ const fromDtype = normalizeDtype(options.fromDtype, 'fromDtype');
110
+ const toDtype = normalizeDtype(options.toDtype, 'toDtype');
111
+ if (!fromDtype || !toDtype || fromDtype === toDtype) {
112
+ return;
113
+ }
114
+
115
+ const op = typeof options.op === 'string' && options.op.trim()
116
+ ? options.op.trim()
117
+ : 'operation';
118
+ const detail = typeof options.detail === 'string' && options.detail.trim()
119
+ ? ` ${options.detail.trim()}`
120
+ : '';
121
+ throw new Error(
122
+ `[ExecutionV1] ${op} requires implicit dtype transition ${fromDtype} -> ${toDtype}.${detail} ` +
123
+ `execution.policies.dtypeTransition="${policy}" requires an explicit cast step in manifest.inference.execution.`
124
+ );
125
+ }
@@ -4,6 +4,7 @@
4
4
 
5
5
  import type { CommandRecorder } from '../../../gpu/command-recorder.js';
6
6
  import type { ProbeConfigSchema } from '../../../config/schema/index.js';
7
+ import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
7
8
  import type { Tensor } from '../../../gpu/tensor.js';
8
9
  import type { CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
9
10
 
@@ -20,6 +21,9 @@ export interface EmbedConfig {
20
21
  transpose?: boolean;
21
22
  activationDtype?: 'f16' | 'f32';
22
23
  embeddingDtype?: 'f16' | 'f32';
24
+ executionPolicies?: ExecutionV1PoliciesSchema | null;
25
+ inputHiddenSize?: number;
26
+ hiddenOffset?: number;
23
27
  }
24
28
 
25
29
  export interface ValidationResult {
@@ -2,7 +2,7 @@
2
2
 
3
3
  import { getDevice, getKernelCapabilities } from '../../../gpu/device.js';
4
4
  import { acquireBuffer, releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
5
- import { runGather, recordGather } from '../../../gpu/kernel-selector.js';
5
+ import { runGather, recordGather, runScale, recordScale } from '../../../gpu/kernel-selector.js';
6
6
  import { log, trace } from '../../../debug/index.js';
7
7
  import { runProbes } from './probes.js';
8
8
  import { decodeReadback } from './debug-utils/index.js';
@@ -12,165 +12,65 @@ import { isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buff
12
12
  import { f16ToF32 } from '../../../loader/dtype-utils.js';
13
13
  import { selectRuleValue } from '../../../rules/rule-registry.js';
14
14
 
15
- const scaleShaderCode = `
16
- struct Uniforms { scale: f32, count: u32 }
17
- @group(0) @binding(0) var<uniform> uniforms: Uniforms;
18
- @group(0) @binding(1) var<storage, read> input: array<f32>;
19
- @group(0) @binding(2) var<storage, read_write> output: array<f32>;
15
+ const bf16ScratchU32 = new Uint32Array(1);
16
+ const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
20
17
 
21
- @compute @workgroup_size(256)
22
- fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
23
- if (gid.x >= uniforms.count) { return; }
24
- output[gid.x] = input[gid.x] * uniforms.scale;
25
- }
26
- `;
27
-
28
- // F16 scale shader: F16 input -> F16 output (for F16 activation mode)
29
- const scaleShaderCodeF16 = `
30
- enable f16;
31
-
32
- struct Uniforms { scale: f32, count: u32 }
33
- @group(0) @binding(0) var<uniform> uniforms: Uniforms;
34
- @group(0) @binding(1) var<storage, read> input: array<f16>;
35
- @group(0) @binding(2) var<storage, read_write> output: array<f16>;
36
-
37
- @compute @workgroup_size(256)
38
- fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
39
- if (gid.x >= uniforms.count) { return; }
40
- // Read F16, compute in F32 for precision, write F16
41
- output[gid.x] = f16(f32(input[gid.x]) * uniforms.scale);
42
- }
43
- `;
44
-
45
-
46
- let scalePipeline = null;
47
-
48
- let scalePipelineF16 = null;
49
-
50
-
51
- export function recordScale(recorder, inputBuffer, scale, count, useF16 = false) {
52
- const device = recorder.device;
53
- const bytesPerElement = useF16 ? 2 : 4;
54
- const outputBuffer = acquireBuffer(count * bytesPerElement, undefined, 'scaled_embed');
55
-
56
- const uniformData = new ArrayBuffer(8);
57
- const uniformView = new DataView(uniformData);
58
- uniformView.setFloat32(0, scale, true);
59
- uniformView.setUint32(4, count, true);
18
+ function bf16ToF32(value) {
19
+ bf16ScratchU32[0] = (value & 0xffff) << 16;
20
+ return bf16ScratchF32[0];
21
+ }
60
22
 
61
- const uniformBuffer = recorder.createUniformBuffer(uniformData, 'scale_uniforms');
23
+ export function isRangeBackedCpuEmbeddingSource(value) {
24
+ return (
25
+ typeof value === 'object'
26
+ && value !== null
27
+ && value.kind === 'tensor_range_source'
28
+ && typeof value.loadRange === 'function'
29
+ );
30
+ }
62
31
 
63
- // Select and cache appropriate pipeline
64
-
65
- let pipeline;
66
- if (useF16) {
67
- if (!scalePipelineF16) {
68
- const shaderModule = device.createShaderModule({ code: scaleShaderCodeF16 });
69
- scalePipelineF16 = device.createComputePipeline({
70
- layout: 'auto',
71
- compute: { module: shaderModule, entryPoint: 'main' },
72
- });
73
- }
74
- pipeline = scalePipelineF16;
75
- } else {
76
- if (!scalePipeline) {
77
- const shaderModule = device.createShaderModule({ code: scaleShaderCode });
78
- scalePipeline = device.createComputePipeline({
79
- layout: 'auto',
80
- compute: { module: shaderModule, entryPoint: 'main' },
81
- });
82
- }
83
- pipeline = scalePipeline;
32
+ export function normalizeRangeBytes(value, label) {
33
+ if (value instanceof Uint8Array) return value;
34
+ if (value instanceof ArrayBuffer) return new Uint8Array(value);
35
+ if (ArrayBuffer.isView(value)) {
36
+ return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
84
37
  }
85
-
86
- const bindGroup = device.createBindGroup({
87
- layout: pipeline.getBindGroupLayout(0),
88
- entries: [
89
- { binding: 0, resource: { buffer: uniformBuffer } },
90
- { binding: 1, resource: { buffer: inputBuffer } },
91
- { binding: 2, resource: { buffer: outputBuffer } },
92
- ],
93
- });
94
-
95
- const pass = recorder.beginComputePass('scale');
96
- pass.setPipeline(pipeline);
97
- pass.setBindGroup(0, bindGroup);
98
- pass.dispatchWorkgroups(Math.ceil(count / 256));
99
- pass.end();
100
-
101
- return outputBuffer;
38
+ throw new Error(`[Embed] ${label} returned unsupported byte payload type "${value?.constructor?.name ?? typeof value}".`);
102
39
  }
103
40
 
104
-
105
- export async function scaleGPUBuffer(inputBuffer, scale, count, useF16 = false) {
106
- const device = getDevice();
107
- if (!device) throw new Error('GPU device not available');
108
-
109
- const bytesPerElement = useF16 ? 2 : 4;
110
- const outputBuffer = acquireBuffer(count * bytesPerElement, undefined, 'scaled_embed');
111
-
112
- const uniformData = new ArrayBuffer(8);
113
- const uniformView = new DataView(uniformData);
114
- uniformView.setFloat32(0, scale, true);
115
- uniformView.setUint32(4, count, true);
116
-
117
- const uniformBuffer = device.createBuffer({
118
- label: 'scale_uniforms',
119
- size: 16,
120
- usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
121
- });
122
- device.queue.writeBuffer(uniformBuffer, 0, uniformData);
123
-
124
- // Select and cache appropriate pipeline
125
-
126
- let pipeline;
127
- if (useF16) {
128
- if (!scalePipelineF16) {
129
- const shaderModule = device.createShaderModule({ code: scaleShaderCodeF16 });
130
- scalePipelineF16 = device.createComputePipeline({
131
- layout: 'auto',
132
- compute: { module: shaderModule, entryPoint: 'main' },
133
- });
41
+ export function decodeRangeChunkIntoOutput(bytes, sourceDtype, output, dstOffset, hiddenSize) {
42
+ if (sourceDtype === 'f16') {
43
+ const values = new Uint16Array(bytes.buffer, bytes.byteOffset, hiddenSize);
44
+ for (let i = 0; i < hiddenSize; i++) {
45
+ output[dstOffset + i] = f16ToF32(values[i]);
134
46
  }
135
- pipeline = scalePipelineF16;
136
- } else {
137
- if (!scalePipeline) {
138
- const shaderModule = device.createShaderModule({ code: scaleShaderCode });
139
- scalePipeline = device.createComputePipeline({
140
- layout: 'auto',
141
- compute: { module: shaderModule, entryPoint: 'main' },
142
- });
47
+ return;
48
+ }
49
+ if (sourceDtype === 'bf16') {
50
+ const values = new Uint16Array(bytes.buffer, bytes.byteOffset, hiddenSize);
51
+ for (let i = 0; i < hiddenSize; i++) {
52
+ output[dstOffset + i] = bf16ToF32(values[i]);
143
53
  }
144
- pipeline = scalePipeline;
54
+ return;
55
+ }
56
+ if (((bytes.byteOffset % 4) === 0) && ((bytes.byteLength % 4) === 0)) {
57
+ output.set(new Float32Array(bytes.buffer, bytes.byteOffset, hiddenSize), dstOffset);
58
+ return;
59
+ }
60
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
61
+ for (let i = 0; i < hiddenSize; i++) {
62
+ output[dstOffset + i] = view.getFloat32(i * 4, true);
145
63
  }
146
-
147
- const bindGroup = device.createBindGroup({
148
- layout: pipeline.getBindGroupLayout(0),
149
- entries: [
150
- { binding: 0, resource: { buffer: uniformBuffer } },
151
- { binding: 1, resource: { buffer: inputBuffer } },
152
- { binding: 2, resource: { buffer: outputBuffer } },
153
- ],
154
- });
155
-
156
- const encoder = device.createCommandEncoder();
157
- const pass = encoder.beginComputePass();
158
- pass.setPipeline(pipeline);
159
- pass.setBindGroup(0, bindGroup);
160
- pass.dispatchWorkgroups(Math.ceil(count / 256));
161
- pass.end();
162
-
163
- device.queue.submit([encoder.finish()]);
164
-
165
- // CRITICAL: Wait for GPU to complete scale operation before returning
166
- // Without this, the caller may read stale data from outputBuffer
167
- await device.queue.onSubmittedWorkDone();
168
-
169
- uniformBuffer.destroy();
170
-
171
- return outputBuffer;
172
64
  }
173
65
 
66
+ async function readGpuTokenIdsForCpuEmbeddingGather(tokenIds, numTokens, indexOffset) {
67
+ if (numTokens <= 0) {
68
+ throw new Error('[Embed] numTokens must be provided when tokenIds is a GPUBuffer.');
69
+ }
70
+ const totalTokenCount = indexOffset + numTokens;
71
+ const readback = await readBuffer(tokenIds, totalTokenCount * Uint32Array.BYTES_PER_ELEMENT);
72
+ return new Uint32Array(readback).subarray(indexOffset, totalTokenCount);
73
+ }
174
74
 
175
75
  export async function embed(tokenIds, embedBuffer, config) {
176
76
  const {
@@ -183,10 +83,13 @@ export async function embed(tokenIds, embedBuffer, config) {
183
83
  transpose = false,
184
84
  activationDtype,
185
85
  embeddingDtype,
86
+ operatorDiagnostics,
87
+ inputHiddenSize = hiddenSize,
88
+ hiddenOffset = 0,
186
89
  } = config;
187
90
  const device = getDevice();
188
91
  const tokenBufferInput = isGpuBufferInstance(tokenIds);
189
- const tokenIdArray = tokenBufferInput ? null : (tokenIds);
92
+ let tokenIdArray = tokenBufferInput ? null : (tokenIds);
190
93
  const numTokens = tokenBufferInput
191
94
  ? (config.numTokens ?? 0)
192
95
  : (tokenIdArray?.length ?? 0);
@@ -199,7 +102,11 @@ export async function embed(tokenIds, embedBuffer, config) {
199
102
 
200
103
  // Check if F16 output is requested and supported
201
104
  const caps = getKernelCapabilities();
202
- const useF16 = activationDtype === 'f16' && caps.hasF16;
105
+ const requiresF16Output = activationDtype === 'f16';
106
+ if (requiresF16Output && !caps.hasF16) {
107
+ throw new Error('[Embed] activationDtype="f16" requires shader-f16 support.');
108
+ }
109
+ const useF16 = requiresF16Output;
203
110
 
204
111
  const dtype = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16 });
205
112
 
@@ -218,7 +125,11 @@ export async function embed(tokenIds, embedBuffer, config) {
218
125
  }
219
126
 
220
127
  if (debug) {
221
- trace.embed(`tokens=${numTokens}, hidden=${hiddenSize}, vocab=${vocabSize}, scaleEmbeddings=${scaleEmbeddings}, transpose=${transpose}, indexOffset=${indexOffset}, activationDtype=${activationDtype}, useF16=${useF16}`);
128
+ trace.embed(
129
+ `tokens=${numTokens}, hidden=${hiddenSize}, vocab=${vocabSize}, scaleEmbeddings=${scaleEmbeddings}, ` +
130
+ `transpose=${transpose}, indexOffset=${indexOffset}, inputHiddenSize=${inputHiddenSize}, ` +
131
+ `hiddenOffset=${hiddenOffset}, activationDtype=${activationDtype}, useF16=${useF16}`
132
+ );
222
133
  if (tokenBufferInput) {
223
134
  trace.embed('TOKEN_IDS: [gpu-buffer]');
224
135
  } else {
@@ -228,7 +139,7 @@ export async function embed(tokenIds, embedBuffer, config) {
228
139
 
229
140
  if (cpuEmbeddings) {
230
141
  if (tokenBufferInput) {
231
- throw new Error('[Embed] GPU token buffer requires GPU-resident embeddings.');
142
+ tokenIdArray = await readGpuTokenIdsForCpuEmbeddingGather(tokenIds, numTokens, indexOffset);
232
143
  }
233
144
  if (debug) {
234
145
  trace.embed('Using CPU embedding gather (oversized embedding)');
@@ -251,31 +162,96 @@ export async function embed(tokenIds, embedBuffer, config) {
251
162
  }
252
163
 
253
164
  const output = new Float32Array(numTokens * hiddenSize);
254
- // Check actual data type: loader's f16_to_f32 CPU path already decodes F16 into Float32Array,
255
- // so dtype='f16' does not reliably indicate raw F16 bytes. Only Uint16Array needs per-element decoding.
256
- const isF16Cpu = cpuEmbeddings instanceof Uint16Array;
257
- if (!transpose) {
165
+
166
+ // Step 6: Batched prefill rows — per-token pre-decoded PLE data for all layers.
167
+ // Layout: Float32Array[numTokens × inputHiddenSize], indexed by
168
+ // [t * inputHiddenSize + hiddenOffset] to select each token's layer slice.
169
+ if (config.preloadedCpuBatchedRows) {
170
+ for (let t = 0; t < numTokens; t++) {
171
+ const srcOffset = t * inputHiddenSize + hiddenOffset;
172
+ const dstOffset = t * hiddenSize;
173
+ output.set(config.preloadedCpuBatchedRows.subarray(srcOffset, srcOffset + hiddenSize), dstOffset);
174
+ }
175
+ } else
176
+
177
+ // Fast path: use pre-loaded and pre-decoded PLE row (coalesced read optimization).
178
+ // preloadedCpuRow is a Float32Array containing the full PLE row for the token,
179
+ // indexed by hiddenOffset to select the correct layer's slice.
180
+ if (config.preloadedCpuRow) {
181
+ const rowSlice = config.preloadedCpuRow.subarray(hiddenOffset, hiddenOffset + hiddenSize);
258
182
  for (let t = 0; t < numTokens; t++) {
259
- const tokenId = (tokenIdArray)[t];
260
- const srcOffset = tokenId * hiddenSize;
261
- if (isF16Cpu) {
183
+ output.set(rowSlice, t * hiddenSize);
184
+ }
185
+ } else
186
+
187
+ // Range-backed path: per-element loadRange calls (original path)
188
+ {
189
+ const rangeBackedSource = isRangeBackedCpuEmbeddingSource(cpuEmbeddings)
190
+ ? cpuEmbeddings
191
+ : null;
192
+ if (rangeBackedSource) {
193
+ const sourceDtype = String(rangeBackedSource.sourceDtype ?? embedBuffer.dtype ?? 'f32').toLowerCase();
194
+ const bytesPerElement = sourceDtype === 'f16' || sourceDtype === 'bf16' ? 2 : 4;
195
+ if (!transpose) {
196
+ for (let t = 0; t < numTokens; t++) {
197
+ const tokenId = (tokenIdArray)[t];
198
+ const srcOffset = tokenId * inputHiddenSize + hiddenOffset;
199
+ const chunk = normalizeRangeBytes(
200
+ await rangeBackedSource.loadRange(srcOffset * bytesPerElement, hiddenSize * bytesPerElement),
201
+ 'CPU embedding range source'
202
+ );
203
+ if (chunk.byteLength !== hiddenSize * bytesPerElement) {
204
+ throw new Error(
205
+ `[Embed] CPU embedding range source returned ${chunk.byteLength} bytes, ` +
206
+ `expected ${hiddenSize * bytesPerElement}.`
207
+ );
208
+ }
209
+ decodeRangeChunkIntoOutput(chunk, sourceDtype, output, t * hiddenSize, hiddenSize);
210
+ }
211
+ } else {
212
+ for (let t = 0; t < numTokens; t++) {
213
+ const tokenId = (tokenIdArray)[t];
214
+ const dstOffset = t * hiddenSize;
262
215
  for (let h = 0; h < hiddenSize; h++) {
263
- output[t * hiddenSize + h] = f16ToF32(cpuEmbeddings[srcOffset + h]);
216
+ const chunk = normalizeRangeBytes(
217
+ await rangeBackedSource.loadRange(
218
+ ((hiddenOffset + h) * vocabSize + tokenId) * bytesPerElement,
219
+ bytesPerElement
220
+ ),
221
+ 'CPU embedding range source'
222
+ );
223
+ decodeRangeChunkIntoOutput(chunk, sourceDtype, output, dstOffset + h, 1);
264
224
  }
265
- } else {
266
- output.set(cpuEmbeddings.subarray(srcOffset, srcOffset + hiddenSize), t * hiddenSize);
267
225
  }
268
226
  }
269
227
  } else {
270
- for (let t = 0; t < numTokens; t++) {
271
- const tokenId = (tokenIdArray)[t];
272
- const dstOffset = t * hiddenSize;
273
- for (let h = 0; h < hiddenSize; h++) {
274
- const raw = cpuEmbeddings[h * vocabSize + tokenId];
275
- output[dstOffset + h] = isF16Cpu ? f16ToF32(raw) : raw;
228
+ // Check actual data type: loader's f16_to_f32 CPU path already decodes F16 into Float32Array,
229
+ // so dtype='f16' does not reliably indicate raw F16 bytes. Only Uint16Array needs per-element decoding.
230
+ const isF16Cpu = cpuEmbeddings instanceof Uint16Array;
231
+ if (!transpose) {
232
+ for (let t = 0; t < numTokens; t++) {
233
+ const tokenId = (tokenIdArray)[t];
234
+ const srcOffset = tokenId * inputHiddenSize + hiddenOffset;
235
+ if (isF16Cpu) {
236
+ for (let h = 0; h < hiddenSize; h++) {
237
+ output[t * hiddenSize + h] = f16ToF32(cpuEmbeddings[srcOffset + h]);
238
+ }
239
+ } else {
240
+ output.set(cpuEmbeddings.subarray(srcOffset, srcOffset + hiddenSize), t * hiddenSize);
241
+ }
242
+ }
243
+ } else {
244
+ for (let t = 0; t < numTokens; t++) {
245
+ const tokenId = (tokenIdArray)[t];
246
+ const dstOffset = t * hiddenSize;
247
+ for (let h = 0; h < hiddenSize; h++) {
248
+ const raw = cpuEmbeddings[(hiddenOffset + h) * vocabSize + tokenId];
249
+ output[dstOffset + h] = isF16Cpu ? f16ToF32(raw) : raw;
250
+ }
276
251
  }
277
252
  }
278
253
  }
254
+ } // end else (non-preloaded path)
279
255
 
280
256
  if (scaleEmbeddings) {
281
257
  const scaleFactor = Math.sqrt(hiddenSize);
@@ -303,6 +279,7 @@ export async function embed(tokenIds, embedBuffer, config) {
303
279
  hiddenSize,
304
280
  probes: config.debugProbes,
305
281
  recorder,
282
+ operatorDiagnostics,
306
283
  dtype: 'f16',
307
284
  });
308
285
  return f16Tensor;
@@ -318,6 +295,7 @@ export async function embed(tokenIds, embedBuffer, config) {
318
295
  hiddenSize,
319
296
  probes: config.debugProbes,
320
297
  recorder,
298
+ operatorDiagnostics,
321
299
  });
322
300
  return createTensor(outputBuffer, dtype, [numTokens, hiddenSize], 'embed_output');
323
301
  }
@@ -341,6 +319,8 @@ export async function embed(tokenIds, embedBuffer, config) {
341
319
  outputDtype: selectRuleValue('shared', 'dtype', 'f16OrF32', { useF16 }),
342
320
  embeddingDtype,
343
321
  indexOffset,
322
+ inputHiddenSize,
323
+ hiddenOffset,
344
324
  };
345
325
  if (!isGpuBufferInstance(embedBuffer)) {
346
326
  throw new Error('[Embed] GPU embeddings required for gather path.');
@@ -385,6 +365,7 @@ export async function embed(tokenIds, embedBuffer, config) {
385
365
  hiddenSize,
386
366
  probes: config.debugProbes,
387
367
  recorder,
368
+ operatorDiagnostics,
388
369
  dtype: gatherOptions.outputDtype,
389
370
  });
390
371
  return gatherOutput;
@@ -407,9 +388,20 @@ export async function embed(tokenIds, embedBuffer, config) {
407
388
  trace.embed(`RAW (before scale): maxAbs=${maxAbs.toFixed(4)}, scaleFactor=${scaleFactor.toFixed(4)}`);
408
389
  }
409
390
 
410
- const scaledBuffer = recorder
411
- ? recordScale(recorder, gatherOutput.buffer, scaleFactor, numTokens * hiddenSize, useF16)
412
- : await scaleGPUBuffer(gatherOutput.buffer, scaleFactor, numTokens * hiddenSize, useF16);
391
+ const gatheredTensor = createTensor(
392
+ gatherOutput.buffer,
393
+ gatherOptions.outputDtype,
394
+ [numTokens, hiddenSize],
395
+ 'embed_gather_output'
396
+ );
397
+ const scaledTensor = recorder
398
+ ? await recordScale(recorder, gatheredTensor, scaleFactor, {
399
+ count: numTokens * hiddenSize,
400
+ })
401
+ : await runScale(gatheredTensor, scaleFactor, {
402
+ count: numTokens * hiddenSize,
403
+ });
404
+ const scaledBuffer = scaledTensor.buffer;
413
405
  if (recorder) {
414
406
  // Only track if we created this buffer (not pre-allocated)
415
407
  // Pre-allocated buffers are managed by the caller (e.g., DecodeBufferManager)
@@ -444,6 +436,7 @@ export async function embed(tokenIds, embedBuffer, config) {
444
436
  hiddenSize,
445
437
  probes: config.debugProbes,
446
438
  recorder,
439
+ operatorDiagnostics,
447
440
  dtype,
448
441
  });
449
442
 
@@ -13,6 +13,7 @@ export interface StaticExecutionPlan {
13
13
  kernelPathSource: KernelPathSource | 'rule' | 'self' | 'none';
14
14
  activationDtype: 'f16' | 'f32';
15
15
  finitenessGuardEnabled: boolean;
16
+ finitenessOnTrigger: 'error' | 'fallback-plan';
16
17
  finitenessAbsThreshold: number;
17
18
  finitenessIncludeNonFinite: boolean;
18
19
  deferredRoundingWindowTokens: number;
@@ -43,6 +44,7 @@ export interface ExecutionSessionPlan {
43
44
  kernelPathId: string | null;
44
45
  activationDtype: 'f16' | 'f32';
45
46
  finitenessGuardEnabled: boolean;
47
+ finitenessOnTrigger: 'error' | 'fallback-plan';
46
48
  finitenessAbsThreshold: number;
47
49
  finitenessIncludeNonFinite: boolean;
48
50
  deferredRoundingWindowTokens: number;
@@ -61,6 +63,10 @@ export interface ExecutionSessionPlan {
61
63
  batchSize?: number;
62
64
  stopCheckMode?: 'batch' | 'per-token';
63
65
  maxTokens?: number;
66
+ readbackInterval?: number;
67
+ ringTokens?: number;
68
+ ringStop?: number;
69
+ ringStaging?: number;
64
70
  };
65
71
  }
66
72
 
@@ -90,6 +96,10 @@ export declare function resolveExecutionSessionPlan(
90
96
  batchSize?: number;
91
97
  stopCheckMode?: 'batch' | 'per-token';
92
98
  maxTokens?: number;
99
+ readbackInterval?: number;
100
+ ringTokens?: number;
101
+ ringStop?: number;
102
+ ringStaging?: number;
93
103
  }
94
104
  ): ExecutionSessionPlan;
95
105
 
@@ -106,8 +116,17 @@ export declare function isBatchDecodeEnabled(config: {
106
116
  disableCommandBatching: boolean;
107
117
  isBdpaPagedLayout?: boolean;
108
118
  finitenessFallbackWindowOpen?: boolean;
119
+ hasLinearAttentionLayers?: boolean;
120
+ selfSpeculationEnabled?: boolean;
121
+ hasRangeBackedPerLayerInputs?: boolean;
109
122
  }): boolean;
110
123
 
124
+ export declare function resolveMaxBatchDecodeTokens(config: {
125
+ hasHotVocabularyBatchDecode?: boolean;
126
+ hasGpuSplitPerLayerInputs?: boolean;
127
+ hasLinearAttentionLayers?: boolean;
128
+ }): number | null;
129
+
111
130
  export declare function isDecodeRecorderEnabled(config: {
112
131
  hasDevice: boolean;
113
132
  debug: boolean;