doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -31,7 +31,9 @@ import type {
31
31
  } from '../../kv-cache.js';
32
32
  import type { DecodeRingStats } from '../../decode-ring.js';
33
33
  import type { LinearAttentionRuntime } from './linear-attention.js';
34
- import type { PerLayerInputWeights } from '../../../loader/loader-types.js';
34
+ import type { TokenizerLoadTiming } from '../../tokenizer.js';
35
+ import type { LoaderLoadTiming, PerLayerInputWeights } from '../../../loader/loader-types.js';
36
+ import type { UniformCacheStats } from '../../../gpu/uniform-cache.js';
35
37
 
36
38
  // ============================================================================
37
39
  // Core Context Types
@@ -66,6 +68,8 @@ export interface LayerContext {
66
68
  currentSeqLen: number;
67
69
  /** Token IDs for the current micro-batch (required by BDPA ingestion). */
68
70
  currentTokenIds?: number[] | null;
71
+ /** DiffusionGemma canvas decoder pass: read encoder KV and do not mutate cache. */
72
+ diffusionGemmaDecoder?: boolean;
69
73
  /** Absolute-position multimodal span that should remain bidirectional during causal prefill. */
70
74
  multimodalBidirectionalSpan?: {
71
75
  start: number;
@@ -89,6 +93,17 @@ export interface LayerContext {
89
93
  stats?: PipelineStats;
90
94
  /** Optional layer pipeline plan (JSON-configured) */
91
95
  pipelinePlan?: CompiledLayerPipeline | null;
96
+ __pendingFfnResidualTensor?: unknown;
97
+ __ffnResidualFusedFired?: boolean;
98
+ __layerScalarFusedFired?: boolean;
99
+ __precomputedInputNorm?: {
100
+ layerIdx: number;
101
+ tensor: unknown;
102
+ } | null;
103
+ __postFfnNextInputNorm?: {
104
+ layerIdx: number;
105
+ weight: unknown;
106
+ } | null;
92
107
  /** RoPE frequency buffers (global for full_attention layers) */
93
108
  ropeFreqsCos: GPUBuffer | Float32Array | null;
94
109
  ropeFreqsSin: GPUBuffer | Float32Array | null;
@@ -490,12 +505,17 @@ export interface LayerWeights {
490
505
 
491
506
  // Sandwich norms (Gemma 3)
492
507
  preFeedforwardNorm?: GPUBuffer | Float32Array;
508
+ preFeedforwardNorm2?: GPUBuffer | Float32Array;
493
509
  postFeedforwardNorm?: GPUBuffer | Float32Array;
510
+ postFeedforwardNorm1?: GPUBuffer | Float32Array;
511
+ postFeedforwardNorm2?: GPUBuffer | Float32Array;
494
512
  layerScalar?: Float32Array | null;
495
513
 
496
514
  // MoE
497
515
  routerWeight?: GPUBuffer | import('../../../gpu/weight-buffer.js').WeightBuffer | Float32Array;
498
516
  routerBias?: GPUBuffer | Float32Array | null;
517
+ routerScale?: GPUBuffer | Float32Array | null;
518
+ routerPerExpertScale?: GPUBuffer | Float32Array | null;
499
519
  qNorm?: GPUBuffer | Float32Array;
500
520
  kNorm?: GPUBuffer | Float32Array;
501
521
  experts?: ExpertWeights[];
@@ -505,11 +525,13 @@ export interface LayerWeights {
505
525
  * Weights for a single MoE expert.
506
526
  */
507
527
  export interface ExpertWeights {
508
- expertFormat?: 'mixtral' | 'gpt-oss';
528
+ expertFormat?: 'mixtral' | 'gpt-oss' | 'gemma4';
509
529
  gate?: LayerWeightBuffer;
510
530
  up?: LayerWeightBuffer;
511
531
  down?: LayerWeightBuffer;
532
+ gateUp?: LayerWeightBuffer;
512
533
  numExperts?: number;
534
+ expertIntermediateSize?: number;
513
535
  gateUpBlocks?: GPUBuffer;
514
536
  gateUpScales?: GPUBuffer;
515
537
  gateUpBias?: GPUBuffer;
@@ -522,8 +544,10 @@ export interface ExpertWeights {
522
544
  * Router weights for MoE layers.
523
545
  */
524
546
  export interface RouterWeights {
525
- weight: GPUBuffer | Float32Array;
547
+ weight: GPUBuffer | Float32Array | import('../../../gpu/weight-buffer.js').WeightBuffer;
526
548
  bias?: GPUBuffer | Float32Array | null;
549
+ scale?: GPUBuffer | Float32Array | import('../../../gpu/weight-buffer.js').WeightBuffer | null;
550
+ perExpertScale?: GPUBuffer | Float32Array | import('../../../gpu/weight-buffer.js').WeightBuffer | null;
527
551
  }
528
552
 
529
553
  // ============================================================================
@@ -533,6 +557,28 @@ export interface RouterWeights {
533
557
  /**
534
558
  * Pipeline performance statistics.
535
559
  */
560
+ export type PipelineLoadTimingPhase =
561
+ | 'reset'
562
+ | 'configResolution'
563
+ | 'kernelWarmup'
564
+ | 'tokenizer'
565
+ | 'executionSetup'
566
+ | 'loadWeights'
567
+ | 'rope'
568
+ | 'convStates';
569
+
570
+ export interface PipelineLoadTiming {
571
+ schemaVersion: 1;
572
+ source: 'doppler-pipeline';
573
+ modelId: string | null;
574
+ status: 'running' | 'complete' | 'failed';
575
+ phasesMs: Record<PipelineLoadTimingPhase, number | null>;
576
+ details: {
577
+ tokenizer: TokenizerLoadTiming | null;
578
+ };
579
+ totalMs: number | null;
580
+ }
581
+
536
582
  export interface PipelineStats {
537
583
  /** Total prefill time in milliseconds */
538
584
  prefillTimeMs: number;
@@ -553,17 +599,30 @@ export interface PipelineStats {
553
599
  memoryUsageBytes: number;
554
600
 
555
601
  // Fields from pipeline.ts
602
+ modelLoadMs?: number;
603
+ loadTiming?: LoaderLoadTiming | null;
604
+ pipelineLoadTiming?: PipelineLoadTiming | null;
556
605
  tokensGenerated: number;
557
606
  totalTimeMs: number;
558
607
  gpuTimePrefillMs?: number;
559
608
  gpuTimeDecodeMs?: number;
560
609
  decodeRecordMs?: number;
610
+ decodeRecordOps?: number;
611
+ decodeRecordPasses?: number;
612
+ decodeRecordOpLabels?: Record<string, number>;
613
+ uniformCache?: UniformCacheStats | null;
561
614
  decodeSubmitWaitMs?: number;
562
615
  decodeReadbackWaitMs?: number;
616
+ decodeReadbackMapWaitMs?: number;
617
+ decodeReadbackCleanupMs?: number;
618
+ decodeReadbackCopyMs?: number;
563
619
  decodeMode?: 'single_token' | 'batched_gpu' | 'batched_gpu_stepwise_ple' | null;
564
620
  batchGuardReason?: string | null;
565
621
  singleTokenSubmitWaitMs?: number;
566
622
  singleTokenReadbackWaitMs?: number;
623
+ singleTokenReadbackMapWaitMs?: number;
624
+ singleTokenReadbackCleanupMs?: number;
625
+ singleTokenReadbackCopyMs?: number;
567
626
  singleTokenOrchestrationMs?: number;
568
627
  batching?: BatchingStats | null;
569
628
  plePreparedTokenCacheHits?: number;
@@ -580,8 +639,10 @@ export interface PipelineStats {
580
639
  kernelPathSource: string;
581
640
  activationDtype: string;
582
641
  readbackInterval: number | null;
642
+ readbackMode: string | null;
583
643
  batchSize: number;
584
644
  stopCheckMode: string;
645
+ disableCommandBatching?: boolean;
585
646
  ringTokens: number | null;
586
647
  ringStop: number | null;
587
648
  ringStaging: number | null;
@@ -592,8 +653,10 @@ export interface PipelineStats {
592
653
  kernelPathSource: string;
593
654
  activationDtype: string;
594
655
  readbackInterval: number | null;
656
+ readbackMode: string | null;
595
657
  batchSize: number;
596
658
  stopCheckMode: string;
659
+ disableCommandBatching?: boolean;
597
660
  ringTokens: number | null;
598
661
  ringStop: number | null;
599
662
  ringStaging: number | null;
@@ -646,6 +709,24 @@ export interface BatchingStats {
646
709
 
647
710
  /** Number of GPU command submissions */
648
711
  gpuSubmissions: number;
712
+
713
+ /** Largest requested decode burst in tokens before runtime safety bounding */
714
+ requestedBatchTokens?: number;
715
+
716
+ /** Largest actual decode burst submitted in tokens after runtime safety bounding */
717
+ effectiveBatchTokens?: number;
718
+
719
+ /** Total GPU batch-path decode tokens recorded/submitted for execution */
720
+ executedBatchTokens?: number;
721
+
722
+ /** Total batch-path decode tokens retained by stop resolution */
723
+ resolvedBatchTokens?: number;
724
+
725
+ /** Active max decode-burst cap selected by execution rules, when present */
726
+ maxBatchTokenCap?: number | null;
727
+
728
+ /** Number of decode loop iterations where requested burst tokens were clamped */
729
+ batchClampCount?: number;
649
730
  }
650
731
 
651
732
  // ============================================================================
@@ -46,7 +46,8 @@ export function getLayerWeights(
46
46
  */
47
47
  export function getWeightBuffer(
48
48
  weight: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | ArrayBuffer,
49
- label: string
49
+ label: string,
50
+ deviceOverride?: GPUDevice | null
50
51
  ): GPUBuffer | WeightBuffer;
51
52
 
52
53
  /**
@@ -56,5 +57,6 @@ export function getNormWeightBuffer(
56
57
  weight: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | { buffer: ArrayBuffer; byteOffset: number; byteLength: number } | CpuWeightBuffer,
57
58
  label: string,
58
59
  config: WeightBufferConfig,
59
- debugFlags?: WeightDebugFlags
60
+ debugFlags?: WeightDebugFlags,
61
+ deviceOverride?: GPUDevice | null
60
62
  ): GPUBuffer;
@@ -26,7 +26,7 @@ export function getLayerWeights(weights, key) {
26
26
  // ============================================================================
27
27
 
28
28
 
29
- export function getWeightBuffer(weight, label) {
29
+ export function getWeightBuffer(weight, label, deviceOverride = null) {
30
30
  // Preserve WeightBuffer to maintain dtype/layout for matmul
31
31
  if (isWeightBuffer(weight)) {
32
32
  return weight;
@@ -35,7 +35,7 @@ export function getWeightBuffer(weight, label) {
35
35
  return weight;
36
36
  }
37
37
 
38
- const device = getDevice();
38
+ const device = deviceOverride ?? getDevice();
39
39
  if (!device) {
40
40
  throw new Error('No GPU device available for weight buffer creation');
41
41
  }
@@ -64,7 +64,7 @@ export function getWeightBuffer(weight, label) {
64
64
  }
65
65
 
66
66
 
67
- export function getNormWeightBuffer(weight, label, config, debugFlags) {
67
+ export function getNormWeightBuffer(weight, label, config, debugFlags, deviceOverride = null) {
68
68
  // Debug: Log whether weight is GPUBuffer (first time only)
69
69
  if (debugFlags && !debugFlags.normBufferTypeLogged) {
70
70
  debugFlags.normBufferTypeLogged = true;
@@ -80,7 +80,7 @@ export function getNormWeightBuffer(weight, label, config, debugFlags) {
80
80
  return weight;
81
81
  }
82
82
 
83
- const device = getDevice();
83
+ const device = deviceOverride ?? getDevice();
84
84
  if (!device) {
85
85
  throw new Error('No GPU device available for norm weight buffer creation');
86
86
  }
@@ -125,4 +125,3 @@ function getGPUWeightBuffer(weight, label) {
125
125
  // At this point weight is Float32Array or ArrayBuffer, so getWeightBuffer returns GPUBuffer
126
126
  return (getWeightBuffer(weight, label));
127
127
  }
128
-
@@ -24,6 +24,11 @@ import type { StructuredJsonHeadPipeline } from './structured/json-head-pipeline
24
24
  import type { EnergyRowHeadPipeline } from './energy-head/row-head-pipeline.js';
25
25
  import { getBufferPool as getGlobalBufferPool } from '../../memory/buffer-pool.js';
26
26
  import type { EmulationStats } from '../../config/schema/index.js';
27
+ import type {
28
+ DiffusionGemmaCanvasLogitsInput,
29
+ DiffusionGemmaCanvasStepInput,
30
+ DiffusionGemmaCanvasStepResult,
31
+ } from './text/generator.js';
27
32
 
28
33
  // Re-export types for external use
29
34
  export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
@@ -39,6 +44,13 @@ export declare function buildConservativeMultimodalGenerationOptions(
39
44
  options?: GenerateOptions
40
45
  ): GenerateOptions;
41
46
 
47
+ export declare class AbortError extends Error {
48
+ code: 'ABORT_ERR';
49
+ constructor(message?: string);
50
+ }
51
+
52
+ export declare function isAbortError(err: unknown): boolean;
53
+
42
54
  // ============================================================================
43
55
  // Main Inference Pipeline Class
44
56
  // ============================================================================
@@ -81,6 +93,8 @@ export declare class InferencePipeline extends PipelineState {
81
93
  options?: GenerateOptions
82
94
  ): Promise<{ tokenIds: number[]; stats: PipelineStats }>;
83
95
 
96
+ resetToSeqLen(seqLen: number): void;
97
+
84
98
  decodeStepLogits(currentIds: number[], options?: GenerateOptions): Promise<LogitsStepResult>;
85
99
 
86
100
  advanceWithToken(tokenId: number, options?: GenerateOptions): Promise<void>;
@@ -89,6 +103,16 @@ export declare class InferencePipeline extends PipelineState {
89
103
 
90
104
  prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
91
105
 
106
+ computeDiffusionGemmaCanvasLogits(
107
+ args: DiffusionGemmaCanvasLogitsInput,
108
+ options?: GenerateOptions & { __internalGenerate?: boolean }
109
+ ): Promise<Float32Array>;
110
+
111
+ computeDiffusionGemmaCanvasStep(
112
+ args: DiffusionGemmaCanvasStepInput,
113
+ options?: GenerateOptions & { __internalGenerate?: boolean }
114
+ ): Promise<DiffusionGemmaCanvasStepResult>;
115
+
92
116
  prefillWithEmbedding(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillEmbeddingResult>;
93
117
 
94
118
  embed(prompt: string, options?: GenerateOptions): Promise<{
@@ -1,5 +1,6 @@
1
1
 
2
2
  import { getDevice, initDevice, getKernelCapabilities } from '../../gpu/device.js';
3
+ import { getUniformCacheStats } from '../../gpu/uniform-cache.js';
3
4
  import { getBufferPool as getGlobalBufferPool, readBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
4
5
  import { log } from '../../debug/index.js';
5
6
  import { configurePerfGuards } from '../../gpu/perf-guards.js';
@@ -111,6 +112,51 @@ async function withPipelineLoadPhase(phase, context, run) {
111
112
  }
112
113
  }
113
114
 
115
+ function roundPipelineTimingMs(value) {
116
+ return Number.isFinite(value) ? Number(value.toFixed(3)) : null;
117
+ }
118
+
119
+ function createPipelineLoadTiming(modelId) {
120
+ return {
121
+ schemaVersion: 1,
122
+ source: 'doppler-pipeline',
123
+ modelId: typeof modelId === 'string' ? modelId : null,
124
+ status: 'running',
125
+ phasesMs: {
126
+ reset: null,
127
+ configResolution: null,
128
+ kernelWarmup: null,
129
+ tokenizer: null,
130
+ executionSetup: null,
131
+ loadWeights: null,
132
+ rope: null,
133
+ convStates: null,
134
+ },
135
+ details: {
136
+ tokenizer: null,
137
+ },
138
+ totalMs: null,
139
+ };
140
+ }
141
+
142
+ function finishPipelineLoadTimingPhase(loadTiming, phase, startMs) {
143
+ if (!loadTiming?.phasesMs || !phase) return;
144
+ const elapsedMs = roundPipelineTimingMs(performance.now() - startMs);
145
+ const currentMs = loadTiming.phasesMs[phase];
146
+ loadTiming.phasesMs[phase] = Number.isFinite(currentMs)
147
+ ? roundPipelineTimingMs(currentMs + elapsedMs)
148
+ : elapsedMs;
149
+ }
150
+
151
+ async function timedPipelineLoadPhase(loadTiming, phase, context, run) {
152
+ const startMs = performance.now();
153
+ try {
154
+ return await withPipelineLoadPhase(phase, context, run);
155
+ } finally {
156
+ finishPipelineLoadTimingPhase(loadTiming, phase, startMs);
157
+ }
158
+ }
159
+
114
160
  function resolveSingleSpecialTokenId(tokenizer, tokenText, label) {
115
161
  const rawTokenIds = tokenizer?.encode?.(tokenText);
116
162
  const tokenIds = Array.isArray(rawTokenIds)
@@ -265,6 +311,9 @@ export class InferencePipeline extends PipelineState {
265
311
 
266
312
  async loadModel(manifest) {
267
313
  const loadStart = performance.now();
314
+ const pipelineLoadTiming = createPipelineLoadTiming(manifest?.modelId ?? null);
315
+ const resetStart = performance.now();
316
+ this.stats.pipelineLoadTiming = pipelineLoadTiming;
268
317
  this.manifest = manifest;
269
318
  this.decodeRing?.release();
270
319
  if (this.sampleReadbackBuffer) {
@@ -274,6 +323,7 @@ export class InferencePipeline extends PipelineState {
274
323
  this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
275
324
  destroyMoERouter(this.moeRouter);
276
325
  this.moeRouter = null;
326
+ finishPipelineLoadTimingPhase(pipelineLoadTiming, 'reset', resetStart);
277
327
 
278
328
  // ========================================================================
279
329
  // Config Resolution Passes
@@ -306,6 +356,7 @@ export class InferencePipeline extends PipelineState {
306
356
  // ========================================================================
307
357
 
308
358
  let configResolutionPhase = 0;
359
+ const configResolutionStart = performance.now();
309
360
 
310
361
  // Phase 1: execution-v1 runtime config
311
362
  configResolutionPhase = 1;
@@ -407,13 +458,20 @@ export class InferencePipeline extends PipelineState {
407
458
  } else {
408
459
  this.audioCapable = false;
409
460
  }
461
+ finishPipelineLoadTimingPhase(pipelineLoadTiming, 'configResolution', configResolutionStart);
410
462
 
411
- await runKernelWarmup({
412
- useGPU: this.useGPU,
413
- kernelWarmup: this.runtimeConfig.shared?.kernelWarmup,
414
- modelConfig: this.modelConfig,
415
- });
463
+ await timedPipelineLoadPhase(
464
+ pipelineLoadTiming,
465
+ 'kernelWarmup',
466
+ { modelId: manifest.modelId ?? null },
467
+ () => runKernelWarmup({
468
+ useGPU: this.useGPU,
469
+ kernelWarmup: this.runtimeConfig.shared?.kernelWarmup,
470
+ modelConfig: this.modelConfig,
471
+ })
472
+ );
416
473
 
474
+ const executionSetupStart = performance.now();
417
475
  // Phase 3: kernel path resolution + dtype contract
418
476
  configResolutionPhase = 3;
419
477
  log.debug('Pipeline', `Config resolution phase ${configResolutionPhase}: resolveKernelPathState`);
@@ -437,8 +495,10 @@ export class InferencePipeline extends PipelineState {
437
495
  const moeStr = cfg.useMoE ? `, MoE(${cfg.numExperts}x${cfg.moeTopK})` : '';
438
496
  const kernelInfo = this.resolvedKernelPath ? `kernelPath=${this.resolvedKernelPath.id}` : 'kernelPath=none';
439
497
  log.info('Pipeline', `${cfg.numLayers}L/${cfg.hiddenSize}H/${cfg.numHeads}heads (${cfg.headDim}dim)${moeStr}, ${kernelInfo}`);
498
+ finishPipelineLoadTimingPhase(pipelineLoadTiming, 'executionSetup', executionSetupStart);
440
499
 
441
- this.tokenizer = await withPipelineLoadPhase(
500
+ this.tokenizer = await timedPipelineLoadPhase(
501
+ pipelineLoadTiming,
442
502
  'tokenizer',
443
503
  { modelId: manifest.modelId ?? null },
444
504
  () => initTokenizerFromManifest(
@@ -447,6 +507,9 @@ export class InferencePipeline extends PipelineState {
447
507
  this.storageContext
448
508
  )
449
509
  );
510
+ pipelineLoadTiming.details.tokenizer = typeof this.tokenizer?.getLoadTiming === 'function'
511
+ ? this.tokenizer.getLoadTiming()
512
+ : null;
450
513
  const tokenizerVocabSize = this.tokenizer.getVocabSize();
451
514
  if (Number.isFinite(tokenizerVocabSize) && tokenizerVocabSize > 0) {
452
515
  if (tokenizerVocabSize !== this.modelConfig.vocabSize) {
@@ -454,6 +517,7 @@ export class InferencePipeline extends PipelineState {
454
517
  }
455
518
  }
456
519
 
520
+ const postTokenizerExecutionSetupStart = performance.now();
457
521
  // Manifest quantizationInfo.compute is the binding lane identity.
458
522
  assertManifestComputeLaneBinding({ manifest, runtimeConfig: this.runtimeConfig });
459
523
 
@@ -512,23 +576,41 @@ export class InferencePipeline extends PipelineState {
512
576
  this.runtimeConfig.inference.speculative
513
577
  );
514
578
  }
579
+ finishPipelineLoadTimingPhase(
580
+ pipelineLoadTiming,
581
+ 'executionSetup',
582
+ postTokenizerExecutionSetupStart
583
+ );
515
584
 
516
585
  // Load weights
517
- await withPipelineLoadPhase(
586
+ await timedPipelineLoadPhase(
587
+ pipelineLoadTiming,
518
588
  'loadWeights',
519
589
  { modelId: manifest.modelId ?? null },
520
590
  () => this._loadWeights()
521
591
  );
522
592
 
523
593
  // Initialize RoPE frequencies
524
- await this._initRoPE();
594
+ await timedPipelineLoadPhase(
595
+ pipelineLoadTiming,
596
+ 'rope',
597
+ { modelId: manifest.modelId ?? null },
598
+ () => this._initRoPE()
599
+ );
525
600
 
526
601
  // Initialize conv layer states for gated short conv layers (LFM2)
527
- await this._initConvLayerStates();
602
+ await timedPipelineLoadPhase(
603
+ pipelineLoadTiming,
604
+ 'convStates',
605
+ { modelId: manifest.modelId ?? null },
606
+ () => this._initConvLayerStates()
607
+ );
528
608
 
529
609
  this.isLoaded = true;
530
610
  const loadMs = performance.now() - loadStart;
531
611
  this.stats.modelLoadMs = loadMs;
612
+ pipelineLoadTiming.totalMs = roundPipelineTimingMs(loadMs);
613
+ pipelineLoadTiming.status = 'complete';
532
614
  log.info('Pipeline', `Model loaded successfully (${loadMs.toFixed(0)}ms)`);
533
615
  }
534
616
 
@@ -570,12 +652,14 @@ export class InferencePipeline extends PipelineState {
570
652
  this.weights.set('embed', result.embeddings);
571
653
  this.weights.set('lm_head', result.lmHead);
572
654
  this.weights.set('final_norm', result.finalNorm);
655
+ this.weights.set('diffusion_gemma_self_conditioning', result.diffusionGemmaSelfConditioning);
573
656
  this.weights.set('per_layer_inputs', result.perLayerInputWeights);
574
657
  this.embeddingPostprocessor = result.embeddingPostprocessor;
575
658
 
576
659
  this.layerRouterWeights = result.layerRouterWeights;
577
660
 
578
661
  this.dopplerLoader = getDopplerLoader(this.runtimeConfig.loading);
662
+ this.stats.loadTiming = result.loadTiming ?? this.dopplerLoader?.getLoadTiming?.() ?? null;
579
663
 
580
664
  if ((this.modelConfig).useMoE && this.moeRouter) {
581
665
  this.moeRouter = initMoERouter(
@@ -739,11 +823,14 @@ export class InferencePipeline extends PipelineState {
739
823
  });
740
824
 
741
825
  if (vc.visionArchitecture === 'gemma4') {
826
+ const isEncoderFree = (depth === 0);
742
827
  const visionWeights = {
743
828
  textHiddenSize: this.modelConfig.hiddenSize,
744
829
  patchInputProj: await loadRequiredTensor('model.vision_tower.patch_embedder.input_proj.weight'),
745
830
  patchPositionEmbeddingTable: await loadRequiredTensor('model.vision_tower.patch_embedder.position_embedding_table', false),
746
- projector: await loadRequiredTensor('model.embed_vision.embedding_projection.weight'),
831
+ projector: isEncoderFree
832
+ ? await loader.loadTensor('model.embed_vision.embedding_projection.weight', true, true)
833
+ : await loadRequiredTensor('model.embed_vision.embedding_projection.weight'),
747
834
  layers: [],
748
835
  };
749
836
 
@@ -877,16 +964,17 @@ export class InferencePipeline extends PipelineState {
877
964
  outputMax: await loadScalar(`${prefix}.output_max`),
878
965
  });
879
966
 
967
+ const isEncoderFree = (depth === 0);
880
968
  const audioWeights = {
881
969
  // Subsampling
882
- subsampleConv0Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.conv.weight'),
883
- subsampleNorm0Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.norm.weight'),
884
- subsampleConv1Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.conv.weight'),
885
- subsampleNorm1Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.norm.weight'),
886
- subsampleInputProjWeight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.input_proj_linear.weight'),
970
+ subsampleConv0Weight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.conv.weight'),
971
+ subsampleNorm0Weight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.norm.weight'),
972
+ subsampleConv1Weight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.conv.weight'),
973
+ subsampleNorm1Weight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.norm.weight'),
974
+ subsampleInputProjWeight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.input_proj_linear.weight'),
887
975
  // Output
888
- outputProjWeight: await loadRequiredTensor('model.audio_tower.output_proj.weight'),
889
- outputProjBias: await loadRequiredTensor('model.audio_tower.output_proj.bias'),
976
+ outputProjWeight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.output_proj.weight'),
977
+ outputProjBias: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.output_proj.bias'),
890
978
  audioEmbeddingProjWeight: await loadRequiredTensor('model.embed_audio.embedding_projection.weight'),
891
979
  layers: [],
892
980
  };
@@ -1252,20 +1340,26 @@ export class InferencePipeline extends PipelineState {
1252
1340
  this.reset();
1253
1341
 
1254
1342
  // Lazy-load audio modules
1255
- const { extractLogMelSpectrogram } = await import('./audio/mel.js');
1256
1343
  const { encodeAudio } = await import('./audio/index.js');
1257
1344
 
1258
- // Step 1: Extract mel spectrogram
1259
- const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
1260
-
1261
- // Step 2: Encode audio through conformer pipeline
1262
- const encodeResult = await encodeAudio({
1263
- melFeatures,
1264
- numFrames,
1265
- nMels,
1266
- audioConfig: this.audioConfig,
1267
- weights: this.audioWeights,
1268
- });
1345
+ let encodeResult;
1346
+ if (this.audioConfig.depth === 0) {
1347
+ encodeResult = await encodeAudio({
1348
+ rawAudio: audio,
1349
+ audioConfig: this.audioConfig,
1350
+ weights: this.audioWeights,
1351
+ });
1352
+ } else {
1353
+ const { extractLogMelSpectrogram } = await import('./audio/mel.js');
1354
+ const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
1355
+ encodeResult = await encodeAudio({
1356
+ melFeatures,
1357
+ numFrames,
1358
+ nMels,
1359
+ audioConfig: this.audioConfig,
1360
+ weights: this.audioWeights,
1361
+ });
1362
+ }
1269
1363
 
1270
1364
  // Step 3: Build the multimodal prompt with <|audio|> placeholder
1271
1365
  const requestedPrompt = prompt ?? 'Transcribe the audio.';
@@ -1424,6 +1518,10 @@ export class InferencePipeline extends PipelineState {
1424
1518
  return this.generator.generateTokenIds(prompt, options);
1425
1519
  }
1426
1520
 
1521
+ resetToSeqLen(seqLen) {
1522
+ return this.generator.resetToSeqLen(seqLen);
1523
+ }
1524
+
1427
1525
  decodeStepLogits(currentIds, options = {}) {
1428
1526
  assertNotAborted(options?.signal);
1429
1527
  return this.generator.decodeStepLogits(currentIds, options);
@@ -1445,6 +1543,16 @@ export class InferencePipeline extends PipelineState {
1445
1543
  return this.generator.prefillKVOnly(prompt, options);
1446
1544
  }
1447
1545
 
1546
+ computeDiffusionGemmaCanvasLogits(args, options = {}) {
1547
+ assertNotAborted(options?.signal);
1548
+ return this.generator.computeDiffusionGemmaCanvasLogits(args, options);
1549
+ }
1550
+
1551
+ computeDiffusionGemmaCanvasStep(args, options = {}) {
1552
+ assertNotAborted(options?.signal);
1553
+ return this.generator.computeDiffusionGemmaCanvasStep(args, options);
1554
+ }
1555
+
1448
1556
  prefillWithEmbedding(prompt, options = {}) {
1449
1557
  assertNotAborted(options?.signal);
1450
1558
  return this.generator.prefillWithEmbedding(prompt, options);
@@ -1557,16 +1665,26 @@ export class InferencePipeline extends PipelineState {
1557
1665
  await this._ensureAudioWeightsLoaded();
1558
1666
  this.reset();
1559
1667
 
1560
- const { extractLogMelSpectrogram } = await import('./audio/mel.js');
1561
1668
  const { encodeAudio } = await import('./audio/index.js');
1562
- const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
1563
- const encodeResult = await encodeAudio({
1564
- melFeatures,
1565
- numFrames,
1566
- nMels,
1567
- audioConfig: this.audioConfig,
1568
- weights: this.audioWeights,
1569
- });
1669
+
1670
+ let encodeResult;
1671
+ if (this.audioConfig.depth === 0) {
1672
+ encodeResult = await encodeAudio({
1673
+ rawAudio: audio,
1674
+ audioConfig: this.audioConfig,
1675
+ weights: this.audioWeights,
1676
+ });
1677
+ } else {
1678
+ const { extractLogMelSpectrogram } = await import('./audio/mel.js');
1679
+ const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
1680
+ encodeResult = await encodeAudio({
1681
+ melFeatures,
1682
+ numFrames,
1683
+ nMels,
1684
+ audioConfig: this.audioConfig,
1685
+ weights: this.audioWeights,
1686
+ });
1687
+ }
1570
1688
 
1571
1689
  const hiddenSize = Number(this.audioConfig?.outputProjDims ?? this.modelConfig?.hiddenSize);
1572
1690
  const numTokens = encodeResult.numTokens;
@@ -1656,6 +1774,7 @@ export class InferencePipeline extends PipelineState {
1656
1774
  kernelPathSource: this.executionPlanState.primaryPlan.kernelPathSource ?? 'none',
1657
1775
  activationDtype: this.executionPlanState.primaryPlan.activationDtype,
1658
1776
  readbackInterval: this.executionPlanState.primaryPlan.readbackInterval ?? null,
1777
+ readbackMode: this.executionPlanState.primaryPlan.readbackMode ?? null,
1659
1778
  batchSize: this.executionPlanState.primaryPlan.defaultBatchSize,
1660
1779
  stopCheckMode: this.executionPlanState.primaryPlan.defaultStopCheckMode,
1661
1780
  disableCommandBatching: this.executionPlanState.primaryPlan.defaultDisableCommandBatching === true,
@@ -1671,6 +1790,7 @@ export class InferencePipeline extends PipelineState {
1671
1790
  kernelPathSource: this.executionPlanState.fallbackPlan.kernelPathSource ?? 'none',
1672
1791
  activationDtype: this.executionPlanState.fallbackPlan.activationDtype,
1673
1792
  readbackInterval: this.executionPlanState.fallbackPlan.readbackInterval ?? null,
1793
+ readbackMode: this.executionPlanState.fallbackPlan.readbackMode ?? null,
1674
1794
  batchSize: this.executionPlanState.fallbackPlan.defaultBatchSize,
1675
1795
  stopCheckMode: this.executionPlanState.fallbackPlan.defaultStopCheckMode,
1676
1796
  disableCommandBatching: this.executionPlanState.fallbackPlan.defaultDisableCommandBatching === true,
@@ -1695,6 +1815,10 @@ export class InferencePipeline extends PipelineState {
1695
1815
  if (ringStats) {
1696
1816
  stats.decodeRing = ringStats;
1697
1817
  }
1818
+ const uniformCacheStats = getUniformCacheStats();
1819
+ if (uniformCacheStats) {
1820
+ stats.uniformCache = uniformCacheStats;
1821
+ }
1698
1822
  return stats;
1699
1823
  }
1700
1824