doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -0,0 +1,93 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://clocksmith.dev/doppler/schemas/claim-evidence-contract-v1.schema.json",
4
+ "title": "Doppler Claim Evidence Contract v1",
5
+ "type": "object",
6
+ "additionalProperties": false,
7
+ "required": [
8
+ "$schema",
9
+ "schemaVersion",
10
+ "source",
11
+ "releaseClaimPolicyPath",
12
+ "benchmarkPolicyPath",
13
+ "localInferenceClaimMatrixPath",
14
+ "releaseMatrixPath",
15
+ "goalMatrixPath",
16
+ "requiredScripts",
17
+ "requiredGoalEvidence",
18
+ "requiredReleaseMatrixSources"
19
+ ],
20
+ "properties": {
21
+ "$schema": {
22
+ "type": "string",
23
+ "minLength": 1
24
+ },
25
+ "schemaVersion": {
26
+ "const": 1
27
+ },
28
+ "source": {
29
+ "const": "doppler"
30
+ },
31
+ "releaseClaimPolicyPath": {
32
+ "$ref": "#/$defs/repoRelativeJsonPath"
33
+ },
34
+ "benchmarkPolicyPath": {
35
+ "$ref": "#/$defs/repoRelativeJsonPath"
36
+ },
37
+ "localInferenceClaimMatrixPath": {
38
+ "$ref": "#/$defs/repoRelativeJsonPath"
39
+ },
40
+ "releaseMatrixPath": {
41
+ "$ref": "#/$defs/repoRelativeJsonPath"
42
+ },
43
+ "goalMatrixPath": {
44
+ "$ref": "#/$defs/repoRelativeJsonPath"
45
+ },
46
+ "requiredScripts": {
47
+ "type": "array",
48
+ "minItems": 1,
49
+ "items": {
50
+ "type": "string",
51
+ "minLength": 1
52
+ },
53
+ "uniqueItems": true
54
+ },
55
+ "requiredGoalEvidence": {
56
+ "type": "array",
57
+ "minItems": 1,
58
+ "items": {
59
+ "$ref": "#/$defs/repoRelativeJsonPath"
60
+ },
61
+ "uniqueItems": true
62
+ },
63
+ "requiredReleaseMatrixSources": {
64
+ "type": "array",
65
+ "minItems": 1,
66
+ "items": {
67
+ "type": "string",
68
+ "minLength": 1
69
+ },
70
+ "uniqueItems": true
71
+ }
72
+ },
73
+ "$defs": {
74
+ "repoRelativeJsonPath": {
75
+ "type": "string",
76
+ "minLength": 1,
77
+ "pattern": "\\.json$",
78
+ "not": {
79
+ "anyOf": [
80
+ {
81
+ "pattern": "^/"
82
+ },
83
+ {
84
+ "pattern": "\\\\"
85
+ },
86
+ {
87
+ "pattern": "(^|/)\\.\\.(/|$)"
88
+ }
89
+ ]
90
+ }
91
+ }
92
+ }
93
+ }
@@ -0,0 +1,81 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://clocksmith.dev/doppler/schemas/command-surface-contract-v1.schema.json",
4
+ "title": "Doppler Command Surface Contract v1",
5
+ "type": "object",
6
+ "additionalProperties": false,
7
+ "required": [
8
+ "schemaVersion",
9
+ "source",
10
+ "commands"
11
+ ],
12
+ "properties": {
13
+ "$schema": {
14
+ "type": "string"
15
+ },
16
+ "schemaVersion": {
17
+ "const": 1
18
+ },
19
+ "source": {
20
+ "const": "doppler"
21
+ },
22
+ "commands": {
23
+ "type": "array",
24
+ "minItems": 1,
25
+ "items": {
26
+ "$ref": "#/$defs/command"
27
+ },
28
+ "uniqueItems": true
29
+ }
30
+ },
31
+ "$defs": {
32
+ "commandId": {
33
+ "enum": [
34
+ "convert",
35
+ "refresh-integrity",
36
+ "debug",
37
+ "bench",
38
+ "verify",
39
+ "lora",
40
+ "distill",
41
+ "diagnose"
42
+ ]
43
+ },
44
+ "surface": {
45
+ "enum": [
46
+ "browser",
47
+ "node"
48
+ ]
49
+ },
50
+ "command": {
51
+ "type": "object",
52
+ "additionalProperties": false,
53
+ "required": [
54
+ "id",
55
+ "surfaces",
56
+ "cliUsageRequired",
57
+ "reason"
58
+ ],
59
+ "properties": {
60
+ "id": {
61
+ "$ref": "#/$defs/commandId"
62
+ },
63
+ "surfaces": {
64
+ "type": "array",
65
+ "minItems": 1,
66
+ "items": {
67
+ "$ref": "#/$defs/surface"
68
+ },
69
+ "uniqueItems": true
70
+ },
71
+ "cliUsageRequired": {
72
+ "type": "boolean"
73
+ },
74
+ "reason": {
75
+ "type": "string",
76
+ "minLength": 1
77
+ }
78
+ }
79
+ }
80
+ }
81
+ }
@@ -9,10 +9,14 @@
9
9
  import type {
10
10
  HashAlgorithm,
11
11
  QuantizationValue,
12
+ SourceQuantizationFormat,
13
+ SourceQuantizationTarget,
14
+ SourceTrainingQuantization,
12
15
  RuntimeOptimizationsSchema,
13
16
  ConversionInfoSchema,
14
17
  ManifestArtifactIdentitySchema,
15
18
  ManifestWeightsRefSchema,
19
+ MoEConfigSchema,
16
20
  } from './manifest.schema.js';
17
21
  import type {
18
22
  ExecutionV1GraphSchema,
@@ -28,6 +32,9 @@ export interface ConverterQuantizationConfigSchema {
28
32
  weights: QuantizationValue | null;
29
33
  embeddings: QuantizationValue | null;
30
34
  lmHead: QuantizationValue | null;
35
+ sourceTrainingQuantization: SourceTrainingQuantization | null;
36
+ sourceQuantizationTarget: SourceQuantizationTarget | null;
37
+ sourceQuantizationFormat: SourceQuantizationFormat | null;
31
38
  vision: QuantizationValue | null;
32
39
  audio: QuantizationValue | null;
33
40
  projector: QuantizationValue | null;
@@ -105,6 +112,10 @@ export interface ConverterConfigSchema {
105
112
  manifest: ConverterManifestConfigSchema;
106
113
  inference: ConverterInferenceConfigSchema;
107
114
  output: ConverterOutputConfigSchema;
115
+ moeConfig?: MoEConfigSchema | null;
116
+ execution?: ExecutionV1GraphSchema;
117
+ session?: ExecutionV1SessionSchema;
118
+ modelType?: string;
108
119
  }
109
120
 
110
121
  export declare const DEFAULT_CONVERTER_QUANTIZATION_CONFIG: ConverterQuantizationConfigSchema;
@@ -10,6 +10,9 @@ export const DEFAULT_CONVERTER_QUANTIZATION_CONFIG = {
10
10
  weights: null,
11
11
  embeddings: null,
12
12
  lmHead: null,
13
+ sourceTrainingQuantization: null,
14
+ sourceQuantizationTarget: null,
15
+ sourceQuantizationFormat: null,
13
16
  vision: DEFAULT_QUANTIZATION_DEFAULTS.visionDtype,
14
17
  audio: DEFAULT_QUANTIZATION_DEFAULTS.audioDtype,
15
18
  projector: DEFAULT_QUANTIZATION_DEFAULTS.projectorDtype,
@@ -44,6 +47,7 @@ export const DEFAULT_CONVERTER_MANIFEST_CONFIG = {
44
47
  conversion: null,
45
48
  artifactIdentity: null,
46
49
  weightsRef: null,
50
+ eosTokenId: null,
47
51
  visionConfig: null,
48
52
  audioConfig: null,
49
53
  };
@@ -87,6 +91,7 @@ export const DEFAULT_CONVERTER_CONFIG = {
87
91
  manifest: DEFAULT_CONVERTER_MANIFEST_CONFIG,
88
92
  inference: DEFAULT_CONVERTER_INFERENCE_CONFIG,
89
93
  output: DEFAULT_CONVERTER_OUTPUT_CONFIG,
94
+ moeConfig: null,
90
95
  };
91
96
 
92
97
  export function createConverterConfig(overrides) {
@@ -128,6 +133,9 @@ export function createConverterConfig(overrides) {
128
133
  output: overrides.output
129
134
  ? { ...DEFAULT_CONVERTER_OUTPUT_CONFIG, ...overrides.output }
130
135
  : { ...DEFAULT_CONVERTER_OUTPUT_CONFIG },
136
+ moeConfig: Object.prototype.hasOwnProperty.call(overrides, 'moeConfig')
137
+ ? (overrides.moeConfig ?? null)
138
+ : null,
131
139
  };
132
140
  // V1 conversion configs place execution, session, and modelType at the top level.
133
141
  // Pass them through so isV1Config() can detect the v1 format and resolveConversionPlanV1
@@ -38,6 +38,7 @@ export interface RuntimeInferenceConfigSchema {
38
38
  largeWeights: LargeWeightConfigSchema;
39
39
  kvcache: Record<string, unknown>;
40
40
  diffusion: Record<string, unknown>;
41
+ diffusionGemma: Record<string, unknown>;
41
42
  energy: Record<string, unknown>;
42
43
  moe: Record<string, unknown>;
43
44
  speculative: Record<string, unknown>;
@@ -45,12 +45,16 @@ export const DEFAULT_RUNTIME_CONFIG = {
45
45
  repetitionPenalty: 1.1,
46
46
  greedyThreshold: 0.01,
47
47
  repetitionPenaltyWindow: 100,
48
+ suppressSpecialTokens: false,
49
+ suppressSpecialLikeTokens: false,
50
+ suppressTokenIds: [],
48
51
  },
49
52
  compute: {},
50
53
  tokenizer: {},
51
54
  largeWeights: DEFAULT_LARGE_WEIGHT_CONFIG,
52
55
  kvcache: DEFAULT_KVCACHE_CONFIG,
53
56
  diffusion: {},
57
+ diffusionGemma: {},
54
58
  energy: {},
55
59
  moe: DEFAULT_MOE_RUNTIME_CONFIG,
56
60
  speculative: DEFAULT_SPECULATIVE_CONFIG,
@@ -79,6 +83,7 @@ export const DEFAULT_RUNTIME_CONFIG = {
79
83
  // transitions but longer intermediate-buffer lifetime. Default 4 is
80
84
  // memory-conservative; profiles with known-safe memory can raise it.
81
85
  prefillChunkLayers: 4,
86
+ prefillTokenChunkSize: null,
82
87
  // Opt into the flash-attention prefill kernel (head_dim=256, f16 KV,
83
88
  // contiguous layout). Two-pass dispatch raises RDNA3 workgroup
84
89
  // occupancy by KV-axis splitting. The kernel itself enforces head_dim
@@ -113,6 +118,22 @@ export const DEFAULT_RUNTIME_CONFIG = {
113
118
  // Default false until correctness + perf parity validated on target
114
119
  // hardware.
115
120
  useWideTileQ4KPrefill: false,
121
+ // Opt into the WideTile Q4_K decode matmul. This reuses the WideTile
122
+ // register kernel for one-row decode matmuls when explicitly enabled.
123
+ // Default false until correctness + perf parity validated on target
124
+ // hardware.
125
+ useWideTileQ4KDecode: false,
126
+ // Opt into a two-output sandwich RMSNorm decode kernel:
127
+ // post_attn_norm and pre_ffn_norm execute in one dispatch while still
128
+ // materializing both tensors for the downstream residual and FFN paths.
129
+ // Default false until correctness + perf validated.
130
+ useSandwichRMSNormPairFusion: false,
131
+ // Opt into a decode-only cross-layer RMSNorm pair:
132
+ // post_ffn_norm for layer N and input_norm for layer N+1 execute in one
133
+ // dispatch. The next layer consumes the precomputed input-norm tensor at
134
+ // its normal observation point. Default false until correctness + perf
135
+ // validated.
136
+ usePostFfnNextInputRMSNormPairFusion: false,
116
137
  // Opt into the single-pass flash attention prefill kernel adapted from
117
138
  // ORT's flash_attention.wgsl.template. 64 threads = 64 queries per WG;
118
139
  // private Q/O tiles, shared K/V tiles, online softmax, no reduce pass.
@@ -123,19 +144,26 @@ export const DEFAULT_RUNTIME_CONFIG = {
123
144
  useOrtFlashPrefillAttention: false,
124
145
  // Opt into the WideTile Q4_K matmul + residual fusion at the ffn_down
125
146
  // + ffn_residual call site (dense.js). Eliminates one separate residual
126
- // dispatch per layer (~0.85 ms bubble × 35 layers ≈ 30 ms prefill
127
- // savings). Requires f32 activations + Q4_K weights retained + prefill.
147
+ // dispatch per layer. Requires f32 activations + retained Q4_K weights
148
+ // plus an enabled WideTile phase.
128
149
  // Default false until correctness + perf validated.
129
150
  useWideTileResidualFusion: false,
130
151
  // Opt into the RMSNorm + WideTile Q4_K matmul fusion at pre-matmul
131
152
  // norm sites (input_norm→q/k/v_proj, pre_feedforward_norm→gate/up).
132
153
  // Each fused call recomputes RMS internally (redundant across q/k/v
133
154
  // but negligible) and skips the standalone rmsnorm dispatch upstream.
134
- // Saves ~2 dispatches/layer × 35 layers = 70 dispatches × ~0.85 ms
135
- // bubble ≈ 60 ms prefill savings. Requires f32 activations + Q4_K
136
- // weights retained + prefill. Default false until wiring lands and
137
- // correctness validates.
155
+ // Requires f32 activations + Q4_K weights retained + prefill. Default
156
+ // false until wiring lands and correctness validates.
138
157
  useFusedRmsnormWideTile: false,
158
+ // Opt into fused packed-QKV split plus weighted Q/K RMSNorm. Replaces
159
+ // split_qkv + rmsnorm_qk when diagnostics do not need raw Q/K projection
160
+ // readbacks. Requires f32 QKV output and weighted Q/K norm.
161
+ useFusedQKVSplitQKNorm: false,
162
+ // Opt into fused packed-QKV split plus weighted Q/K RMSNorm and full-head
163
+ // non-interleaved RoPE. Replaces split_qkv + rmsnorm_qk + rope_qk when
164
+ // diagnostics do not need intermediate Q/K stage readbacks.
165
+ useFusedQKVSplitQKNormRoPE: false,
166
+ useGreedyLmHeadArgmaxFusion: false,
139
167
  },
140
168
  executionPatch: {},
141
169
  },
@@ -230,6 +258,7 @@ function mergeSharedRuntimeConfig(
230
258
  bucket: { ...base.bufferPool.bucket, ...overrides.bufferPool.bucket },
231
259
  limits: { ...base.bufferPool.limits, ...overrides.bufferPool.limits },
232
260
  alignment: { ...base.bufferPool.alignment, ...overrides.bufferPool.alignment },
261
+ budget: { ...base.bufferPool.budget, ...overrides.bufferPool.budget },
233
262
  }
234
263
  : { ...base.bufferPool },
235
264
  gpuCache: { ...base.gpuCache, ...overrides.gpuCache },
@@ -312,6 +341,7 @@ function mergeInferenceConfig(
312
341
  const overrideKernelPathPolicy = overrides.kernelPathPolicy ?? {};
313
342
  const baseDiffusion = base.diffusion ?? {};
314
343
  const baseDiffusionDecode = baseDiffusion.decode ?? {};
344
+ const baseDiffusionGemma = base.diffusionGemma ?? {};
315
345
  const baseEnergy = base.energy ?? {};
316
346
  const baseEnergyQuintel = baseEnergy.quintel ?? {};
317
347
  const baseMoe = base.moe ?? {};
@@ -345,6 +375,7 @@ function mergeInferenceConfig(
345
375
  quantization: { ...baseDiffusion.quantization, ...overrides.diffusion.quantization },
346
376
  }
347
377
  : { ...baseDiffusion },
378
+ diffusionGemma: { ...baseDiffusionGemma, ...overrides.diffusionGemma },
348
379
  energy: overrides.energy
349
380
  ? {
350
381
  ...baseEnergy,
@@ -396,6 +427,7 @@ function mergeInferenceConfig(
396
427
  kvcache: replaceSubtree(overrideSession.kvcache, baseSession.kvcache),
397
428
  decodeLoop: replaceSubtree(overrideSession.decodeLoop, baseSession.decodeLoop),
398
429
  perLayerInputs: replaceSubtree(overrideSession.perLayerInputs, baseSession.perLayerInputs),
430
+ prefillTokenChunkSize: overrideSession.prefillTokenChunkSize ?? baseSession.prefillTokenChunkSize,
399
431
  },
400
432
  executionPatch: mergeExecutionPatchLists(baseExecutionPatch, overrideExecutionPatch),
401
433
  // Model-specific inference overrides (merged with manifest.inference at load time)
@@ -175,14 +175,28 @@ export interface ExecutionV1SessionSchema {
175
175
  speculation: ExecutionV1SelfSpeculationSchema | null;
176
176
  /** "sync" (default): wait each prefill chunk. "async": queue without waiting. */
177
177
  prefillChunkSubmitMode: 'sync' | 'async';
178
+ /** Nullable token count for streaming prompt prefill before final logits. */
179
+ prefillTokenChunkSize: number | null;
178
180
  /** Opt into flash-attention prefill kernel. Requires head_dim=256, f16 KV, contiguous layout. */
179
181
  useFlashPrefillAttention: boolean;
180
182
  /** Opt into WideTile Q4_K prefill matmul. Requires f32 activations + Q4_K weights + shader-f16 + M>=TILE_M. */
181
183
  useWideTileQ4KPrefill: boolean;
184
+ /** Opt into WideTile Q4_K decode matmul. Requires f32 activations + Q4_K weights + shader-f16 + M=1. */
185
+ useWideTileQ4KDecode: boolean;
186
+ /** Opt into two-output sandwich RMSNorm decode fusion. Requires f32 activations + post/pre FFN sandwich norm weights + M=1. */
187
+ useSandwichRMSNormPairFusion: boolean;
188
+ /** Opt into post-FFN plus next-layer input RMSNorm decode fusion. Requires f32 activations + M=1. */
189
+ usePostFfnNextInputRMSNormPairFusion: boolean;
190
+ /** Opt into fused packed-QKV split plus weighted Q/K RMSNorm. Requires f32 QKV output and weighted Q/K norm. */
191
+ useFusedQKVSplitQKNorm: boolean;
192
+ /** Opt into fused packed-QKV split plus weighted Q/K RMSNorm and full-head non-interleaved RoPE. */
193
+ useFusedQKVSplitQKNormRoPE: boolean;
182
194
  /** Retain Q4_K packed weights alongside dense buffer (mixed materialization). ~50% extra Q4_K memory. */
183
195
  retainQ4KMaterialization: boolean;
184
196
  /** Use f32-accumulator twin of fused-Q4K f16a multicol gemv. Closes the f16-accum decode gap on AMD RDNA3 where f32 accumulation outperforms f16 by ~60% at FFN gemv shape. Gated by capability rule. */
185
197
  useF32AccumF16ioMatmul: boolean;
198
+ /** Fuse greedy decode LM-head f16-weight GEMV with top-1 selection. Requires greedy sampling with no repetition penalty. */
199
+ useGreedyLmHeadArgmaxFusion: boolean;
186
200
  }
187
201
 
188
202
  // === Policies ===
@@ -242,6 +256,8 @@ export interface ExecutionV1ConfigSchema {
242
256
  export interface ExecutionV1PatchSetSchema {
243
257
  /** Op name to target (matches step[0]) */
244
258
  op: string;
259
+ /** Optional graph section to target */
260
+ section?: 'decode' | 'prefill' | 'preLayer' | 'postLayer' | null;
245
261
  /** Replace kernel key */
246
262
  kernelKey?: string;
247
263
  /** Replace weights */
@@ -253,6 +269,8 @@ export interface ExecutionV1PatchSetSchema {
253
269
  export interface ExecutionV1PatchRemoveSchema {
254
270
  /** Op name to remove */
255
271
  op: string;
272
+ /** Optional graph section to target */
273
+ section?: 'decode' | 'prefill' | 'preLayer' | 'postLayer' | null;
256
274
  /** Target specific layers only (null = all matching ops) */
257
275
  layers?: number[] | null;
258
276
  }
@@ -260,6 +278,8 @@ export interface ExecutionV1PatchRemoveSchema {
260
278
  export interface ExecutionV1PatchAddSchema {
261
279
  /** Step to insert */
262
280
  step: ExecutionV1StepTuple;
281
+ /** Optional graph section to target */
282
+ section?: 'decode' | 'prefill' | 'preLayer' | 'postLayer' | null;
263
283
  /** Insert before this op */
264
284
  insertBefore?: string;
265
285
  /** Insert after this op */
@@ -291,11 +311,16 @@ export interface ExecutionV1PatchSchema {
291
311
  /** Expanded step — what the runtime actually works with after tuple expansion. */
292
312
  export interface ExecutionV1ExpandedStepSchema {
293
313
  op: string;
314
+ src: string;
315
+ dst: string;
294
316
  kernel: string;
295
317
  entry: string;
296
318
  digest: string;
297
319
  weights: string | null;
298
320
  constants: Record<string, number | boolean> | null;
321
+ precision?: ExecutionV1PrecisionSchema;
322
+ fromDtype?: ExecutionV1Dtype;
323
+ toDtype?: ExecutionV1Dtype;
299
324
  layers: 'all' | number[];
300
325
  phase: 'decode' | 'prefill' | 'both';
301
326
  section: 'preLayer' | 'layer' | 'postLayer';
@@ -328,6 +353,8 @@ export interface ExpandExecutionV1Options {
328
353
  knownOps?: ReadonlySet<string> | null;
329
354
  /** When true, unknown ops throw instead of warn. Default false. */
330
355
  strict?: boolean;
356
+ /** Skip kernel digest validation for derived transform kernels. */
357
+ skipDigestValidation?: boolean;
331
358
  }
332
359
 
333
360
  /** Validate and expand a v1 execution graph into runtime-ready expanded steps. */
@@ -1,3 +1,5 @@
1
+ import { log } from '../../debug/index.js';
2
+
1
3
  // =============================================================================
2
4
  // Execution v1 Schema
3
5
  // =============================================================================
@@ -68,10 +70,17 @@ export const DEFAULT_EXECUTION_V1_SESSION = {
68
70
  perLayerInputs: { ...DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION },
69
71
  speculation: null,
70
72
  prefillChunkSubmitMode: 'sync',
73
+ prefillTokenChunkSize: null,
71
74
  useFlashPrefillAttention: false,
72
75
  useWideTileQ4KPrefill: false,
76
+ useWideTileQ4KDecode: false,
77
+ useSandwichRMSNormPairFusion: false,
78
+ usePostFfnNextInputRMSNormPairFusion: false,
79
+ useFusedQKVSplitQKNorm: false,
80
+ useFusedQKVSplitQKNormRoPE: false,
73
81
  retainQ4KMaterialization: false,
74
82
  useF32AccumF16ioMatmul: false,
83
+ useGreedyLmHeadArgmaxFusion: false,
75
84
  };
76
85
 
77
86
  export const DEFAULT_EXECUTION_V1_POLICIES = {
@@ -81,6 +90,7 @@ export const DEFAULT_EXECUTION_V1_POLICIES = {
81
90
  };
82
91
 
83
92
  export const DEFAULT_EXECUTION_V1_PATCH = {
93
+ addKernels: [],
84
94
  set: [],
85
95
  remove: [],
86
96
  add: [],
@@ -159,21 +169,34 @@ function expandTuple(tuple, kernels, phase, section, layers, context) {
159
169
  throw new Error(`${context}: step weights must be a string if provided.`);
160
170
  }
161
171
  const decl = resolveKernel(kernels, kernelKey, `${context}[${op}]`);
172
+ const precision = decl.precision ?? null;
173
+ const castDtypes = {};
174
+ if (op === 'cast') {
175
+ if (!precision?.inputDtype || !precision?.outputDtype) {
176
+ throw new Error(
177
+ `${context}[cast]: cast steps require kernel precision.inputDtype and precision.outputDtype.`
178
+ );
179
+ }
180
+ castDtypes.fromDtype = precision.inputDtype;
181
+ castDtypes.toDtype = precision.outputDtype;
182
+ }
162
183
  return {
163
184
  op,
185
+ src: 'state',
186
+ dst: 'state',
164
187
  kernel: decl.kernel,
165
188
  entry: decl.entry,
166
189
  digest: decl.digest,
167
190
  weights: weights ?? null,
168
191
  constants: decl.constants ?? null,
169
- ...(decl.precision ? { precision: decl.precision } : {}),
192
+ ...(precision ? { precision } : {}),
193
+ ...castDtypes,
170
194
  layers,
171
195
  phase,
172
196
  section,
173
197
  };
174
198
  }
175
199
 
176
-
177
200
  function expandStepEntries(entries, kernels, phase, context) {
178
201
  const expanded = [];
179
202
  for (let i = 0; i < entries.length; i++) {
@@ -206,16 +229,6 @@ function expandBoundarySteps(entries, kernels, section, context) {
206
229
  }
207
230
 
208
231
 
209
- /**
210
- * Expand a v1 execution graph into runtime-ready expanded steps.
211
- *
212
- * @param {object} graph - The execution graph from the manifest.
213
- * @param {{ knownOps?: Set<string> | null, strict?: boolean }} [options]
214
- * knownOps: when provided, any op not in this set triggers a warning (or
215
- * throw in strict mode) with the op name and step index.
216
- * strict: when true, unknown ops throw instead of warn. Default false.
217
- * @returns {object[]} Expanded step array.
218
- */
219
232
  export function expandExecutionV1(graph, options = {}) {
220
233
  const { knownOps = null, strict = false, skipDigestValidation = false } = options;
221
234
  if (!graph || typeof graph !== 'object') {
@@ -238,9 +251,8 @@ export function expandExecutionV1(graph, options = {}) {
238
251
  const sections = { preLayer, decode, prefill, postLayer };
239
252
  for (const [name, steps] of Object.entries(sections)) {
240
253
  if (steps.length > MAX_STEPS_PER_SECTION) {
241
- // Use console.warn directly here because the schema module is a
242
- // low-level config layer that should not depend on the debug module.
243
- console.warn(
254
+ log.warn(
255
+ 'ExecutionV1',
244
256
  `[ExecutionV1] Section "${name}" has ${steps.length} expanded steps ` +
245
257
  `(max recommended: ${MAX_STEPS_PER_SECTION}). This may indicate a misconfigured execution graph.`
246
258
  );
@@ -267,10 +279,7 @@ export function expandExecutionV1(graph, options = {}) {
267
279
  if (strict) {
268
280
  throw new Error(message);
269
281
  }
270
- // Use console.warn directly here because the schema module is a
271
- // low-level config layer that should not depend on the debug module.
272
- // Callers in runtime code can catch the strict error or filter post-hoc.
273
- console.warn(message);
282
+ log.warn('ExecutionV1', message);
274
283
  }
275
284
  }
276
285