doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -0,0 +1,395 @@
1
+ const UINT32_SCALE = 1 / 0x100000000;
2
+
3
+ export function createSeededRandom(seed) {
4
+ if (!Number.isInteger(seed)) {
5
+ throw new Error('DiffusionGemma seeded random requires an integer seed.');
6
+ }
7
+ let state = seed >>> 0;
8
+ return () => {
9
+ state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
10
+ return state * UINT32_SCALE;
11
+ };
12
+ }
13
+
14
+ function assertCanvas(canvas, canvasLength, label) {
15
+ if (!Array.isArray(canvas) && !ArrayBuffer.isView(canvas)) {
16
+ throw new Error(`${label} must be an array or typed array.`);
17
+ }
18
+ if (canvas.length !== canvasLength) {
19
+ throw new Error(`${label} length ${canvas.length} does not match canvasLength ${canvasLength}.`);
20
+ }
21
+ }
22
+
23
+ function randomToken(random, vocabSize) {
24
+ return Math.min(vocabSize - 1, Math.floor(random() * vocabSize));
25
+ }
26
+
27
+ function normalizeTemperature(value) {
28
+ if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
29
+ throw new Error(`DiffusionGemma temperature must be positive; got ${String(value)}.`);
30
+ }
31
+ return value;
32
+ }
33
+
34
+ export function resolveDenoisingTemperature(config, step) {
35
+ const maxSteps = config.maxDenoisingSteps;
36
+ if (!Number.isInteger(step) || step < 1 || step > maxSteps) {
37
+ throw new Error(`DiffusionGemma denoising step must be in [1, ${maxSteps}], got ${String(step)}.`);
38
+ }
39
+ return normalizeTemperature(config.tMin + ((config.tMax - config.tMin) * (step / maxSteps)));
40
+ }
41
+
42
+ export function initializeCanvas(config, random) {
43
+ if (typeof random !== 'function') {
44
+ throw new Error('DiffusionGemma canvas initialization requires a random function.');
45
+ }
46
+ const canvas = new Int32Array(config.canvasLength);
47
+ for (let i = 0; i < canvas.length; i += 1) {
48
+ canvas[i] = randomToken(random, config.vocabSize);
49
+ }
50
+ return canvas;
51
+ }
52
+
53
+ function getLogitsView(logits, tokenIndex, vocabSize) {
54
+ const start = tokenIndex * vocabSize;
55
+ const end = start + vocabSize;
56
+ if (logits.length < end) {
57
+ throw new Error(
58
+ `DiffusionGemma logits length ${logits.length} is too small for token ${tokenIndex} and vocabSize ${vocabSize}.`
59
+ );
60
+ }
61
+ return logits.subarray(start, end);
62
+ }
63
+
64
+ function softmaxStats(tokenLogits, processedLogits, temperature, random) {
65
+ let max = -Infinity;
66
+ for (let i = 0; i < tokenLogits.length; i += 1) {
67
+ const value = tokenLogits[i] / temperature;
68
+ processedLogits[i] = value;
69
+ if (value > max) max = value;
70
+ }
71
+
72
+ let sum = 0;
73
+ let weightedLogitSum = 0;
74
+ let argmaxToken = 0;
75
+ let argmaxValue = -Infinity;
76
+ const probs = new Float32Array(tokenLogits.length);
77
+ for (let i = 0; i < tokenLogits.length; i += 1) {
78
+ const scaled = processedLogits[i];
79
+ if (scaled > argmaxValue) {
80
+ argmaxValue = scaled;
81
+ argmaxToken = i;
82
+ }
83
+ const expValue = Math.exp(scaled - max);
84
+ probs[i] = expValue;
85
+ sum += expValue;
86
+ weightedLogitSum += expValue * scaled;
87
+ }
88
+ if (!Number.isFinite(sum) || sum <= 0) {
89
+ throw new Error('DiffusionGemma logits produced an invalid softmax denominator.');
90
+ }
91
+
92
+ const draw = random() * sum;
93
+ let cumulative = 0;
94
+ let sampledToken = tokenLogits.length - 1;
95
+ for (let i = 0; i < probs.length; i += 1) {
96
+ cumulative += probs[i];
97
+ if (draw <= cumulative) {
98
+ sampledToken = i;
99
+ break;
100
+ }
101
+ }
102
+
103
+ const entropy = Math.log(sum) + max - (weightedLogitSum / sum);
104
+ return {
105
+ argmaxToken,
106
+ sampledToken,
107
+ entropy,
108
+ };
109
+ }
110
+
111
+ export function applyEntropyBoundStep(currentCanvas, logits, config, options = {}) {
112
+ const random = options.random;
113
+ if (typeof random !== 'function') {
114
+ throw new Error('DiffusionGemma entropy-bound step requires a random function.');
115
+ }
116
+ assertCanvas(currentCanvas, config.canvasLength, 'currentCanvas');
117
+ if (!ArrayBuffer.isView(logits)) {
118
+ throw new Error('DiffusionGemma entropy-bound step requires typed-array logits.');
119
+ }
120
+ const temperature = normalizeTemperature(options.temperature);
121
+ const nextCanvas = new Int32Array(config.canvasLength);
122
+ const argmaxCanvas = new Int32Array(config.canvasLength);
123
+ const sampledCanvas = new Int32Array(config.canvasLength);
124
+ const entropies = new Float32Array(config.canvasLength);
125
+ const processedLogits = new Float32Array(config.canvasLength * config.vocabSize);
126
+ const entropyOrder = [];
127
+
128
+ for (let i = 0; i < config.canvasLength; i += 1) {
129
+ const stats = softmaxStats(
130
+ getLogitsView(logits, i, config.vocabSize),
131
+ getLogitsView(processedLogits, i, config.vocabSize),
132
+ temperature,
133
+ random
134
+ );
135
+ argmaxCanvas[i] = stats.argmaxToken;
136
+ sampledCanvas[i] = stats.sampledToken;
137
+ entropies[i] = stats.entropy;
138
+ entropyOrder.push({ index: i, entropy: stats.entropy });
139
+ }
140
+
141
+ entropyOrder.sort((a, b) => a.entropy - b.entropy || a.index - b.index);
142
+ const accepted = new Uint8Array(config.canvasLength);
143
+ let cumulativeEntropy = 0;
144
+ let acceptedCount = 0;
145
+ for (const entry of entropyOrder) {
146
+ if (cumulativeEntropy <= config.entropyBound) {
147
+ accepted[entry.index] = 1;
148
+ acceptedCount += 1;
149
+ }
150
+ cumulativeEntropy += entry.entropy;
151
+ }
152
+
153
+ for (let i = 0; i < config.canvasLength; i += 1) {
154
+ nextCanvas[i] = accepted[i] ? sampledCanvas[i] : randomToken(random, config.vocabSize);
155
+ }
156
+
157
+ return {
158
+ canvas: nextCanvas,
159
+ argmaxCanvas,
160
+ sampledCanvas,
161
+ entropies,
162
+ processedLogits,
163
+ accepted,
164
+ acceptedCount,
165
+ meanEntropy: entropies.reduce((sum, value) => sum + value, 0) / entropies.length,
166
+ };
167
+ }
168
+
169
+ export function applyEntropyBoundStatsStep(currentCanvas, stats, config, options = {}) {
170
+ const random = options.random;
171
+ if (typeof random !== 'function') {
172
+ throw new Error('DiffusionGemma entropy-bound stats step requires a random function.');
173
+ }
174
+ assertCanvas(currentCanvas, config.canvasLength, 'currentCanvas');
175
+ const argmaxCanvas = stats?.argmaxCanvas;
176
+ const entropies = stats?.entropies;
177
+ assertCanvas(argmaxCanvas, config.canvasLength, 'argmaxCanvas');
178
+ if (!(entropies instanceof Float32Array) || entropies.length !== config.canvasLength) {
179
+ throw new Error('DiffusionGemma entropy stats step requires entropies as Float32Array[canvasLength].');
180
+ }
181
+
182
+ const nextCanvas = new Int32Array(config.canvasLength);
183
+ const accepted = new Uint8Array(config.canvasLength);
184
+ const entropyOrder = [];
185
+ for (let i = 0; i < config.canvasLength; i += 1) {
186
+ const entropy = entropies[i];
187
+ if (!Number.isFinite(entropy) || entropy < 0) {
188
+ throw new Error(`DiffusionGemma entropy stats step received invalid entropy at ${i}.`);
189
+ }
190
+ entropyOrder.push({ index: i, entropy });
191
+ }
192
+ entropyOrder.sort((a, b) => a.entropy - b.entropy || a.index - b.index);
193
+
194
+ let cumulativeEntropy = 0;
195
+ let acceptedCount = 0;
196
+ for (const entry of entropyOrder) {
197
+ if (cumulativeEntropy <= config.entropyBound) {
198
+ accepted[entry.index] = 1;
199
+ acceptedCount += 1;
200
+ }
201
+ cumulativeEntropy += entry.entropy;
202
+ }
203
+
204
+ for (let i = 0; i < config.canvasLength; i += 1) {
205
+ nextCanvas[i] = accepted[i] ? argmaxCanvas[i] : randomToken(random, config.vocabSize);
206
+ }
207
+
208
+ return {
209
+ canvas: nextCanvas,
210
+ argmaxCanvas: Int32Array.from(argmaxCanvas),
211
+ sampledCanvas: Int32Array.from(argmaxCanvas),
212
+ entropies,
213
+ processedLogits: null,
214
+ accepted,
215
+ acceptedCount,
216
+ meanEntropy: entropies.reduce((sum, value) => sum + value, 0) / entropies.length,
217
+ };
218
+ }
219
+
220
+ export function updateStabilityState(previousArgmaxCanvas, argmaxCanvas, previousCounts, config) {
221
+ assertCanvas(argmaxCanvas, config.canvasLength, 'argmaxCanvas');
222
+ const counts = new Uint16Array(config.canvasLength);
223
+ if (config.stabilityThreshold === 0) {
224
+ counts.fill(0xffff);
225
+ return {
226
+ counts,
227
+ stableTokens: config.canvasLength,
228
+ allStable: true,
229
+ };
230
+ }
231
+ let stableTokens = 0;
232
+ for (let i = 0; i < config.canvasLength; i += 1) {
233
+ const previousMatches = previousArgmaxCanvas && previousArgmaxCanvas[i] === argmaxCanvas[i];
234
+ counts[i] = previousMatches ? ((previousCounts?.[i] ?? 0) + 1) : 0;
235
+ if (counts[i] >= config.stabilityThreshold) {
236
+ stableTokens += 1;
237
+ }
238
+ }
239
+ return {
240
+ counts,
241
+ stableTokens,
242
+ allStable: stableTokens === config.canvasLength,
243
+ };
244
+ }
245
+
246
+ function releaseSelfConditioningState(state) {
247
+ if (state && !ArrayBuffer.isView(state) && typeof state.release === 'function') {
248
+ state.release();
249
+ }
250
+ }
251
+
252
+ export async function denoiseCanvas(config, options) {
253
+ const logitsProvider = options?.logitsProvider;
254
+ const random = options?.random;
255
+ if (typeof logitsProvider !== 'function') {
256
+ throw new Error('DiffusionGemma denoiseCanvas requires a logitsProvider function.');
257
+ }
258
+ if (typeof random !== 'function') {
259
+ throw new Error('DiffusionGemma denoiseCanvas requires a random function.');
260
+ }
261
+
262
+ let canvas = options.initialCanvas == null
263
+ ? initializeCanvas(config, random)
264
+ : Int32Array.from(options.initialCanvas);
265
+ assertCanvas(canvas, config.canvasLength, 'initialCanvas');
266
+ let previousArgmaxCanvas = null;
267
+ let stabilityCounts = null;
268
+ let selfConditioningLogits = options.selfConditioningLogits ?? null;
269
+ let lastStep = null;
270
+ let stepsRun = 0;
271
+
272
+ for (let step = config.maxDenoisingSteps; step >= 1; step -= 1) {
273
+ stepsRun += 1;
274
+ const temperature = resolveDenoisingTemperature(config, step);
275
+ const logits = await logitsProvider({
276
+ canvas,
277
+ step,
278
+ temperature,
279
+ selfConditioningLogits,
280
+ canvasIndex: options.canvasIndex ?? 0,
281
+ inputIds: options.inputIds ?? null,
282
+ });
283
+ const stepResult = applyEntropyBoundStep(canvas, logits, config, {
284
+ temperature,
285
+ random,
286
+ });
287
+ const stability = updateStabilityState(
288
+ previousArgmaxCanvas,
289
+ stepResult.argmaxCanvas,
290
+ stabilityCounts,
291
+ config
292
+ );
293
+
294
+ canvas = stepResult.canvas;
295
+ previousArgmaxCanvas = stepResult.argmaxCanvas;
296
+ stabilityCounts = stability.counts;
297
+ selfConditioningLogits = stepResult.processedLogits;
298
+ lastStep = {
299
+ ...stepResult,
300
+ step,
301
+ temperature,
302
+ stability,
303
+ };
304
+
305
+ if (stability.allStable && stepResult.meanEntropy < config.confidenceThreshold) {
306
+ break;
307
+ }
308
+ }
309
+
310
+ return {
311
+ canvas,
312
+ argmaxCanvas: previousArgmaxCanvas,
313
+ selfConditioningLogits,
314
+ lastStep,
315
+ stepsRun,
316
+ };
317
+ }
318
+
319
+ export async function denoiseCanvasWithStatsProvider(config, options) {
320
+ const statsProvider = options?.statsProvider;
321
+ const random = options?.random;
322
+ if (typeof statsProvider !== 'function') {
323
+ throw new Error('DiffusionGemma denoiseCanvasWithStatsProvider requires a statsProvider function.');
324
+ }
325
+ if (typeof random !== 'function') {
326
+ throw new Error('DiffusionGemma denoiseCanvasWithStatsProvider requires a random function.');
327
+ }
328
+
329
+ let canvas = options.initialCanvas == null
330
+ ? initializeCanvas(config, random)
331
+ : Int32Array.from(options.initialCanvas);
332
+ assertCanvas(canvas, config.canvasLength, 'initialCanvas');
333
+ let previousArgmaxCanvas = null;
334
+ let stabilityCounts = null;
335
+ const initialSelfConditioningLogits = options.selfConditioningLogits ?? null;
336
+ let selfConditioningLogits = initialSelfConditioningLogits;
337
+ let lastStep = null;
338
+ let stepsRun = 0;
339
+ let completed = false;
340
+
341
+ try {
342
+ for (let step = config.maxDenoisingSteps; step >= 1; step -= 1) {
343
+ stepsRun += 1;
344
+ const temperature = resolveDenoisingTemperature(config, step);
345
+ const inputSelfConditioningLogits = selfConditioningLogits;
346
+ selfConditioningLogits = null;
347
+ const stats = await statsProvider({
348
+ canvas,
349
+ step,
350
+ temperature,
351
+ selfConditioningLogits: inputSelfConditioningLogits,
352
+ canvasIndex: options.canvasIndex ?? 0,
353
+ inputIds: options.inputIds ?? null,
354
+ });
355
+ selfConditioningLogits = stats.selfConditioningLogits ?? null;
356
+ const stepResult = applyEntropyBoundStatsStep(canvas, stats, config, {
357
+ temperature,
358
+ random,
359
+ });
360
+ const stability = updateStabilityState(
361
+ previousArgmaxCanvas,
362
+ stepResult.argmaxCanvas,
363
+ stabilityCounts,
364
+ config
365
+ );
366
+
367
+ canvas = stepResult.canvas;
368
+ previousArgmaxCanvas = stepResult.argmaxCanvas;
369
+ stabilityCounts = stability.counts;
370
+ lastStep = {
371
+ ...stepResult,
372
+ step,
373
+ temperature,
374
+ stability,
375
+ };
376
+
377
+ if (stability.allStable && stepResult.meanEntropy < config.confidenceThreshold) {
378
+ break;
379
+ }
380
+ }
381
+
382
+ completed = true;
383
+ return {
384
+ canvas,
385
+ argmaxCanvas: previousArgmaxCanvas,
386
+ selfConditioningLogits,
387
+ lastStep,
388
+ stepsRun,
389
+ };
390
+ } finally {
391
+ if (!completed && selfConditioningLogits !== initialSelfConditioningLogits) {
392
+ releaseSelfConditioningState(selfConditioningLogits);
393
+ }
394
+ }
395
+ }
@@ -10,6 +10,7 @@
10
10
  */
11
11
 
12
12
  import type { Tensor } from '../../../../gpu/tensor.js';
13
+ import type { CommandRecorder } from '../../../../gpu/command-recorder.js';
13
14
  import type { AttentionState } from './types.js';
14
15
 
15
16
  /**
@@ -47,6 +48,10 @@ export interface KVCacheDispatchState {
47
48
  bdpaBasisCount: number;
48
49
  hasCache: boolean;
49
50
  totalSeqLen: number;
51
+ diffusionGemmaDecoder?: boolean;
52
+ ownedBuffers?: GPUBuffer[] | null;
53
+ encoderSeqLen?: number;
54
+ encoderWindow?: number;
50
55
  }
51
56
 
52
57
  /**
@@ -108,12 +113,31 @@ export function isSlidingLayerType(layerType: string | undefined): boolean;
108
113
  export function resolveKVCacheState(
109
114
  state: AttentionState,
110
115
  layerIdx: number,
111
- kTensor: Tensor,
112
- vTensor: Tensor,
116
+ kTensor: Tensor | null,
117
+ vTensor: Tensor | null,
113
118
  currentSeqLen: number,
114
119
  numTokens: number
115
120
  ): KVCacheDispatchState;
116
121
 
122
+ export interface DiffusionGemmaDecoderKVStateInput {
123
+ state: AttentionState;
124
+ layerIdx: number;
125
+ kTensor: Tensor;
126
+ vTensor: Tensor;
127
+ currentSeqLen: number;
128
+ numTokens: number;
129
+ numKVHeads: number;
130
+ headDim: number;
131
+ layerType?: string | null;
132
+ slidingWindow?: number | null;
133
+ kvDtype?: string | null;
134
+ recorder?: CommandRecorder | null;
135
+ }
136
+
137
+ export function createDiffusionGemmaDecoderKVState(
138
+ args: DiffusionGemmaDecoderKVStateInput
139
+ ): Promise<KVCacheDispatchState>;
140
+
117
141
  /**
118
142
  * Build attention dispatch parameters from the KV cache state and layer config.
119
143
  *
@@ -124,8 +148,8 @@ export function resolveKVCacheState(
124
148
  export function buildAttentionDispatchParams(
125
149
  config: DispatchParamsConfig,
126
150
  state: AttentionState,
127
- kTensor: Tensor,
128
- vTensor: Tensor,
151
+ kTensor: Tensor | null,
152
+ vTensor: Tensor | null,
129
153
  kvState: KVCacheDispatchState
130
154
  ): AttentionDispatchParams;
131
155
 
@@ -3,6 +3,8 @@
3
3
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
4
4
  import { createTensor } from '../../../../gpu/tensor.js';
5
5
  import { SlidingWindowKVCache } from '../../../kv-cache.js';
6
+ import { getDevice } from '../../../../gpu/device.js';
7
+ import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
6
8
 
7
9
  // ============================================================================
8
10
  // Layer Type Helpers
@@ -141,6 +143,173 @@ export function resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSe
141
143
  return kvState;
142
144
  }
143
145
 
146
+ function resolveDecoderEncoderWindow(encoderSeqLen, slidingWindow, layerType) {
147
+ if (!isSlidingLayerType(layerType)) {
148
+ return encoderSeqLen;
149
+ }
150
+ if (!Number.isFinite(slidingWindow) || slidingWindow <= 1) {
151
+ return encoderSeqLen;
152
+ }
153
+ return Math.min(encoderSeqLen, Math.max(0, Math.trunc(slidingWindow) - 1));
154
+ }
155
+
156
+ function resolveDecoderKVCacheBuffers(state, layerIdx) {
157
+ if (!state.kvCache?.hasGPUCache?.()) {
158
+ throw new Error(
159
+ `DiffusionGemma decoder attention at layer ${layerIdx} requires an initialized encoder KV cache.`
160
+ );
161
+ }
162
+ const gpuBuffers = state.kvCache.getGPUBuffers(layerIdx);
163
+ const layout = gpuBuffers?.layout ?? state.kvCache?.layout ?? 'contiguous';
164
+ if (layout !== 'contiguous' && layout !== undefined && layout !== null) {
165
+ throw new Error(
166
+ `DiffusionGemma decoder attention requires contiguous encoder KV cache at layer ${layerIdx}; got "${layout}".`
167
+ );
168
+ }
169
+ if (!gpuBuffers?.keysGPU || !gpuBuffers?.valuesGPU) {
170
+ throw new Error(
171
+ `DiffusionGemma decoder attention missing contiguous GPU KV buffers at layer ${layerIdx}.`
172
+ );
173
+ }
174
+ if (!Number.isFinite(gpuBuffers.seqLen) || gpuBuffers.seqLen < 0) {
175
+ throw new Error(
176
+ `DiffusionGemma decoder attention received invalid encoder KV length at layer ${layerIdx}: ${String(gpuBuffers.seqLen)}.`
177
+ );
178
+ }
179
+ return gpuBuffers;
180
+ }
181
+
182
+ function copyKVRange(encoder, source, sourceOffset, target, targetOffset, size) {
183
+ if (size <= 0) return;
184
+ encoder.copyBufferToBuffer(source, sourceOffset, target, targetOffset, size);
185
+ }
186
+
187
+ export async function createDiffusionGemmaDecoderKVState({
188
+ state,
189
+ layerIdx,
190
+ kTensor,
191
+ vTensor,
192
+ currentSeqLen,
193
+ numTokens,
194
+ numKVHeads,
195
+ headDim,
196
+ layerType,
197
+ slidingWindow,
198
+ kvDtype,
199
+ recorder = null,
200
+ }) {
201
+ const gpuBuffers = resolveDecoderKVCacheBuffers(state, layerIdx);
202
+ const encoderSeqLen = Math.trunc(gpuBuffers.seqLen);
203
+ if (encoderSeqLen !== currentSeqLen) {
204
+ throw new Error(
205
+ `DiffusionGemma decoder attention expected currentSeqLen=${encoderSeqLen} from encoder KV cache, ` +
206
+ `got ${currentSeqLen}.`
207
+ );
208
+ }
209
+
210
+ const encoderWindow = resolveDecoderEncoderWindow(encoderSeqLen, slidingWindow, layerType);
211
+ const concatSeqLen = encoderWindow + numTokens;
212
+ if (concatSeqLen <= 0) {
213
+ throw new Error(`DiffusionGemma decoder attention has empty KV span at layer ${layerIdx}.`);
214
+ }
215
+
216
+ const dtype = kvDtype ?? kTensor.dtype;
217
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
218
+ const rowBytes = numKVHeads * headDim * bytesPerElement;
219
+ const encoderStart = encoderSeqLen - encoderWindow;
220
+ const encoderBytes = encoderWindow * rowBytes;
221
+ const canvasBytes = numTokens * rowBytes;
222
+ const concatBytes = concatSeqLen * rowBytes;
223
+ let keysGPU = null;
224
+ let valuesGPU = null;
225
+ try {
226
+ keysGPU = acquireBuffer(concatBytes, undefined, 'diffusion_gemma_decoder_keys');
227
+ valuesGPU = acquireBuffer(concatBytes, undefined, 'diffusion_gemma_decoder_values');
228
+ const device = recorder?.device ?? getDevice();
229
+ if (!device) {
230
+ throw new Error('DiffusionGemma decoder attention requires a GPU device.');
231
+ }
232
+ const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
233
+ copyKVRange(
234
+ encoder,
235
+ gpuBuffers.keysGPU,
236
+ encoderStart * rowBytes,
237
+ keysGPU,
238
+ 0,
239
+ encoderBytes
240
+ );
241
+ copyKVRange(
242
+ encoder,
243
+ gpuBuffers.valuesGPU,
244
+ encoderStart * rowBytes,
245
+ valuesGPU,
246
+ 0,
247
+ encoderBytes
248
+ );
249
+ copyKVRange(
250
+ encoder,
251
+ kTensor.buffer,
252
+ 0,
253
+ keysGPU,
254
+ encoderBytes,
255
+ canvasBytes
256
+ );
257
+ copyKVRange(
258
+ encoder,
259
+ vTensor.buffer,
260
+ 0,
261
+ valuesGPU,
262
+ encoderBytes,
263
+ canvasBytes
264
+ );
265
+ if (!recorder) {
266
+ device.queue.submit([encoder.finish()]);
267
+ }
268
+ } catch (error) {
269
+ if (keysGPU) releaseBuffer(keysGPU);
270
+ if (valuesGPU && valuesGPU !== keysGPU) releaseBuffer(valuesGPU);
271
+ throw error;
272
+ }
273
+
274
+ return {
275
+ cachedK: keysGPU,
276
+ cachedV: valuesGPU,
277
+ kvLenForAttention: concatSeqLen,
278
+ causalForAttention: false,
279
+ startPosForMask: encoderSeqLen,
280
+ kvStart: 0,
281
+ kvLayout: 'contiguous',
282
+ kvPageTable: null,
283
+ kvPageSize: 0,
284
+ cachedKHot: undefined,
285
+ cachedVHot: undefined,
286
+ cachedKCold: undefined,
287
+ cachedVCold: undefined,
288
+ coldScalesK: null,
289
+ coldScalesV: null,
290
+ coldPackedStride: 0,
291
+ coldQuantMode: 'none',
292
+ coldLen: 0,
293
+ hotLen: 0,
294
+ hotStart: 0,
295
+ hotWindow: 0,
296
+ coldPageTable: null,
297
+ coldPageSize: 0,
298
+ bdpaBasisK: null,
299
+ bdpaBasisV: null,
300
+ bdpaPagedK: null,
301
+ bdpaPagedV: null,
302
+ bdpaIndex: null,
303
+ bdpaBasisCount: 0,
304
+ hasCache: true,
305
+ totalSeqLen: concatSeqLen,
306
+ diffusionGemmaDecoder: true,
307
+ ownedBuffers: [keysGPU, valuesGPU],
308
+ encoderSeqLen,
309
+ encoderWindow,
310
+ };
311
+ }
312
+
144
313
  // ============================================================================
145
314
  // Dispatch Parameter Construction
146
315
  // ============================================================================
@@ -221,13 +390,18 @@ export function buildAttentionDispatchParams(config, state, kTensor, vTensor, kv
221
390
  const attnScale = queryPreAttnScalar ? 1.0 / Math.sqrt(queryPreAttnScalar) : 1.0 / Math.sqrt(headDim);
222
391
 
223
392
  // Cached K/V dtypes
393
+ const kFallbackDtype = kTensor?.dtype ?? resolvedKvCacheDtype;
394
+ const vFallbackDtype = vTensor?.dtype ?? resolvedKvCacheDtype;
395
+ if (!kFallbackDtype || !vFallbackDtype) {
396
+ throw new Error('Attention dispatch requires K/V tensor dtypes or an explicit KV cache dtype.');
397
+ }
224
398
  const cachedKDtype = selectRuleValue('inference', 'dtype', 'f16OrFallback', {
225
399
  kvDtype: resolvedKvCacheDtype,
226
- fallback: kTensor.dtype,
400
+ fallback: kFallbackDtype,
227
401
  });
228
402
  const cachedVDtype = selectRuleValue('inference', 'dtype', 'f16OrFallback', {
229
403
  kvDtype: resolvedKvCacheDtype,
230
- fallback: vTensor.dtype,
404
+ fallback: vFallbackDtype,
231
405
  });
232
406
 
233
407
  // Cached K/V tensors (null for tiered, contiguousQuant, and prefill-fallback paths)