doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -1,6 +1,6 @@
1
1
 
2
-
3
2
  import { getDevice } from '../../gpu/device.js';
3
+ import { recordKVCacheWriteF32ToF16 } from '../../gpu/kernel-selector.js';
4
4
  import { allowReadback } from '../../gpu/perf-guards.js';
5
5
  import { log } from '../../debug/index.js';
6
6
  import { readBuffer } from '../../memory/buffer-pool.js';
@@ -473,6 +473,114 @@ export class KVCache {
473
473
  this.currentSeqLen = Math.max(this.currentSeqLen, startPos + numTokens);
474
474
  }
475
475
 
476
+
477
+ async recordUpdateF32ToF16FromGPU(
478
+ recorder,
479
+ layerIdx,
480
+ keysBuffer,
481
+ valuesBuffer,
482
+ startPos,
483
+ numTokens
484
+ ) {
485
+ this._assertLayerIndex(layerIdx);
486
+ this._assertStartPos(startPos);
487
+ if (this.kvDtype !== 'f16') {
488
+ throw new Error('KVCache recordUpdateF32ToF16FromGPU requires an f16 KV cache.');
489
+ }
490
+ if (!Number.isInteger(numTokens) || numTokens < 0) {
491
+ throw new Error('KVCache recordUpdateF32ToF16FromGPU requires a non-negative integer token count.');
492
+ }
493
+ if (numTokens === 0) {
494
+ return;
495
+ }
496
+ this.counters.recordedGpuUpdateCalls += 1;
497
+ this.counters.tokensWritten += numTokens;
498
+
499
+ const layer = (this.layers[layerIdx]);
500
+ if (!layer.keysGPU) {
501
+ throw new Error('GPU cache not initialized');
502
+ }
503
+
504
+ if (startPos + numTokens > this.maxSeqLen) {
505
+ throw new Error(
506
+ `Cache overflow: ${startPos + numTokens} > ${this.maxSeqLen}`
507
+ );
508
+ }
509
+
510
+ const elementCount = numTokens * this.kvSize;
511
+ await recordKVCacheWriteF32ToF16(
512
+ recorder,
513
+ keysBuffer,
514
+ valuesBuffer,
515
+ layer.keysGPU,
516
+ layer.valuesGPU,
517
+ {
518
+ srcOffset: 0,
519
+ dstOffset: startPos * this.kvSize,
520
+ elementCount,
521
+ }
522
+ );
523
+
524
+ if (this.layout === 'paged') {
525
+ const neededPages = Math.ceil((startPos + numTokens) / this.pageSize);
526
+ if (Number.isFinite(layer.allocatedPages)) {
527
+ layer.allocatedPages = Math.max(layer.allocatedPages, neededPages);
528
+ } else {
529
+ layer.allocatedPages = neededPages;
530
+ }
531
+ }
532
+
533
+ layer.seqLen = Math.max(layer.seqLen, startPos + numTokens);
534
+ this.totalTokensSeen = Math.max(this.totalTokensSeen, startPos + numTokens);
535
+
536
+ this.currentSeqLen = Math.max(this.currentSeqLen, startPos + numTokens);
537
+ }
538
+
539
+ recordF16UpdateAlreadyWrittenFromGPU(
540
+ layerIdx,
541
+ startPos,
542
+ numTokens
543
+ ) {
544
+ this._assertLayerIndex(layerIdx);
545
+ this._assertStartPos(startPos);
546
+ if (this.kvDtype !== 'f16') {
547
+ throw new Error('KVCache recordF16UpdateAlreadyWrittenFromGPU requires an f16 KV cache.');
548
+ }
549
+ if (this.layout !== 'contiguous') {
550
+ throw new Error('KVCache recordF16UpdateAlreadyWrittenFromGPU requires contiguous layout.');
551
+ }
552
+ if (!Number.isInteger(numTokens) || numTokens < 0) {
553
+ throw new Error('KVCache recordF16UpdateAlreadyWrittenFromGPU requires a non-negative integer token count.');
554
+ }
555
+ if (numTokens === 0) {
556
+ return;
557
+ }
558
+
559
+ const layer = (this.layers[layerIdx]);
560
+ if (!isContiguousLayer(layer) || !layer.keysGPU || !layer.valuesGPU) {
561
+ throw new Error('GPU cache not initialized');
562
+ }
563
+ if (startPos + numTokens > this.maxSeqLen) {
564
+ throw new Error(
565
+ `Cache overflow: ${startPos + numTokens} > ${this.maxSeqLen}`
566
+ );
567
+ }
568
+
569
+ const requiredBytes = (startPos + numTokens) * this.kvSize * this.bytesPerElem;
570
+ if (Number.isFinite(layer.keysGPU.size) && requiredBytes > layer.keysGPU.size) {
571
+ throw new Error('KVCache direct f16 keys write exceeds GPU cache buffer size.');
572
+ }
573
+ if (Number.isFinite(layer.valuesGPU.size) && requiredBytes > layer.valuesGPU.size) {
574
+ throw new Error('KVCache direct f16 values write exceeds GPU cache buffer size.');
575
+ }
576
+
577
+ this.counters.recordedGpuUpdateCalls += 1;
578
+ this.counters.tokensWritten += numTokens;
579
+ layer.seqLen = Math.max(layer.seqLen, startPos + numTokens);
580
+ this.totalTokensSeen = Math.max(this.totalTokensSeen, startPos + numTokens);
581
+ this.currentSeqLen = Math.max(this.currentSeqLen, startPos + numTokens);
582
+ }
583
+
476
584
 
477
585
  _updateContiguous(
478
586
  layer,
@@ -12,6 +12,7 @@ export interface MixedGeometryKVCacheConfig {
12
12
  pageSize: number;
13
13
  kvDtype: 'f16' | 'f32';
14
14
  slidingWindow?: number | null;
15
+ slidingLayerLayout: 'ring' | 'contiguous';
15
16
  layerTypes: string[];
16
17
  }
17
18
 
@@ -70,6 +71,15 @@ export declare class MixedGeometryKVCache {
70
71
  numTokens: number,
71
72
  tokenIds?: number[] | null
72
73
  ): void;
74
+ recordUpdateF32ToF16FromGPU(
75
+ recorder: import('../../gpu/kernel-selector.js').CommandRecorder,
76
+ layerIdx: number,
77
+ keysBuffer: GPUBuffer,
78
+ valuesBuffer: GPUBuffer,
79
+ startPos: number,
80
+ numTokens: number,
81
+ tokenIds?: number[] | null
82
+ ): void | Promise<void>;
73
83
  get(): never;
74
84
  getKeyCache(layerIdx: number): GPUBuffer | null;
75
85
  getValueCache(layerIdx: number): GPUBuffer | null;
@@ -1,4 +1,5 @@
1
1
  import { getDevice } from '../../gpu/device.js';
2
+ import { recordKVCacheWriteF32ToF16 } from '../../gpu/kernel-selector.js';
2
3
 
3
4
  function normalizeLayerType(layerType) {
4
5
  return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
@@ -19,6 +20,12 @@ function buildLayerSpecs(config, bytesPerElem) {
19
20
  'MixedGeometryKVCache requires explicit layerTypes for every layer so per-layer KV geometry stays manifest-owned.'
20
21
  );
21
22
  }
23
+ const slidingLayerLayout = config.slidingLayerLayout;
24
+ if (slidingLayerLayout !== 'ring' && slidingLayerLayout !== 'contiguous') {
25
+ throw new Error(
26
+ 'MixedGeometryKVCache requires slidingLayerLayout to be "ring" or "contiguous".'
27
+ );
28
+ }
22
29
 
23
30
  const baseHeadDim = Number(config.headDim);
24
31
  const globalHeadDim = Number.isFinite(config.globalHeadDim) && config.globalHeadDim > 0
@@ -46,7 +53,10 @@ function buildLayerSpecs(config, bytesPerElem) {
46
53
  return layerTypes.map((layerType, layerIdx) => {
47
54
  const isSliding = isSlidingLayerType(layerType);
48
55
  const headDim = isSliding ? baseHeadDim : globalHeadDim;
49
- const capacityTokens = isSliding && slidingWindow != null
56
+ const layout = isSliding && slidingWindow != null && slidingLayerLayout === 'ring'
57
+ ? 'ring'
58
+ : 'contiguous';
59
+ const capacityTokens = layout === 'ring'
50
60
  ? slidingWindow
51
61
  : maxSeqLen;
52
62
  const layerNumHeads = isSliding ? numHeads : globalNumHeads;
@@ -55,7 +65,7 @@ function buildLayerSpecs(config, bytesPerElem) {
55
65
  return {
56
66
  layerIdx,
57
67
  layerType,
58
- layout: isSliding && slidingWindow != null ? 'ring' : 'contiguous',
68
+ layout,
59
69
  headDim,
60
70
  numHeads: layerNumHeads,
61
71
  kvSize,
@@ -154,6 +164,7 @@ export class MixedGeometryKVCache {
154
164
  globalHeadDim: Number.isFinite(config.globalHeadDim) ? Math.trunc(config.globalHeadDim) : null,
155
165
  maxSeqLen: this.maxSeqLen,
156
166
  pageSize: this.pageSize,
167
+ slidingLayerLayout: config.slidingLayerLayout,
157
168
  };
158
169
  this.layerSpecs = buildLayerSpecs(this._config, this.bytesPerElem);
159
170
  this.layers = new Array(this.numLayers);
@@ -206,6 +217,27 @@ export class MixedGeometryKVCache {
206
217
  encoder.copyBufferToBuffer(valuesBuffer, 0, layer.valuesGPU, byteOffset, byteSize);
207
218
  }
208
219
 
220
+ async _recordContiguousF32ToF16(recorder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens) {
221
+ const endPos = startPos + numTokens;
222
+ if (endPos > spec.capacityTokens) {
223
+ throw new Error(
224
+ `MixedGeometryKVCache overflow at layer ${spec.layerIdx}: ${endPos} > ${spec.capacityTokens}.`
225
+ );
226
+ }
227
+ await recordKVCacheWriteF32ToF16(
228
+ recorder,
229
+ keysBuffer,
230
+ valuesBuffer,
231
+ layer.keysGPU,
232
+ layer.valuesGPU,
233
+ {
234
+ srcOffset: 0,
235
+ dstOffset: startPos * spec.kvSize,
236
+ elementCount: numTokens * spec.kvSize,
237
+ }
238
+ );
239
+ }
240
+
209
241
  _recordRingCopy(encoder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens) {
210
242
  const windowSize = spec.capacityTokens;
211
243
  const bytesPerToken = spec.bytesPerToken;
@@ -245,6 +277,60 @@ export class MixedGeometryKVCache {
245
277
  layer.seqLen = Math.min(windowSize, Math.max(layer.seqLen, seen));
246
278
  }
247
279
 
280
+ async _recordRingF32ToF16(recorder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens) {
281
+ const windowSize = spec.capacityTokens;
282
+ const fullStart = startPos;
283
+ const fullTokens = numTokens;
284
+ let srcElementOffset = 0;
285
+
286
+ if (numTokens > windowSize) {
287
+ const dropTokens = numTokens - windowSize;
288
+ startPos += dropTokens;
289
+ numTokens = windowSize;
290
+ srcElementOffset = dropTokens * spec.kvSize;
291
+ }
292
+
293
+ const sourceBytesNeeded = (srcElementOffset + (numTokens * spec.kvSize)) * 4;
294
+ if (sourceBytesNeeded > keysBuffer.size || sourceBytesNeeded > valuesBuffer.size) {
295
+ throw new Error('MixedGeometryKVCache ring f32-to-f16 write buffer is smaller than the requested write.');
296
+ }
297
+
298
+ const writePos = startPos % windowSize;
299
+ const firstChunkTokens = Math.min(numTokens, windowSize - writePos);
300
+ const secondChunkTokens = numTokens - firstChunkTokens;
301
+
302
+ await recordKVCacheWriteF32ToF16(
303
+ recorder,
304
+ keysBuffer,
305
+ valuesBuffer,
306
+ layer.keysGPU,
307
+ layer.valuesGPU,
308
+ {
309
+ srcOffset: srcElementOffset,
310
+ dstOffset: writePos * spec.kvSize,
311
+ elementCount: firstChunkTokens * spec.kvSize,
312
+ }
313
+ );
314
+
315
+ if (secondChunkTokens > 0) {
316
+ await recordKVCacheWriteF32ToF16(
317
+ recorder,
318
+ keysBuffer,
319
+ valuesBuffer,
320
+ layer.keysGPU,
321
+ layer.valuesGPU,
322
+ {
323
+ srcOffset: srcElementOffset + (firstChunkTokens * spec.kvSize),
324
+ dstOffset: 0,
325
+ elementCount: secondChunkTokens * spec.kvSize,
326
+ }
327
+ );
328
+ }
329
+
330
+ const seen = Math.max(this.totalTokensSeen, fullStart + fullTokens);
331
+ layer.seqLen = Math.min(windowSize, Math.max(layer.seqLen, seen));
332
+ }
333
+
248
334
  _updateMetadata(layer, spec, startPos, numTokens) {
249
335
  if (spec.layout !== 'ring') {
250
336
  layer.seqLen = Math.max(layer.seqLen, Math.min(startPos + numTokens, spec.capacityTokens));
@@ -304,6 +390,27 @@ export class MixedGeometryKVCache {
304
390
  this._updateMetadata(layer, spec, startPos, numTokens);
305
391
  }
306
392
 
393
+ async recordUpdateF32ToF16FromGPU(recorder, layerIdx, keysBuffer, valuesBuffer, startPos, numTokens) {
394
+ this._assertLayerIndex(layerIdx);
395
+ this._assertStartPos(startPos);
396
+ if (this.kvDtype !== 'f16') {
397
+ throw new Error('MixedGeometryKVCache recordUpdateF32ToF16FromGPU requires an f16 KV cache.');
398
+ }
399
+ this._assertTokenCount(numTokens, 'MixedGeometryKVCache recordUpdateF32ToF16FromGPU');
400
+ if (numTokens === 0) {
401
+ return;
402
+ }
403
+
404
+ const spec = this.layerSpecs[layerIdx];
405
+ const layer = this.layers[layerIdx];
406
+ if (spec.layout === 'ring') {
407
+ await this._recordRingF32ToF16(recorder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens);
408
+ } else {
409
+ await this._recordContiguousF32ToF16(recorder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens);
410
+ }
411
+ this._updateMetadata(layer, spec, startPos, numTokens);
412
+ }
413
+
307
414
  get() {
308
415
  throw new Error('MixedGeometryKVCache CPU get() is not implemented.');
309
416
  }
@@ -68,5 +68,19 @@ export class SlidingWindowKVCache extends KVCache {
68
68
  tokenIds?: number[] | null
69
69
  ): void;
70
70
 
71
+ /**
72
+ * Record direct f32 source to f16 GPU KV cache update with ring-buffer semantics.
73
+ * Does NOT submit - caller is responsible for submitting the recorder.
74
+ */
75
+ recordUpdateF32ToF16FromGPU(
76
+ recorder: import('../../gpu/kernel-selector.js').CommandRecorder,
77
+ layerIdx: number,
78
+ keysBuffer: GPUBuffer,
79
+ valuesBuffer: GPUBuffer,
80
+ startPos: number,
81
+ numTokens: number,
82
+ tokenIds?: number[] | null
83
+ ): void | Promise<void>;
84
+
71
85
  getMemoryStats(): MemoryStats & { windowSize: number; totalTokensSeen: number };
72
86
  }
@@ -1,6 +1,7 @@
1
1
 
2
2
 
3
3
  import { getDevice } from '../../gpu/device.js';
4
+ import { recordKVCacheWriteF32ToF16 } from '../../gpu/kernel-selector.js';
4
5
  import { KVCache } from './base.js';
5
6
 
6
7
  // ============================================================================
@@ -200,6 +201,93 @@ export class SlidingWindowKVCache extends KVCache {
200
201
  }
201
202
  }
202
203
 
204
+
205
+ async recordUpdateF32ToF16FromGPU(
206
+ recorder,
207
+ layerIdx,
208
+ keysBuffer,
209
+ valuesBuffer,
210
+ startPos,
211
+ numTokens
212
+ ) {
213
+ this._assertLayerIndex(layerIdx);
214
+ this._assertStartPos(startPos);
215
+ if (this.kvDtype !== 'f16') {
216
+ throw new Error('SlidingWindowKVCache recordUpdateF32ToF16FromGPU requires an f16 KV cache.');
217
+ }
218
+ if (!Number.isInteger(numTokens) || numTokens < 0) {
219
+ throw new Error('SlidingWindowKVCache recordUpdateF32ToF16FromGPU requires a non-negative integer token count.');
220
+ }
221
+ if (numTokens === 0) {
222
+ return;
223
+ }
224
+
225
+ const layer = (this.layers[layerIdx]);
226
+ if (!layer.keysGPU) {
227
+ throw new Error('GPU cache not initialized');
228
+ }
229
+
230
+ const windowSize = this.windowSize;
231
+ const sourceBytesPerToken = this.kvSize * 4;
232
+ const fullStart = startPos;
233
+ const fullTokens = numTokens;
234
+ let srcElementOffset = 0;
235
+
236
+ if (numTokens > windowSize) {
237
+ const dropTokens = numTokens - windowSize;
238
+ startPos += dropTokens;
239
+ numTokens = windowSize;
240
+ srcElementOffset = dropTokens * this.kvSize;
241
+ }
242
+ const writePos = startPos % windowSize;
243
+ const sourceBytesNeeded = (srcElementOffset * 4) + (numTokens * sourceBytesPerToken);
244
+ if (sourceBytesNeeded > keysBuffer.size || sourceBytesNeeded > valuesBuffer.size) {
245
+ throw new Error('SlidingWindowKVCache recordUpdateF32ToF16FromGPU buffer size is smaller than requested write.');
246
+ }
247
+
248
+ const firstChunkTokens = Math.min(numTokens, windowSize - writePos);
249
+ const secondChunkTokens = numTokens - firstChunkTokens;
250
+
251
+ await recordKVCacheWriteF32ToF16(
252
+ recorder,
253
+ keysBuffer,
254
+ valuesBuffer,
255
+ layer.keysGPU,
256
+ layer.valuesGPU,
257
+ {
258
+ srcOffset: srcElementOffset,
259
+ dstOffset: writePos * this.kvSize,
260
+ elementCount: firstChunkTokens * this.kvSize,
261
+ }
262
+ );
263
+
264
+ if (secondChunkTokens > 0) {
265
+ await recordKVCacheWriteF32ToF16(
266
+ recorder,
267
+ keysBuffer,
268
+ valuesBuffer,
269
+ layer.keysGPU,
270
+ layer.valuesGPU,
271
+ {
272
+ srcOffset: srcElementOffset + (firstChunkTokens * this.kvSize),
273
+ dstOffset: 0,
274
+ elementCount: secondChunkTokens * this.kvSize,
275
+ }
276
+ );
277
+ }
278
+
279
+ const seen = Math.max(this.totalTokensSeen, fullStart + fullTokens);
280
+ const storedLen = Math.min(windowSize, Math.max(0, seen));
281
+ this.totalTokensSeen = Math.max(this.totalTokensSeen, seen);
282
+ const prevLayerSeqLen = Math.min(Math.max(layer.seqLen || 0, 0), windowSize);
283
+
284
+ layer.seqLen = Math.max(prevLayerSeqLen, storedLen);
285
+ if (layerIdx === this.numLayers - 1) {
286
+ this.currentSeqLen = Math.min(this.currentSeqLen, windowSize);
287
+ this.currentSeqLen = Math.max(this.currentSeqLen, storedLen);
288
+ }
289
+ }
290
+
203
291
 
204
292
  _slideWindow(numNewTokens) {
205
293
  if (!Number.isInteger(numNewTokens) || numNewTokens <= 0) {
@@ -0,0 +1,61 @@
1
+ import type { CatalogModelLane } from '../../config/model-lanes/catalog-lane-resolver.js';
2
+
3
+ export interface KernelCapabilities {
4
+ hasF16?: boolean;
5
+ hasSubgroups?: boolean;
6
+ hasSubgroupsF16?: boolean;
7
+ maxBufferSize?: number;
8
+ maxWorkgroupSize?: number;
9
+ maxWorkgroupStorageSize?: number;
10
+ adapterInfo?: {
11
+ vendor?: string;
12
+ architecture?: string;
13
+ device?: string;
14
+ description?: string;
15
+ };
16
+ [key: string]: unknown;
17
+ }
18
+
19
+ export interface ExecutionLaneSelection {
20
+ entry: CatalogModelLane;
21
+ selectedModelId: string | null;
22
+ usedFallback: boolean;
23
+ rejected: Array<{
24
+ modelId: string | null;
25
+ reason: string;
26
+ }>;
27
+ reason: string;
28
+ }
29
+
30
+ export declare function hasF16SubgroupLaneSupport(capabilities: KernelCapabilities | null | undefined): boolean;
31
+
32
+ export declare function assertExecutionLaneManifestSupported(
33
+ entry: CatalogModelLane | null | undefined,
34
+ manifest: Record<string, unknown>,
35
+ capabilities: KernelCapabilities | null | undefined,
36
+ options?: {
37
+ normalizeManifest?: (manifest: Record<string, unknown>) => Record<string, unknown>;
38
+ kernelPathPolicy?: unknown;
39
+ platform?: {
40
+ id?: string;
41
+ vendor?: string;
42
+ architecture?: string;
43
+ };
44
+ }
45
+ ): void;
46
+
47
+ export declare function selectExecutionLaneForCapabilities(
48
+ entry: CatalogModelLane,
49
+ options?: {
50
+ capabilities?: KernelCapabilities | null;
51
+ manifestByModelId?: Map<string, Record<string, unknown>> | Record<string, Record<string, unknown>>;
52
+ manifests?: Map<string, Record<string, unknown>> | Record<string, Record<string, unknown>>;
53
+ normalizeManifest?: (manifest: Record<string, unknown>) => Record<string, unknown>;
54
+ kernelPathPolicy?: unknown;
55
+ platform?: {
56
+ id?: string;
57
+ vendor?: string;
58
+ architecture?: string;
59
+ };
60
+ }
61
+ ): ExecutionLaneSelection;
@@ -0,0 +1,126 @@
1
+ import { DEFAULT_RUNTIME_CONFIG } from '../../config/schema/index.js';
2
+ import { compileExecutionV1 } from '../pipelines/text/execution-v1.js';
3
+
4
+ export function hasF16SubgroupLaneSupport(capabilities) {
5
+ return capabilities?.hasF16 === true && capabilities?.hasSubgroups === true;
6
+ }
7
+
8
+ function buildPlatform(capabilities) {
9
+ return {
10
+ id: 'model-lane-runtime',
11
+ vendor: capabilities?.adapterInfo?.vendor ?? 'unknown',
12
+ architecture: capabilities?.adapterInfo?.architecture ?? 'unknown',
13
+ };
14
+ }
15
+
16
+ function getManifest(manifestByModelId, modelId) {
17
+ if (!manifestByModelId || !modelId) return null;
18
+ if (typeof manifestByModelId.get === 'function') return manifestByModelId.get(modelId) ?? null;
19
+ return manifestByModelId[modelId] ?? null;
20
+ }
21
+
22
+ function cloneManifest(manifest) {
23
+ return typeof structuredClone === 'function'
24
+ ? structuredClone(manifest)
25
+ : JSON.parse(JSON.stringify(manifest));
26
+ }
27
+
28
+ export function assertExecutionLaneManifestSupported(entry, manifest, capabilities, options = {}) {
29
+ const normalizedManifest = typeof options.normalizeManifest === 'function'
30
+ ? options.normalizeManifest(cloneManifest(manifest))
31
+ : cloneManifest(manifest);
32
+ compileExecutionV1({
33
+ manifestInference: normalizedManifest.inference,
34
+ modelId: normalizedManifest.modelId ?? entry?.modelId ?? 'unknown',
35
+ numLayers: normalizedManifest.architecture?.numLayers ?? 0,
36
+ headDim: normalizedManifest.architecture?.headDim ?? null,
37
+ capabilities,
38
+ platform: options.platform ?? buildPlatform(capabilities),
39
+ kernelPathPolicy: options.kernelPathPolicy ?? DEFAULT_RUNTIME_CONFIG.inference.kernelPathPolicy,
40
+ });
41
+ }
42
+
43
+ function unsupportedError(entry, fallback, preferredError, fallbackError) {
44
+ return new Error(
45
+ `${entry?.modelId ?? 'unknown'}: preferred lane is unsupported on this device (${preferredError.message}); ` +
46
+ `${fallback?.modelId ?? 'unknown'}: fallback lane is also unsupported (${fallbackError.message}).`
47
+ );
48
+ }
49
+
50
+ export function selectExecutionLaneForCapabilities(entry, options = {}) {
51
+ const capabilities = options.capabilities;
52
+ const manifestByModelId = options.manifestByModelId ?? options.manifests;
53
+ const fallback = entry?.demoFallbackVariant ?? null;
54
+ const manifest = getManifest(manifestByModelId, entry?.modelId);
55
+ if (!manifest) {
56
+ throw new Error(`${entry?.modelId ?? 'unknown'}: execution lane preflight missing manifest.`);
57
+ }
58
+
59
+ const assertSupported = (candidate, candidateManifest) => assertExecutionLaneManifestSupported(
60
+ candidate,
61
+ candidateManifest,
62
+ capabilities,
63
+ {
64
+ normalizeManifest: options.normalizeManifest,
65
+ kernelPathPolicy: options.kernelPathPolicy,
66
+ platform: options.platform,
67
+ }
68
+ );
69
+
70
+ if (!fallback) {
71
+ assertSupported(entry, manifest);
72
+ return {
73
+ entry,
74
+ selectedModelId: entry?.modelId ?? null,
75
+ usedFallback: false,
76
+ rejected: [],
77
+ reason: 'primary lane supported',
78
+ };
79
+ }
80
+
81
+ const fallbackManifest = getManifest(manifestByModelId, fallback.modelId);
82
+ if (!fallbackManifest) {
83
+ throw new Error(`${fallback.modelId ?? 'unknown'}: execution lane fallback preflight missing manifest.`);
84
+ }
85
+
86
+ if (!hasF16SubgroupLaneSupport(capabilities)) {
87
+ assertSupported(fallback, fallbackManifest);
88
+ return {
89
+ entry: fallback,
90
+ selectedModelId: fallback.modelId ?? null,
91
+ usedFallback: true,
92
+ rejected: [{
93
+ modelId: entry?.modelId ?? null,
94
+ reason: 'device lacks shader-f16 plus subgroups',
95
+ }],
96
+ reason: 'fallback selected because preferred lane requires shader-f16 plus subgroups',
97
+ };
98
+ }
99
+
100
+ try {
101
+ assertSupported(entry, manifest);
102
+ return {
103
+ entry,
104
+ selectedModelId: entry?.modelId ?? null,
105
+ usedFallback: false,
106
+ rejected: [],
107
+ reason: 'preferred lane supported',
108
+ };
109
+ } catch (preferredError) {
110
+ try {
111
+ assertSupported(fallback, fallbackManifest);
112
+ return {
113
+ entry: fallback,
114
+ selectedModelId: fallback.modelId ?? null,
115
+ usedFallback: true,
116
+ rejected: [{
117
+ modelId: entry?.modelId ?? null,
118
+ reason: preferredError.message,
119
+ }],
120
+ reason: 'fallback selected after preferred lane preflight rejection',
121
+ };
122
+ } catch (fallbackError) {
123
+ throw unsupportedError(entry, fallback, preferredError, fallbackError);
124
+ }
125
+ }
126
+ }
@@ -53,6 +53,8 @@ interface LoadBalanceStats {
53
53
  totalTokens: number;
54
54
  }
55
55
 
56
+ type RouterVector = Float32Array | GPUBuffer | import('../gpu/weight-buffer.js').WeightBuffer;
57
+
56
58
  /**
57
59
  * Expert utilization stats
58
60
  */
@@ -77,6 +79,10 @@ export declare class MoERouter {
77
79
  gateWeight: Float32Array | GPUBuffer | import('../gpu/weight-buffer.js').WeightBuffer | null;
78
80
  // Router bias (optional, used by GPT-OSS)
79
81
  gateBias: Float32Array | GPUBuffer | null;
82
+ // Optional DiffusionGemma router input scale
83
+ gateScale: RouterVector | null;
84
+ // Optional DiffusionGemma per-expert output scale
85
+ perExpertScale: RouterVector | null;
80
86
 
81
87
  // Track active experts for the current batch
82
88
  activeExperts: Set<number>;
@@ -91,8 +97,15 @@ export declare class MoERouter {
91
97
  * Load router gate weights from model
92
98
  * @param weights - Gate weight matrix [hidden_size, num_experts]
93
99
  * @param bias - Optional gate bias vector [num_experts]
100
+ * @param scale - Optional router input scale vector [hidden_size]
101
+ * @param perExpertScale - Optional selected-expert weight scale [num_experts]
94
102
  */
95
- loadWeights(weights: Float32Array | GPUBuffer, bias?: Float32Array | GPUBuffer | null): void;
103
+ loadWeights(
104
+ weights: Float32Array | GPUBuffer | import('../gpu/weight-buffer.js').WeightBuffer,
105
+ bias?: Float32Array | GPUBuffer | null,
106
+ scale?: RouterVector | null,
107
+ perExpertScale?: RouterVector | null
108
+ ): void;
96
109
 
97
110
  /**
98
111
  * Compute router logits from hidden states (CPU fallback)