doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -7,7 +7,12 @@
7
7
  * @module inference/pipelines/text/ops
8
8
  */
9
9
 
10
- import type { SiLURowSplitOptions, CommandRecorder } from '../../../gpu/kernel-selector.js';
10
+ import type {
11
+ SiLURowSplitOptions,
12
+ CommandRecorder,
13
+ SandwichRMSNormPairOptions,
14
+ SandwichRMSNormPairResult,
15
+ } from '../../../gpu/kernel-selector.js';
11
16
  import type { Tensor } from '../../../gpu/tensor.js';
12
17
  import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
13
18
  import type { DecodeBufferManager } from '../../decode-buffers.js';
@@ -35,10 +40,31 @@ export function doRMSNorm(
35
40
  input: Tensor,
36
41
  weight: GPUBuffer,
37
42
  eps: number,
38
- options: { batchSize: number; hiddenSize: number; residual?: Tensor | null; outputBuffer?: GPUBuffer | null; label?: string; layerIdx?: number; rmsNormWeightOffset?: boolean },
43
+ options: {
44
+ batchSize: number;
45
+ hiddenSize: number;
46
+ residual?: Tensor | null;
47
+ preResidual?: Tensor | null;
48
+ residualSumOutput?: GPUBuffer | Tensor | null;
49
+ outputBuffer?: GPUBuffer | null;
50
+ outputScale?: number | null;
51
+ label?: string;
52
+ layerIdx?: number;
53
+ rmsNormWeightOffset?: boolean;
54
+ },
39
55
  recorder?: CommandRecorder
40
56
  ): Promise<Tensor>;
41
57
 
58
+ export function doSandwichRMSNormPair(
59
+ input: Tensor,
60
+ residual: Tensor | null,
61
+ postWeight: GPUBuffer | WeightBuffer | CpuWeightBuffer,
62
+ preWeight: GPUBuffer | WeightBuffer | CpuWeightBuffer,
63
+ eps: number,
64
+ options?: SandwichRMSNormPairOptions,
65
+ recorder?: CommandRecorder
66
+ ): Promise<SandwichRMSNormPairResult>;
67
+
42
68
  /**
43
69
  * ResidualAdd that uses record variant when recorder is provided.
44
70
  * Accepts Tensor for inputs, returns Tensor.
@@ -52,6 +78,7 @@ export function doResidualAdd(
52
78
  label?: string;
53
79
  layerIdx?: number;
54
80
  outputBuffer?: GPUBuffer | null;
81
+ outputScale?: number | null;
55
82
  executionPolicies?: ExecutionV1PoliciesSchema | null;
56
83
  }
57
84
  ): Promise<Tensor>;
@@ -5,6 +5,7 @@ import {
5
5
  recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
6
6
  runSiLURowSplit, recordSiLURowSplit,
7
7
  runMatmulRMSNormFused, recordMatmulRMSNormFused,
8
+ runSandwichRMSNormPair, recordSandwichRMSNormPair,
8
9
  runConv2D, recordConv2D,
9
10
  } from '../../../gpu/kernel-selector.js';
10
11
  import {
@@ -57,6 +58,21 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
57
58
  return result;
58
59
  }
59
60
 
61
+ export async function doSandwichRMSNormPair(input, residual, postWeight, preWeight, eps, options, recorder) {
62
+ const result = recorder
63
+ ? await recordSandwichRMSNormPair(recorder, input, residual, postWeight, preWeight, eps, options)
64
+ : await runSandwichRMSNormPair(input, residual, postWeight, preWeight, eps, options);
65
+
66
+ if (kernelTrace.enabled && !recorder) {
67
+ const layer = options.layerIdx ?? -1;
68
+ const label = options.label ?? 'rmsnorm_pair';
69
+ await traceStep('rmsnorm_pair.post_attn', label, layer, result.postAttn.buffer, [options.batchSize, options.hiddenSize]);
70
+ await traceStep('rmsnorm_pair.pre_ffn', label, layer, result.ffnInput.buffer, [options.batchSize, options.hiddenSize]);
71
+ }
72
+
73
+ return result;
74
+ }
75
+
60
76
 
61
77
  export async function doResidualAdd(a, b, size, recorder, traceOptions) {
62
78
  const options = {};
@@ -66,6 +82,9 @@ export async function doResidualAdd(a, b, size, recorder, traceOptions) {
66
82
  if (traceOptions?.executionPolicies) {
67
83
  options.executionPolicies = traceOptions.executionPolicies;
68
84
  }
85
+ if (traceOptions?.outputScale !== undefined) {
86
+ options.outputScale = traceOptions.outputScale;
87
+ }
69
88
  const result = recorder
70
89
  ? await recordResidualAdd(recorder, a, b, size, options)
71
90
  : await runResidualAdd(a, b, size, options);
@@ -872,6 +872,11 @@ export function hasGpuSplitPerLayerInputEmbeddings(context) {
872
872
  return false;
873
873
  }
874
874
 
875
+ const sessionConfig = context?.perLayerInputsSession ?? context?.config?.perLayerInputsSession ?? null;
876
+ if (getPleSplitTablePolicy(sessionConfig)) {
877
+ return true;
878
+ }
879
+
875
880
  const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
876
881
  if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
877
882
  return false;
@@ -194,11 +194,10 @@ export async function runProbes(stage, buffer, options) {
194
194
 
195
195
  const diagnostics = options.operatorDiagnostics ?? null;
196
196
  // TSIR fixture writer: when operatorDiagnostics.tsirFixture.dir is
197
- // set and the stage maps to one of the four TSIR boundary points
198
- // (post_input_norm/linear_qkv_proj/post_attn/layer_out), capture
199
- // the activation tensor as a .npy file under
200
- // <dir>/layer_<layerIdx>/<tsirStage>.npy. Builds the rung-5 frozen
201
- // Doppler reference fixture that the Doe rung ladder consumes.
197
+ // set and the stage maps to a Doppler-to-CSL handoff or TSIR
198
+ // boundary point, capture the activation tensor as a .npy file under
199
+ // <dir>/layer_<layerIdx>/<tsirStage>.npy for Doe validators and
200
+ // splice receipts.
202
201
  if (diagnostics?.tsirFixture?.dir) {
203
202
  const tsirRecord = await maybeWriteFixtureSnapshot(stage, buffer, {
204
203
  tsirFixture: diagnostics.tsirFixture,
@@ -26,7 +26,7 @@ export interface ResolvedSamplingConfig {
26
26
 
27
27
  /**
28
28
  * Per-call sampling options. All fields are optional; missing fields fall
29
- * through to runtime config defaults, then to schema defaults.
29
+ * through to resolved runtime config defaults.
30
30
  */
31
31
  export interface SamplingCallOptions {
32
32
  temperature?: number;
@@ -39,16 +39,10 @@ export interface SamplingCallOptions {
39
39
  * Merge per-call sampling options with runtime config defaults.
40
40
  * Returns a complete, validated sampling config object.
41
41
  *
42
- * Precedence: per-call opts > runtimeConfig.inference.sampling > schema defaults.
43
- *
44
- * Invalid values are clamped to defaults with a warning log rather than
45
- * throwing, so this is a non-breaking addition.
46
- *
47
- * @param opts - Per-call options (temperature, topK, topP, etc.)
48
- * @param runtimeConfig - Runtime config with inference.sampling defaults
49
- * @returns Complete sampling config with all fields defined
42
+ * Precedence: per-call opts > runtimeConfig.inference.sampling.
43
+ * Missing runtime fields and invalid call/runtime values throw before execution.
50
44
  */
51
45
  export function resolveSamplingConfig(
52
46
  opts: SamplingCallOptions | null | undefined,
53
- runtimeConfig: { inference?: { sampling?: Partial<ResolvedSamplingConfig> } } | null | undefined
47
+ runtimeConfig: { inference?: { sampling?: ResolvedSamplingConfig } } | null | undefined
54
48
  ): ResolvedSamplingConfig;
@@ -1,79 +1,118 @@
1
- import { log } from '../../../debug/index.js';
1
+ const REQUIRED_RUNTIME_SAMPLING_FIELDS = [
2
+ 'temperature',
3
+ 'topP',
4
+ 'topK',
5
+ 'repetitionPenalty',
6
+ 'repetitionPenaltyWindow',
7
+ 'greedyThreshold',
8
+ 'suppressSpecialTokens',
9
+ 'suppressSpecialLikeTokens',
10
+ 'suppressTokenIds',
11
+ ];
2
12
 
3
- const DEFAULT_SAMPLING_DEFAULTS = Object.freeze({
4
- temperature: 1.0,
5
- topP: 0.95,
6
- topK: 50,
7
- repetitionPenalty: 1.1,
8
- greedyThreshold: 0.01,
9
- repetitionPenaltyWindow: 100,
10
- });
11
-
12
- /**
13
- * Merge per-call sampling options with runtime config defaults.
14
- * Returns a complete, validated sampling config object.
15
- *
16
- * Precedence: per-call opts > runtimeConfig.inference.sampling > schema defaults.
17
- *
18
- * Invalid values are clamped to defaults with a warning log rather than
19
- * throwing, so this is a non-breaking addition.
20
- *
21
- * @param {Object} opts - Per-call options (temperature, topK, topP, etc.)
22
- * @param {Object} runtimeConfig - Runtime config with inference.sampling defaults
23
- * @returns {Object} Complete sampling config with all fields defined
24
- */
25
13
  export function resolveSamplingConfig(opts, runtimeConfig) {
26
- const samplingDefaults = runtimeConfig?.inference?.sampling ?? DEFAULT_SAMPLING_DEFAULTS;
27
-
28
- const raw = {
29
- temperature: opts?.temperature ?? samplingDefaults.temperature ?? DEFAULT_SAMPLING_DEFAULTS.temperature,
30
- topP: opts?.topP ?? samplingDefaults.topP ?? DEFAULT_SAMPLING_DEFAULTS.topP,
31
- topK: opts?.topK ?? samplingDefaults.topK ?? DEFAULT_SAMPLING_DEFAULTS.topK,
32
- repetitionPenalty: opts?.repetitionPenalty ?? samplingDefaults.repetitionPenalty ?? DEFAULT_SAMPLING_DEFAULTS.repetitionPenalty,
33
- repetitionPenaltyWindow: samplingDefaults.repetitionPenaltyWindow ?? DEFAULT_SAMPLING_DEFAULTS.repetitionPenaltyWindow,
34
- greedyThreshold: samplingDefaults.greedyThreshold ?? DEFAULT_SAMPLING_DEFAULTS.greedyThreshold,
35
- };
14
+ const samplingDefaults = runtimeConfig?.inference?.sampling;
15
+ if (!samplingDefaults || typeof samplingDefaults !== 'object' || Array.isArray(samplingDefaults)) {
16
+ throw new Error('[Sampling] runtimeConfig.inference.sampling is required.');
17
+ }
18
+ for (const field of REQUIRED_RUNTIME_SAMPLING_FIELDS) {
19
+ if (samplingDefaults[field] === undefined) {
20
+ throw new Error(`[Sampling] runtimeConfig.inference.sampling.${field} is required.`);
21
+ }
22
+ }
36
23
 
37
- const resolved = {
38
- temperature: clampWithWarning('temperature', raw.temperature, 0, Infinity, DEFAULT_SAMPLING_DEFAULTS.temperature),
39
- topP: clampWithWarning('topP', raw.topP, Number.MIN_VALUE, 1, DEFAULT_SAMPLING_DEFAULTS.topP),
40
- topK: clampWithWarning('topK', raw.topK, 1, Infinity, DEFAULT_SAMPLING_DEFAULTS.topK),
41
- repetitionPenalty: raw.repetitionPenalty,
42
- repetitionPenaltyWindow: raw.repetitionPenaltyWindow,
43
- greedyThreshold: raw.greedyThreshold,
24
+ return {
25
+ temperature: resolveSamplingNumber(
26
+ 'temperature',
27
+ opts?.temperature,
28
+ samplingDefaults.temperature,
29
+ (value) => value >= 0
30
+ ),
31
+ topP: resolveSamplingNumber(
32
+ 'topP',
33
+ opts?.topP,
34
+ samplingDefaults.topP,
35
+ (value) => value > 0 && value <= 1
36
+ ),
37
+ topK: resolveSamplingInteger(
38
+ 'topK',
39
+ opts?.topK,
40
+ samplingDefaults.topK,
41
+ (value) => value >= 1
42
+ ),
43
+ repetitionPenalty: resolveSamplingNumber(
44
+ 'repetitionPenalty',
45
+ opts?.repetitionPenalty,
46
+ samplingDefaults.repetitionPenalty,
47
+ (value) => value > 0
48
+ ),
49
+ repetitionPenaltyWindow: resolveSamplingInteger(
50
+ 'repetitionPenaltyWindow',
51
+ undefined,
52
+ samplingDefaults.repetitionPenaltyWindow,
53
+ (value) => value >= 0
54
+ ),
55
+ greedyThreshold: resolveSamplingNumber(
56
+ 'greedyThreshold',
57
+ undefined,
58
+ samplingDefaults.greedyThreshold,
59
+ (value) => value >= 0
60
+ ),
61
+ suppressSpecialTokens: resolveSamplingBoolean(
62
+ 'suppressSpecialTokens',
63
+ samplingDefaults.suppressSpecialTokens
64
+ ),
65
+ suppressSpecialLikeTokens: resolveSamplingBoolean(
66
+ 'suppressSpecialLikeTokens',
67
+ samplingDefaults.suppressSpecialLikeTokens
68
+ ),
69
+ suppressTokenIds: resolveSamplingTokenIdList(
70
+ 'suppressTokenIds',
71
+ samplingDefaults.suppressTokenIds
72
+ ),
44
73
  };
74
+ }
45
75
 
46
- // Ensure topK is an integer
47
- if (Number.isFinite(resolved.topK)) {
48
- resolved.topK = Math.max(1, Math.floor(resolved.topK));
76
+ function resolveSamplingBoolean(name, runtimeValue) {
77
+ if (runtimeValue === null) {
78
+ throw new Error(`[Sampling] ${name} cannot be null.`);
79
+ }
80
+ if (typeof runtimeValue !== 'boolean') {
81
+ throw new Error(`[Sampling] ${name} must be a boolean; got ${typeof runtimeValue}.`);
49
82
  }
83
+ return runtimeValue;
84
+ }
50
85
 
51
- return resolved;
86
+ function resolveSamplingTokenIdList(name, runtimeValue) {
87
+ if (!Array.isArray(runtimeValue)) {
88
+ throw new Error(`[Sampling] ${name} must be an array of token IDs.`);
89
+ }
90
+ return runtimeValue.map((tokenId, index) => {
91
+ if (!Number.isInteger(tokenId) || tokenId < 0) {
92
+ throw new Error(`[Sampling] ${name}[${index}] must be a non-negative integer token ID; got ${tokenId}.`);
93
+ }
94
+ return tokenId;
95
+ });
52
96
  }
53
97
 
54
- /**
55
- * Clamp a value to [min, max]. If the value is invalid, log a warning and
56
- * return the fallback default.
57
- *
58
- * @param {string} name - Parameter name for logging
59
- * @param {number} value - Raw value
60
- * @param {number} min - Minimum allowed value (inclusive)
61
- * @param {number} max - Maximum allowed value (inclusive)
62
- * @param {number} fallback - Default value to use if clamping fails
63
- * @returns {number} Clamped value
64
- */
65
- function clampWithWarning(name, value, min, max, fallback) {
98
+ function resolveSamplingNumber(name, callValue, runtimeValue, validate) {
99
+ const value = callValue === undefined ? runtimeValue : callValue;
100
+ if (value === null) {
101
+ throw new Error(`[Sampling] ${name} cannot be null.`);
102
+ }
66
103
  if (!Number.isFinite(value)) {
67
- log.warn('Sampling', `Invalid ${name}=${value}, using default=${fallback}`);
68
- return fallback;
104
+ throw new Error(`[Sampling] ${name} must be a finite number; got ${value}.`);
69
105
  }
70
- if (value < min) {
71
- log.warn('Sampling', `${name}=${value} below minimum ${min}, clamping to ${min}`);
72
- return min;
106
+ if (!validate(value)) {
107
+ throw new Error(`[Sampling] ${name} is outside the configured range; got ${value}.`);
73
108
  }
74
- if (value > max) {
75
- log.warn('Sampling', `${name}=${value} above maximum ${max}, clamping to ${max}`);
76
- return max;
109
+ return value;
110
+ }
111
+
112
+ function resolveSamplingInteger(name, callValue, runtimeValue, validate) {
113
+ const value = resolveSamplingNumber(name, callValue, runtimeValue, validate);
114
+ if (Math.floor(value) !== value) {
115
+ throw new Error(`[Sampling] ${name} must be an integer; got ${value}.`);
77
116
  }
78
117
  return value;
79
118
  }
@@ -58,12 +58,9 @@ export function softmax(logits) {
58
58
  return exps;
59
59
  }
60
60
 
61
- function countFiniteCandidates(logits, padTokenId) {
61
+ function countFiniteCandidates(logits) {
62
62
  let finiteCandidateCount = 0;
63
63
  for (let i = 0; i < logits.length; i++) {
64
- if (padTokenId != null && i === padTokenId) {
65
- continue;
66
- }
67
64
  if (Number.isFinite(logits[i])) {
68
65
  finiteCandidateCount += 1;
69
66
  }
@@ -71,13 +68,13 @@ function countFiniteCandidates(logits, padTokenId) {
71
68
  return finiteCandidateCount;
72
69
  }
73
70
 
74
- function assertFiniteSamplingCandidates(logits, padTokenId, label) {
75
- const finiteCandidateCount = countFiniteCandidates(logits, padTokenId);
71
+ function assertFiniteSamplingCandidates(logits, label) {
72
+ const finiteCandidateCount = countFiniteCandidates(logits);
76
73
  if (finiteCandidateCount > 0) {
77
74
  return;
78
75
  }
79
76
  throw new Error(
80
- `[Sampling] ${label} has no finite candidate logits after masking the pad token. ` +
77
+ `[Sampling] ${label} has no finite candidate logits after masking suppressed tokens. ` +
81
78
  'Upstream decode likely produced NaN/Inf or an all-masked distribution.'
82
79
  );
83
80
  }
@@ -217,13 +214,20 @@ function sampleFromLogitCandidates(candidates, temperature, seed, decode, debug,
217
214
 
218
215
 
219
216
  export function sample(logits, opts) {
220
- const { temperature, topP, topK, decode, debug = false, padTokenId, seed } = opts;
217
+ const { temperature, topP, topK, decode, debug = false, padTokenId, seed, suppressTokenIds } = opts;
221
218
 
222
219
  if (padTokenId !== undefined && padTokenId >= 0 && padTokenId < logits.length) {
223
220
  logits[padTokenId] = -Infinity;
224
221
  }
222
+ if (Array.isArray(suppressTokenIds)) {
223
+ for (const tokenId of suppressTokenIds) {
224
+ if (Number.isInteger(tokenId) && tokenId >= 0 && tokenId < logits.length) {
225
+ logits[tokenId] = -Infinity;
226
+ }
227
+ }
228
+ }
225
229
 
226
- assertFiniteSamplingCandidates(logits, padTokenId, 'Logits');
230
+ assertFiniteSamplingCandidates(logits, 'Logits');
227
231
 
228
232
  // Greedy (argmax) when temperature = 0
229
233
  if (temperature === 0) {
@@ -94,14 +94,22 @@ export class PipelineState {
94
94
  prefillTimeMs: 0,
95
95
  decodeTimeMs: 0,
96
96
  ttftMs: 0,
97
+ loadTiming: null,
98
+ pipelineLoadTiming: null,
97
99
  prefillTokens: 0,
98
100
  decodeTokens: 0,
99
101
  memoryUsageBytes: 0,
100
102
  tokensGenerated: 0,
101
103
  totalTimeMs: 0,
102
104
  decodeRecordMs: 0,
105
+ decodeRecordOps: 0,
106
+ decodeRecordPasses: 0,
107
+ decodeRecordOpLabels: {},
103
108
  decodeSubmitWaitMs: 0,
104
109
  decodeReadbackWaitMs: 0,
110
+ decodeReadbackMapWaitMs: 0,
111
+ decodeReadbackCleanupMs: 0,
112
+ decodeReadbackCopyMs: 0,
105
113
  prefillRecordMs: 0,
106
114
  prefillSubmitWaitMs: 0,
107
115
  prefillProfileSteps: [],
@@ -115,6 +123,9 @@ export class PipelineState {
115
123
  batchGuardReason: null,
116
124
  singleTokenSubmitWaitMs: 0,
117
125
  singleTokenReadbackWaitMs: 0,
126
+ singleTokenReadbackMapWaitMs: 0,
127
+ singleTokenReadbackCleanupMs: 0,
128
+ singleTokenReadbackCopyMs: 0,
118
129
  singleTokenOrchestrationMs: 0,
119
130
  plePreparedTokenCacheHits: 0,
120
131
  plePreparedTokenCacheMisses: 0,
@@ -131,6 +142,12 @@ export class PipelineState {
131
142
  totalBatchedTimeMs: 0,
132
143
  totalUnbatchedTimeMs: 0,
133
144
  gpuSubmissions: 0,
145
+ requestedBatchTokens: 0,
146
+ effectiveBatchTokens: 0,
147
+ executedBatchTokens: 0,
148
+ resolvedBatchTokens: 0,
149
+ maxBatchTokenCap: null,
150
+ batchClampCount: 0,
134
151
  };
135
152
 
136
153
  // Base URL for loading assets
@@ -0,0 +1,57 @@
1
+ export interface TsirFixture {
2
+ dir: string;
3
+ layerFilter?: number[] | null;
4
+ prefillOnly?: boolean;
5
+ pendingReads?: TsirPendingRead[];
6
+ records?: TsirFixtureRecord[];
7
+ qkvByLayer?: Map<number, unknown>;
8
+ }
9
+
10
+ export interface TsirPendingRead {
11
+ stage: string;
12
+ tsirStage: string;
13
+ layerIdx: number;
14
+ numTokens: number;
15
+ hiddenSize: number;
16
+ dtype: 'f16' | 'f32';
17
+ carry: GPUBuffer;
18
+ alignedBytes: number;
19
+ }
20
+
21
+ export interface TsirFixtureRecord {
22
+ stage: string;
23
+ tsirStage: string;
24
+ layerIdx: number;
25
+ filePath?: string;
26
+ shape?: number[];
27
+ dtype?: string;
28
+ byteLength?: number;
29
+ payloadByteLength?: number;
30
+ written: boolean;
31
+ note?: string;
32
+ }
33
+
34
+ export declare const TSIR_BOUNDARY_STAGES: string[];
35
+
36
+ export declare function mapStageToTsirBoundary(stage: string): string | null;
37
+
38
+ export declare function writeNpyF32(
39
+ filePath: string,
40
+ shape: readonly number[],
41
+ data: Float32Array
42
+ ): Promise<{ byteLength: number; payloadByteLength: number }>;
43
+
44
+ export declare function maybeWriteFixtureSnapshot(
45
+ stage: string,
46
+ buffer: Float32Array | GPUBuffer,
47
+ options: {
48
+ tsirFixture?: TsirFixture | null;
49
+ layerIdx: number;
50
+ numTokens: number;
51
+ hiddenSize: number;
52
+ dtype?: 'f16' | 'f32';
53
+ recorder?: { getEncoder?(): GPUCommandEncoder } | null;
54
+ }
55
+ ): Promise<TsirFixtureRecord | null>;
56
+
57
+ export declare function drainPendingTsirReads(tsirFixture: TsirFixture | null | undefined): Promise<TsirFixtureRecord[]>;
@@ -1,20 +1,18 @@
1
1
  // Doppler-side TSIR boundary-point fixture writer.
2
2
  //
3
- // Builds rung-5 frozen-Doppler-reference fixtures for the Doe rung
4
- // ladder (see /home/x/deco/doe/docs/cerebras-north-star.md and
5
- // /home/x/deco/doe/config/doe-frozen-doppler-reference.schema.json).
6
- // Captures activation snapshots at the four TSIR boundary points
7
- // (`post_rmsnorm`, `post_qkv`, `post_attn`, `post_ffn`) during a
3
+ // Builds frozen-Doppler-reference fixtures for the Doe Cerebras lane
4
+ // (see /home/x/deco/doe/config/doe-frozen-doppler-reference.schema.json).
5
+ // Captures handoff and TSIR boundary activation snapshots during a
8
6
  // reference inference, writes them as `.npy` files keyed by layer
9
- // index and probe-point name, and emits a manifest the Doe
10
- // validator (bench/tools/validate_frozen_doppler_reference.py) can
11
- // consume.
7
+ // index and probe-point name, and emits a manifest the Doe validator
8
+ // (bench/tools/validate_frozen_doppler_reference.py) can consume.
12
9
  //
13
10
  // Triggered when `operatorDiagnostics.tsirFixture` is set in the
14
11
  // inference context. The CLI flag --tsir-fixture-dir on
15
12
  // tools/run-program-bundle-reference.js plumbs it in.
16
13
  //
17
14
  // Stage name mapping (Doppler stage -> Doe TSIR boundary):
15
+ // layer_in -> pre_layer_input
18
16
  // post_input_norm -> post_rmsnorm
19
17
  // linear_qkv_proj -> post_qkv (fused QKV path; single matmul)
20
18
  // q_proj+k_proj+v_proj -> post_qkv (split QKV path; concatenated along
@@ -50,6 +48,7 @@ async function getNodeFs() {
50
48
  }
51
49
 
52
50
  const STAGE_TO_TSIR = {
51
+ layer_in: 'pre_layer_input',
53
52
  post_input_norm: 'post_rmsnorm',
54
53
  // Two QKV-projection paths exist in Doppler:
55
54
  // - Fused: `linear_qkv_proj` (linear-attention.js variants)
@@ -336,7 +335,7 @@ export async function maybeWriteFixtureSnapshot(stage, buffer, options) {
336
335
  // name `post_qkv`. For models with split q_proj/k_proj/v_proj (Gemma 4 31B,
337
336
  // etc.), we capture each individually for debug introspection AND emit the
338
337
  // concatenated Q∥K∥V tensor along the feature axis as `post_qkv.npy` so the
339
- // Doe-side rung-5 builder + validator + rung-7 oracle can consume it. The
338
+ // Doe-side builder, validator, and splice receipts can consume it. The
340
339
  // fused-QKV path (`linear_qkv_proj`) writes to `post_qkv.npy` directly and
341
340
  // is not re-synthesized.
342
341
  export async function drainPendingTsirReads(tsirFixture) {