doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -4,8 +4,8 @@ import {
4
4
  buildLayerPipelineFromExecution,
5
5
  buildSessionRuntimePatch,
6
6
  PIPELINE_COMPATIBLE_OPS,
7
- normalizeDtype,
8
7
  requireSessionActivationDtype,
8
+ requireSessionKVDtype,
9
9
  } from './execution-runtime-builders.js';
10
10
  import { mergeKernelPathPolicy } from '../../../config/merge-helpers.js';
11
11
  import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
@@ -15,8 +15,8 @@ import {
15
15
  resolveFinitenessFallbackTransform,
16
16
  } from '../../../config/transforms/capability-transform-resolver.js';
17
17
  import { composeTransforms } from '../../../config/transforms/execution-graph-transforms.js';
18
- import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
19
18
  import { log } from '../../../debug/index.js';
19
+ import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
20
20
 
21
21
  const SESSION_CAPABILITY_TRANSFORMS = new Set([
22
22
  'disableRetainQ4KMaterialization',
@@ -43,6 +43,226 @@ function hasOwnProperty(value, key) {
43
43
  return value != null && Object.prototype.hasOwnProperty.call(value, key);
44
44
  }
45
45
 
46
+ function hasPatchEntries(value, label) {
47
+ if (value == null) {
48
+ return false;
49
+ }
50
+ if (!Array.isArray(value)) {
51
+ throw new Error(`[ExecutionV1] ${label} must be an array when provided.`);
52
+ }
53
+ return value.length > 0;
54
+ }
55
+
56
+ function cloneExecutionTuple(tuple) {
57
+ return Array.isArray(tuple) ? [...tuple] : tuple;
58
+ }
59
+
60
+ function cloneExecutionEntry(entry) {
61
+ if (Array.isArray(entry)) {
62
+ return cloneExecutionTuple(entry);
63
+ }
64
+ if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
65
+ return entry;
66
+ }
67
+ return {
68
+ ...entry,
69
+ layers: Array.isArray(entry.layers) ? [...entry.layers] : entry.layers,
70
+ steps: Array.isArray(entry.steps) ? entry.steps.map((step) => cloneExecutionTuple(step)) : entry.steps,
71
+ };
72
+ }
73
+
74
+ function cloneKernelDeclaration(kernel) {
75
+ if (!kernel || typeof kernel !== 'object' || Array.isArray(kernel)) {
76
+ return kernel;
77
+ }
78
+ return {
79
+ ...kernel,
80
+ ...(kernel.constants ? { constants: { ...kernel.constants } } : {}),
81
+ ...(kernel.precision ? { precision: { ...kernel.precision } } : {}),
82
+ };
83
+ }
84
+
85
+ function cloneExecutionGraph(execution) {
86
+ return {
87
+ ...execution,
88
+ kernels: Object.fromEntries(
89
+ Object.entries(execution?.kernels ?? {}).map(
90
+ ([key, kernel]) => [key, cloneKernelDeclaration(kernel)]
91
+ )
92
+ ),
93
+ preLayer: (execution?.preLayer ?? []).map((entry) => cloneExecutionEntry(entry)),
94
+ decode: (execution?.decode ?? []).map((entry) => cloneExecutionEntry(entry)),
95
+ prefill: (execution?.prefill ?? []).map((entry) => cloneExecutionEntry(entry)),
96
+ postLayer: (execution?.postLayer ?? []).map((entry) => cloneExecutionEntry(entry)),
97
+ policies: execution?.policies ? { ...execution.policies } : execution?.policies,
98
+ };
99
+ }
100
+
101
+ function normalizePatchSection(section, label) {
102
+ if (section == null) {
103
+ return null;
104
+ }
105
+ if (typeof section !== 'string') {
106
+ throw new Error(`[ExecutionV1] ${label}.section must be a string when provided.`);
107
+ }
108
+ const normalized = section.trim();
109
+ if (normalized === 'decode' || normalized === 'prefill' || normalized === 'preLayer' || normalized === 'postLayer') {
110
+ return normalized;
111
+ }
112
+ const lower = normalized.toLowerCase();
113
+ if (lower === 'prelayer') {
114
+ return 'preLayer';
115
+ }
116
+ if (lower === 'postlayer') {
117
+ return 'postLayer';
118
+ }
119
+ throw new Error(
120
+ `[ExecutionV1] ${label}.section must be one of decode|prefill|preLayer|postLayer.`
121
+ );
122
+ }
123
+
124
+ function assertPatchLayersUnsupported(layers, label) {
125
+ if (layers != null) {
126
+ throw new Error(`[ExecutionV1] ${label}.layers is not supported yet.`);
127
+ }
128
+ }
129
+
130
+ function applyPatchKernelAdditions(execution, addKernels) {
131
+ for (const [index, addition] of addKernels.entries()) {
132
+ const label = `runtime.inference.executionPatch.addKernels[${index}]`;
133
+ if (!addition || typeof addition !== 'object' || Array.isArray(addition)) {
134
+ throw new Error(`[ExecutionV1] ${label} must be an object.`);
135
+ }
136
+ const key = typeof addition.key === 'string' ? addition.key.trim() : '';
137
+ if (!key) {
138
+ throw new Error(`[ExecutionV1] ${label}.key must be a non-empty string.`);
139
+ }
140
+ if (hasOwnProperty(execution.kernels, key)) {
141
+ throw new Error(`[ExecutionV1] ${label}.key "${key}" already exists in execution.kernels.`);
142
+ }
143
+ if (!addition.kernel || typeof addition.kernel !== 'object' || Array.isArray(addition.kernel)) {
144
+ throw new Error(`[ExecutionV1] ${label}.kernel must be an object.`);
145
+ }
146
+ execution.kernels[key] = cloneKernelDeclaration(addition.kernel);
147
+ }
148
+ }
149
+
150
+ function patchTuple(tuple, patch, label) {
151
+ const patched = [...tuple];
152
+ if (hasOwnProperty(patch, 'kernelKey') && patch.kernelKey !== undefined) {
153
+ const kernelKey = typeof patch.kernelKey === 'string' ? patch.kernelKey.trim() : '';
154
+ if (!kernelKey) {
155
+ throw new Error(`[ExecutionV1] ${label}.kernelKey must be a non-empty string.`);
156
+ }
157
+ patched[1] = kernelKey;
158
+ }
159
+ if (hasOwnProperty(patch, 'weights') && patch.weights !== undefined) {
160
+ const weights = typeof patch.weights === 'string' ? patch.weights.trim() : '';
161
+ if (!weights) {
162
+ throw new Error(`[ExecutionV1] ${label}.weights must be a non-empty string.`);
163
+ }
164
+ if (patched.length > 2) {
165
+ patched[2] = weights;
166
+ } else {
167
+ patched.push(weights);
168
+ }
169
+ }
170
+ return patched;
171
+ }
172
+
173
+ function applyPatchSetToEntries(entries, patch, label) {
174
+ let matchCount = 0;
175
+ for (let index = 0; index < entries.length; index += 1) {
176
+ const entry = entries[index];
177
+ if (Array.isArray(entry)) {
178
+ if (entry[0] !== patch.op) {
179
+ continue;
180
+ }
181
+ entries[index] = patchTuple(entry, patch, label);
182
+ matchCount += 1;
183
+ continue;
184
+ }
185
+ if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
186
+ continue;
187
+ }
188
+ for (let stepIndex = 0; stepIndex < entry.steps.length; stepIndex += 1) {
189
+ const step = entry.steps[stepIndex];
190
+ if (!Array.isArray(step) || step[0] !== patch.op) {
191
+ continue;
192
+ }
193
+ entry.steps[stepIndex] = patchTuple(step, patch, label);
194
+ matchCount += 1;
195
+ }
196
+ }
197
+ return matchCount;
198
+ }
199
+
200
+ function applyPatchSets(execution, setPatches) {
201
+ const defaultSections = ['preLayer', 'decode', 'prefill', 'postLayer'];
202
+ for (const [index, patch] of setPatches.entries()) {
203
+ const label = `runtime.inference.executionPatch.set[${index}]`;
204
+ if (!patch || typeof patch !== 'object' || Array.isArray(patch)) {
205
+ throw new Error(`[ExecutionV1] ${label} must be an object.`);
206
+ }
207
+ const op = typeof patch.op === 'string' ? patch.op.trim() : '';
208
+ if (!op) {
209
+ throw new Error(`[ExecutionV1] ${label}.op must be a non-empty string.`);
210
+ }
211
+ assertPatchLayersUnsupported(patch.layers, label);
212
+ if (!hasOwnProperty(patch, 'kernelKey') && !hasOwnProperty(patch, 'weights')) {
213
+ throw new Error(`[ExecutionV1] ${label} must provide kernelKey or weights.`);
214
+ }
215
+ const normalizedPatch = {
216
+ ...patch,
217
+ op,
218
+ section: normalizePatchSection(patch.section, label),
219
+ };
220
+ const sections = normalizedPatch.section ? [normalizedPatch.section] : defaultSections;
221
+ let matchCount = 0;
222
+ for (const section of sections) {
223
+ matchCount += applyPatchSetToEntries(execution[section] ?? [], normalizedPatch, label);
224
+ }
225
+ if (matchCount === 0) {
226
+ throw new Error(`[ExecutionV1] ${label} did not match any execution step.`);
227
+ }
228
+ }
229
+ }
230
+
231
+ function assertUnsupportedPatchLists(executionPatch) {
232
+ if (hasPatchEntries(executionPatch.remove, 'runtime.inference.executionPatch.remove')) {
233
+ throw new Error('[ExecutionV1] runtime.inference.executionPatch.remove is not supported yet.');
234
+ }
235
+ if (hasPatchEntries(executionPatch.add, 'runtime.inference.executionPatch.add')) {
236
+ throw new Error('[ExecutionV1] runtime.inference.executionPatch.add is not supported yet.');
237
+ }
238
+ }
239
+
240
+ function applyExecutionPatch(execution, executionPatch) {
241
+ if (executionPatch == null) {
242
+ return execution;
243
+ }
244
+ if (typeof executionPatch !== 'object' || Array.isArray(executionPatch)) {
245
+ throw new Error('[ExecutionV1] runtime.inference.executionPatch must be an object.');
246
+ }
247
+ assertUnsupportedPatchLists(executionPatch);
248
+ const hasKernelAdditions = hasPatchEntries(
249
+ executionPatch.addKernels,
250
+ 'runtime.inference.executionPatch.addKernels'
251
+ );
252
+ const hasSets = hasPatchEntries(executionPatch.set, 'runtime.inference.executionPatch.set');
253
+ if (!hasKernelAdditions && !hasSets) {
254
+ return execution;
255
+ }
256
+ const patchedExecution = cloneExecutionGraph(execution);
257
+ if (hasKernelAdditions) {
258
+ applyPatchKernelAdditions(patchedExecution, executionPatch.addKernels);
259
+ }
260
+ if (hasSets) {
261
+ applyPatchSets(patchedExecution, executionPatch.set);
262
+ }
263
+ return patchedExecution;
264
+ }
265
+
46
266
  function resolveRuntimeInferenceOverrideSection(runtimeOverrides, key) {
47
267
  const inferenceOverrides = runtimeOverrides?.inference;
48
268
  if (!inferenceOverrides || typeof inferenceOverrides !== 'object' || Array.isArray(inferenceOverrides)) {
@@ -54,55 +274,25 @@ function resolveRuntimeInferenceOverrideSection(runtimeOverrides, key) {
54
274
  return inferenceOverrides[key] ?? null;
55
275
  }
56
276
 
57
- function applyExecutionV1RuntimeComputeOverrides(session, runtimeSession, runtimeCompute) {
58
- const activationOverrideRaw = runtimeCompute?.activationDtype;
59
- if (activationOverrideRaw == null) {
60
- return session;
277
+ function preserveRuntimeDecodeLoop(updatedInference, runtimeConfig) {
278
+ const runtimeSession = runtimeConfig?.inference?.session;
279
+ if (!hasOwnProperty(runtimeSession, 'decodeLoop')) {
280
+ return updatedInference;
61
281
  }
62
-
63
- const activationOverride = normalizeDtype(
64
- activationOverrideRaw,
65
- 'runtime.inference.compute.activationDtype'
66
- );
67
- const runtimeSessionActivation = runtimeSession?.compute?.defaults?.activationDtype;
68
- if (
69
- runtimeSessionActivation != null
70
- && normalizeDtype(
71
- runtimeSessionActivation,
72
- 'runtime.inference.session.compute.defaults.activationDtype'
73
- ) !== activationOverride
74
- ) {
75
- throw new Error(
76
- '[ExecutionV1] runtime.inference.compute.activationDtype conflicts with ' +
77
- 'runtime.inference.session.compute.defaults.activationDtype. ' +
78
- 'Set both to the same dtype or remove one override.'
79
- );
80
- }
81
-
82
- const runtimeSessionOutput = runtimeSession?.compute?.defaults?.outputDtype;
83
- if (
84
- runtimeSessionOutput != null
85
- && normalizeDtype(
86
- runtimeSessionOutput,
87
- 'runtime.inference.session.compute.defaults.outputDtype'
88
- ) !== activationOverride
89
- ) {
90
- throw new Error(
91
- '[ExecutionV1] runtime.inference.compute.activationDtype conflicts with ' +
92
- 'runtime.inference.session.compute.defaults.outputDtype. ' +
93
- 'Set both to the same dtype or remove one override.'
94
- );
282
+ const updatedSession = updatedInference?.session;
283
+ if (!updatedSession || typeof updatedSession !== 'object' || Array.isArray(updatedSession)) {
284
+ return {
285
+ ...updatedInference,
286
+ session: {
287
+ decodeLoop: runtimeSession.decodeLoop,
288
+ },
289
+ };
95
290
  }
96
-
97
291
  return {
98
- ...session,
99
- compute: {
100
- ...(session?.compute ?? {}),
101
- defaults: {
102
- ...(session?.compute?.defaults ?? {}),
103
- activationDtype: activationOverride,
104
- outputDtype: activationOverride,
105
- },
292
+ ...updatedInference,
293
+ session: {
294
+ ...updatedSession,
295
+ decodeLoop: runtimeSession.decodeLoop,
106
296
  },
107
297
  };
108
298
  }
@@ -343,8 +533,8 @@ function expandV1ToResolvedSteps(execution, options = {}) {
343
533
  phase: step.phase,
344
534
  section: step.section,
345
535
  op: step.op,
346
- src: 'state',
347
- dst: 'state',
536
+ src: step.src,
537
+ dst: step.dst,
348
538
  kernel: step.kernel,
349
539
  entry: step.entry,
350
540
  ...(step.weights ? { weights: step.weights } : {}),
@@ -362,6 +552,99 @@ function expandV1ToResolvedSteps(execution, options = {}) {
362
552
  });
363
553
  }
364
554
 
555
+ function buildFinitenessFallbackSession(session, fallbackKvDtype) {
556
+ const kvDtype = fallbackKvDtype === 'f16' ? 'f16' : 'f32';
557
+ const computeDefaults = session?.compute?.defaults ?? {};
558
+ const fallbackDefaults = {
559
+ ...computeDefaults,
560
+ activationDtype: 'f32',
561
+ outputDtype: 'f32',
562
+ ...(kvDtype === 'f32' ? {
563
+ mathDtype: 'f32',
564
+ accumDtype: 'f32',
565
+ } : {}),
566
+ };
567
+ return {
568
+ ...session,
569
+ compute: {
570
+ ...(session.compute ?? {}),
571
+ defaults: fallbackDefaults,
572
+ },
573
+ kvcache: {
574
+ ...(session.kvcache ?? {}),
575
+ kvDtype,
576
+ },
577
+ };
578
+ }
579
+
580
+ function buildFinitenessFallbackKernelPath({
581
+ execution,
582
+ effectiveSession,
583
+ modelId,
584
+ numLayers,
585
+ headDim,
586
+ layerTypes,
587
+ capabilities,
588
+ platform,
589
+ }) {
590
+ const activationDtype = effectiveSession?.compute?.defaults?.activationDtype ?? null;
591
+ const mathDtype = effectiveSession?.compute?.defaults?.mathDtype ?? null;
592
+ const accumDtype = effectiveSession?.compute?.defaults?.accumDtype ?? null;
593
+ const kvDtype = effectiveSession?.kvcache?.kvDtype ?? null;
594
+ if (activationDtype !== 'f16' || !execution) {
595
+ return null;
596
+ }
597
+
598
+ const fallback = resolveFinitenessFallbackTransform({
599
+ activationDtype,
600
+ mathDtype,
601
+ accumDtype,
602
+ kvDtype,
603
+ headDim,
604
+ modelId,
605
+ layerTypes,
606
+ });
607
+ if (!fallback) {
608
+ return null;
609
+ }
610
+
611
+ const transformed = fallback.transform(execution, {
612
+ capabilities,
613
+ platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
614
+ activationDtype,
615
+ mathDtype,
616
+ accumDtype,
617
+ kvDtype,
618
+ headDim,
619
+ modelId,
620
+ layerTypes,
621
+ });
622
+ if (!transformed) {
623
+ return null;
624
+ }
625
+
626
+ const fallbackSession = buildFinitenessFallbackSession(
627
+ effectiveSession,
628
+ fallback.fallbackKvDtype
629
+ );
630
+ const fallbackSteps = expandV1ToResolvedSteps(transformed, { skipDigestValidation: true });
631
+ const fallbackKernelPath = buildInlineKernelPath(
632
+ fallbackSteps,
633
+ fallbackSession,
634
+ `${modelId || 'model'}-${fallback.name}-finiteness-fallback`,
635
+ numLayers
636
+ );
637
+ if (!fallbackKernelPath) {
638
+ return null;
639
+ }
640
+ return {
641
+ ...fallbackKernelPath,
642
+ name: 'Execution inline finiteness fallback kernel path',
643
+ description: `Generated from manifest.inference.execution using ${fallback.name}`,
644
+ finitenessFallbackTransform: fallback.name,
645
+ };
646
+ }
647
+
365
648
 
366
649
  export function compileExecutionV1(options = {}) {
367
650
  const manifestInference = options.manifestInference;
@@ -373,21 +656,20 @@ export function compileExecutionV1(options = {}) {
373
656
  const runtimeSession = options.runtimeSession ?? null;
374
657
  const runtimeCompute = options.runtimeCompute ?? null;
375
658
  const kernelPathPolicy = mergeKernelPathPolicy(undefined, options.kernelPathPolicy ?? undefined);
659
+ const finitenessPolicy = resolveRangeAwareSelectiveWideningConfig(runtimeCompute);
376
660
 
377
661
  if (!hasExecutionV1(manifestInference)) {
378
662
  throw new Error(`[ExecutionV1] manifest.inference.schema must be "${EXECUTION_V1_SCHEMA_ID}".`);
379
663
  }
380
664
 
381
- let execution = manifestInference.execution;
665
+ const manifestExecution = manifestInference.execution;
666
+ expandExecutionV1(manifestExecution);
667
+ let execution = applyExecutionPatch(manifestExecution, options.executionPatch ?? null);
382
668
  const mergedSession = mergeExecutionV1Session(
383
669
  manifestInference.session ?? {},
384
670
  runtimeSession ?? {}
385
671
  );
386
- const session = applyExecutionV1RuntimeComputeOverrides(
387
- mergedSession,
388
- runtimeSession,
389
- runtimeCompute
390
- );
672
+ const session = mergedSession;
391
673
 
392
674
  if (!session?.compute?.defaults?.activationDtype) {
393
675
  throw new Error('[ExecutionV1] session.compute.defaults.activationDtype is required.');
@@ -396,33 +678,31 @@ export function compileExecutionV1(options = {}) {
396
678
  const activationDtype = session.compute.defaults.activationDtype;
397
679
  const mathDtype = session.compute.defaults.mathDtype ?? null;
398
680
  const accumDtype = session.compute.defaults.accumDtype ?? null;
399
- const kvDtype = session?.kvcache?.kvDtype ?? activationDtype;
681
+ const kvDtype = requireSessionKVDtype(session);
400
682
  const layerTypes = manifestInference?.layerPattern?.layerTypes ?? null;
401
- const finitenessPolicy = resolveRangeAwareSelectiveWideningConfig(runtimeCompute);
402
683
 
403
- // Validate the original manifest graph (full digest checks).
404
- expandExecutionV1(execution);
684
+ if (execution !== manifestExecution) {
685
+ expandExecutionV1(execution);
686
+ }
405
687
 
406
688
  // -------------------------------------------------------------------------
407
689
  // Phase 2: Apply capability transforms to the execution graph
408
690
  // -------------------------------------------------------------------------
409
691
  let appliedTransformNames = [];
410
- let fallbackExecution = null;
411
- let fallbackTransformResult = null;
412
692
  let graphWasTransformed = false;
693
+ const graphContext = {
694
+ activationDtype,
695
+ mathDtype,
696
+ accumDtype,
697
+ kvDtype,
698
+ headDim,
699
+ modelId,
700
+ layerTypes,
701
+ retainQ4KMaterialization: session?.retainQ4KMaterialization === true,
702
+ ...summarizeExecutionGraphContext(execution),
703
+ };
413
704
 
414
705
  if (capabilities) {
415
- const graphContext = {
416
- activationDtype,
417
- mathDtype,
418
- accumDtype,
419
- kvDtype,
420
- headDim,
421
- modelId,
422
- layerTypes,
423
- retainQ4KMaterialization: session?.retainQ4KMaterialization === true,
424
- ...summarizeExecutionGraphContext(execution),
425
- };
426
706
  const resolved = resolveCapabilityTransforms(capabilities, platform, graphContext);
427
707
  const sourceScope = kernelPathPolicy.sourceScope ?? kernelPathPolicy.allowSources ?? [];
428
708
  const remapAllowed = kernelPathPolicy.mode === 'capability-aware'
@@ -472,30 +752,6 @@ export function compileExecutionV1(options = {}) {
472
752
  log.debug('ExecutionV1', `No capability transforms needed (${resolved.reason})`);
473
753
  }
474
754
 
475
- // Build explicit finiteness fallback only when the runtime opted into
476
- // alternate-plan recovery. The default policy is fail-fast.
477
- fallbackTransformResult = finitenessPolicy.onTrigger === 'fallback-plan'
478
- ? resolveFinitenessFallbackTransform(graphContext)
479
- : null;
480
- if (fallbackTransformResult) {
481
- const fallbackGraph = fallbackTransformResult.transform(execution, {
482
- capabilities,
483
- platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
484
- activationDtype,
485
- mathDtype,
486
- accumDtype,
487
- kvDtype,
488
- headDim,
489
- modelId,
490
- });
491
- if (fallbackGraph) {
492
- fallbackExecution = fallbackGraph;
493
- log.info(
494
- 'ExecutionV1',
495
- `Finiteness fallback transform resolved: ${fallbackTransformResult.name}`
496
- );
497
- }
498
- }
499
755
  }
500
756
 
501
757
  assertHybridLinearProjectionIsolation(execution, layerTypes, modelId);
@@ -564,46 +820,22 @@ export function compileExecutionV1(options = {}) {
564
820
  )
565
821
  : null;
566
822
 
567
- // Build fallback inline kernel path from the fallback execution graph
568
- let fallbackKernelPath = null;
569
- if (fallbackExecution && inlineKernelPathEnabled) {
570
- const fallbackSteps = expandV1ToResolvedSteps(fallbackExecution, { skipDigestValidation: true });
571
- const fallbackKvDtype = fallbackTransformResult?.fallbackKvDtype ?? 'f32';
572
- const fallbackSession = {
573
- ...session,
574
- compute: {
575
- ...session.compute,
576
- defaults: {
577
- ...session.compute.defaults,
578
- activationDtype: 'f32',
579
- mathDtype: 'f32',
580
- accumDtype: 'f32',
581
- outputDtype: 'f32',
582
- },
583
- },
584
- kvcache: {
585
- ...(session.kvcache ?? {}),
586
- kvDtype: fallbackKvDtype,
587
- },
588
- };
589
- fallbackKernelPath = buildInlineKernelPath(
590
- fallbackSteps,
591
- fallbackSession,
823
+ const fallbackKernelPath = inlineKernelPathEnabled
824
+ && finitenessPolicy.enabled
825
+ && finitenessPolicy.onTrigger === 'fallback-plan'
826
+ ? buildFinitenessFallbackKernelPath({
827
+ execution,
828
+ effectiveSession,
592
829
  modelId,
593
830
  numLayers,
594
- null
595
- );
596
- if (fallbackKernelPath) {
597
- fallbackKernelPath = {
598
- ...fallbackKernelPath,
599
- id: `${fallbackKernelPath.id}-finiteness-fallback`,
600
- name: 'Execution inline kernel path (finiteness fallback)',
601
- };
602
- log.info(
603
- 'ExecutionV1',
604
- `Finiteness fallback inline kernel path built (${fallbackKernelPath.id})`
605
- );
606
- }
831
+ headDim,
832
+ layerTypes,
833
+ capabilities,
834
+ platform,
835
+ })
836
+ : null;
837
+ if (kernelPath && fallbackKernelPath?.id) {
838
+ kernelPath.finitenessFallbackKernelPathId = fallbackKernelPath.id;
607
839
  }
608
840
 
609
841
  // Lane integrity: capture the pre-transform (manifest+profile-agreed) lane
@@ -716,14 +948,16 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
716
948
  runtimeSession: hasExplicitRuntimeOverrides
717
949
  ? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'session')
718
950
  : (runtimeConfig.inference?.session ?? null),
719
- runtimeCompute: hasExplicitRuntimeOverrides
720
- ? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'compute')
721
- : (runtimeConfig.inference?.compute ?? null),
951
+ runtimeCompute: runtimeConfig.inference?.compute ?? null,
722
952
  kernelPathPolicy: runtimeConfig.inference?.kernelPathPolicy ?? null,
953
+ executionPatch: runtimeConfig.inference?.executionPatch ?? null,
723
954
  });
724
955
 
725
956
  const runtimeInferencePatch = executionV1State.runtimeInferencePatch;
726
- const updatedInference = mergeRuntimeValues(runtimeConfig.inference ?? {}, runtimeInferencePatch);
957
+ const updatedInference = preserveRuntimeDecodeLoop(
958
+ mergeRuntimeValues(runtimeConfig.inference ?? {}, runtimeInferencePatch),
959
+ runtimeConfig
960
+ );
727
961
 
728
962
  return {
729
963
  runtimeConfig: {