doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -7,6 +7,7 @@ import {
7
7
  PER_LAYER_INPUT_PREFETCH_MODES,
8
8
  PER_LAYER_INPUT_GPU_UPLOAD_MODES,
9
9
  PER_LAYER_INPUT_HOT_CACHE_MODES,
10
+ PREFILL_CHUNK_SUBMIT_MODES,
10
11
  } from '../../../config/schema/execution-v1.schema.js';
11
12
 
12
13
  const UNSUPPORTED_RUNTIME_MODEL_TYPES = new Set(['mamba', 'rwkv']);
@@ -34,7 +35,7 @@ const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
34
35
  * @param {string} modelId - Model identifier for diagnostic messages.
35
36
  * @returns {boolean} True if the type is valid or null.
36
37
  */
37
- export function validateChatTemplateType(type, modelId) {
38
+ function validateChatTemplateType(type, modelId) {
38
39
  if (type === null || type === undefined) return true;
39
40
  if (KNOWN_CHAT_TEMPLATE_TYPES.has(type)) return true;
40
41
  throw new Error(
@@ -196,6 +197,58 @@ function resolvePerLayerInputsSession(inferenceConfig, modelId) {
196
197
  };
197
198
  }
198
199
 
200
+ function resolveSessionSettings(inferenceConfig, modelId) {
201
+ // All four fields are optional on the manifest today for backwards compatibility;
202
+ // when absent we fall through to the runtime config (getRuntimeConfig()). When
203
+ // present on the manifest, manifest wins unless an explicit runtime profile overrides.
204
+ // The merge layer in src/config/merge.js already applies that precedence — this
205
+ // resolver just validates the manifest-supplied values and normalizes them.
206
+ const session = inferenceConfig?.session;
207
+ const submit = session?.prefillChunkSubmitMode;
208
+ if (submit !== undefined && submit !== null && !PREFILL_CHUNK_SUBMIT_MODES.includes(submit)) {
209
+ throw new Error(
210
+ `Manifest "${modelId}" has invalid inference.session.prefillChunkSubmitMode ` +
211
+ `"${String(submit)}"; expected one of ${PREFILL_CHUNK_SUBMIT_MODES.join(', ')}.`
212
+ );
213
+ }
214
+ const flash = session?.useFlashPrefillAttention;
215
+ if (flash !== undefined && flash !== null && typeof flash !== 'boolean') {
216
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useFlashPrefillAttention "${String(flash)}"; expected boolean.`);
217
+ }
218
+ const wide = session?.useWideTileQ4KPrefill;
219
+ if (wide !== undefined && wide !== null && typeof wide !== 'boolean') {
220
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KPrefill "${String(wide)}"; expected boolean.`);
221
+ }
222
+ const retain = session?.retainQ4KMaterialization;
223
+ if (retain !== undefined && retain !== null && typeof retain !== 'boolean') {
224
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.retainQ4KMaterialization "${String(retain)}"; expected boolean.`);
225
+ }
226
+ return {
227
+ prefillChunkSubmitMode: submit ?? null,
228
+ useFlashPrefillAttention: flash ?? null,
229
+ useWideTileQ4KPrefill: wide ?? null,
230
+ retainQ4KMaterialization: retain ?? null,
231
+ };
232
+ }
233
+
234
+ function resolveLargeWeightsConfig(inferenceConfig, modelId) {
235
+ const lw = inferenceConfig?.largeWeights;
236
+ if (lw === undefined || lw === null) return { gpuResidentOverrides: null };
237
+ if (typeof lw !== 'object' || Array.isArray(lw)) {
238
+ throw new Error(`Manifest "${modelId}" has invalid inference.largeWeights (must be object).`);
239
+ }
240
+ const overrides = lw.gpuResidentOverrides;
241
+ if (overrides !== null && overrides !== undefined) {
242
+ if (!Array.isArray(overrides) || !overrides.every((v) => typeof v === 'string' && v.length > 0)) {
243
+ throw new Error(
244
+ `Manifest "${modelId}" has invalid inference.largeWeights.gpuResidentOverrides; ` +
245
+ `expected array of non-empty strings or null.`
246
+ );
247
+ }
248
+ }
249
+ return { gpuResidentOverrides: overrides ?? null };
250
+ }
251
+
199
252
  // =============================================================================
200
253
  // Model Detection Functions
201
254
  // =============================================================================
@@ -541,7 +594,7 @@ export function resolveLayerIntermediateSize(config, layerIdx) {
541
594
  // =============================================================================
542
595
 
543
596
 
544
- export function hasManifestInference(manifest) {
597
+ function hasManifestInference(manifest) {
545
598
  return 'inference' in manifest && manifest.inference != null;
546
599
  }
547
600
 
@@ -1120,7 +1173,7 @@ function parseLinearNormMode(value, sharedFlag = null, modelId = 'unknown') {
1120
1173
  }
1121
1174
 
1122
1175
 
1123
- export function toParsedConfigFromMerged(merged, manifest) {
1176
+ function toParsedConfigFromMerged(merged, manifest) {
1124
1177
  const mergedConfig = merged?.config ?? manifest.config ?? {};
1125
1178
  const rawConfig = mergedConfig.text_config ?? mergedConfig;
1126
1179
  const config = {
@@ -1397,6 +1450,49 @@ export function toParsedConfigFromMerged(merged, manifest) {
1397
1450
  return Math.trunc(num);
1398
1451
  };
1399
1452
 
1453
+ const resolveTensorOutputRows = (tensorInfo, hiddenSize) => {
1454
+ const shape = Array.isArray(tensorInfo?.shape) ? tensorInfo.shape : null;
1455
+ if (!shape || shape.length < 2) return null;
1456
+ const dim0 = Number(shape[0]);
1457
+ const dim1 = Number(shape[1]);
1458
+ if (!Number.isFinite(dim0) || !Number.isFinite(dim1)) return null;
1459
+ if (dim1 === hiddenSize) return Math.trunc(dim0);
1460
+ if (dim0 === hiddenSize) return Math.trunc(dim1);
1461
+ return null;
1462
+ };
1463
+
1464
+ const deriveGlobalKVHeadsFromManifest = () => {
1465
+ if (!Array.isArray(layerTypes) || archGlobalHeadDim == null) {
1466
+ return null;
1467
+ }
1468
+ const tensors = manifest?.tensors && typeof manifest.tensors === 'object' ? manifest.tensors : null;
1469
+ if (!tensors) return null;
1470
+ const globalLayerIdx = layerTypes.findIndex((layerType) => {
1471
+ const normalized = typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
1472
+ return normalized && normalized !== 'sliding_attention' && normalized !== 'local_attention'
1473
+ && normalized !== 'local' && normalized !== 'sliding';
1474
+ });
1475
+ if (globalLayerIdx < 0) return null;
1476
+ const layerMarker = `.layers.${globalLayerIdx}.`;
1477
+ for (const [tensorName, tensorInfo] of Object.entries(tensors)) {
1478
+ if (!tensorName.includes(layerMarker) || !tensorName.includes('.self_attn.k_proj.weight')) {
1479
+ continue;
1480
+ }
1481
+ const rows = resolveTensorOutputRows(tensorInfo, arch.hiddenSize);
1482
+ if (rows != null && rows % archGlobalHeadDim === 0) {
1483
+ return rows / archGlobalHeadDim;
1484
+ }
1485
+ }
1486
+ return null;
1487
+ };
1488
+
1489
+ const archNumGlobalKVHeads = parsePositiveInt(
1490
+ arch.numGlobalKeyValueHeads
1491
+ ?? arch.numGlobalKVHeads
1492
+ ?? config.num_global_key_value_heads
1493
+ ?? config.num_global_kv_heads
1494
+ ) ?? deriveGlobalKVHeadsFromManifest();
1495
+
1400
1496
  const linearNumKeyHeads = parsePositiveInt(arch.linearNumKeyHeads ?? config.linear_num_key_heads);
1401
1497
  const linearNumValueHeads = parsePositiveInt(arch.linearNumValueHeads ?? config.linear_num_value_heads);
1402
1498
  const linearKeyHeadDim = parsePositiveInt(arch.linearKeyHeadDim ?? config.linear_key_head_dim);
@@ -1414,6 +1510,8 @@ export function toParsedConfigFromMerged(merged, manifest) {
1414
1510
  ? 'replay_prefill'
1415
1511
  : 'incremental';
1416
1512
  const perLayerInputsSession = resolvePerLayerInputsSession(inf, merged.modelId);
1513
+ const sessionSettings = resolveSessionSettings(inf, merged.modelId);
1514
+ const largeWeightsConfig = resolveLargeWeightsConfig(inf, merged.modelId);
1417
1515
 
1418
1516
  return {
1419
1517
  numLayers: arch.numLayers,
@@ -1423,6 +1521,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
1423
1521
  maxIntermediateSize,
1424
1522
  numHeads: archNumHeads,
1425
1523
  numKVHeads: archNumKVHeads,
1524
+ numGlobalKVHeads: archNumGlobalKVHeads,
1426
1525
  headDim: archHeadDim,
1427
1526
  globalHeadDim: archGlobalHeadDim,
1428
1527
  vocabSize: arch.vocabSize,
@@ -1482,6 +1581,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
1482
1581
  queryKeyNorm: inf.attention.queryKeyNorm,
1483
1582
  valueNorm: inf.attention.valueNorm,
1484
1583
  attentionOutputGate: inf.attention.attentionOutputGate === true,
1584
+ outputGateType: inf.attention.outputGateType ?? null,
1485
1585
  queryPreAttnScalar,
1486
1586
  layerPipeline: inf.pipeline ?? null,
1487
1587
  chatTemplateType,
@@ -1489,6 +1589,8 @@ export function toParsedConfigFromMerged(merged, manifest) {
1489
1589
  chatTemplateThinking,
1490
1590
  decodeStrategy,
1491
1591
  perLayerInputsSession,
1592
+ sessionSettings,
1593
+ largeWeightsConfig,
1492
1594
  kernelPath: null,
1493
1595
  visionConfig: resolveVisionConfig(config, manifest),
1494
1596
  audioConfig: resolveAudioConfig(config, manifest),
@@ -2,30 +2,9 @@
2
2
 
3
3
  import { readBuffer } from '../../../../memory/buffer-pool.js';
4
4
  import { isBufferStatsEnabled } from './config.js';
5
+ import { f16ToF32 } from '../../../../loader/dtype-utils.js';
5
6
 
6
- // ============================================================================
7
- // Math Helpers
8
- // ============================================================================
9
-
10
-
11
- export function f16ToF32(h) {
12
- const sign = (h >> 15) & 0x1;
13
- const exp = (h >> 10) & 0x1f;
14
- const mant = h & 0x3ff;
15
-
16
- if (exp === 0) {
17
- if (mant === 0) return sign ? -0 : 0;
18
- const f = mant / 1024 * Math.pow(2, -14);
19
- return sign ? -f : f;
20
- }
21
- if (exp === 31) {
22
- return mant ? NaN : (sign ? -Infinity : Infinity);
23
- }
24
-
25
- const f = (1 + mant / 1024) * Math.pow(2, exp - 15);
26
- return sign ? -f : f;
27
- }
28
-
7
+ export { f16ToF32 };
29
8
 
30
9
  export function decodeReadback(buffer, dtype) {
31
10
  if (dtype === 'f32') {
@@ -5,7 +5,7 @@
5
5
  // Provides tolerance lookup for first-divergence detection.
6
6
  // Models propagation bounds to prevent unbounded drift amplification.
7
7
 
8
- import driftPoliciesConfig from '../../../config/runtime/diagnostics/drift-policies.json' with { type: 'json' };
8
+ import driftPoliciesConfig from '../../../config/diagnostics/drift-policies.json' with { type: 'json' };
9
9
 
10
10
  // ============================================================================
11
11
  // Policy Lookup
@@ -22,6 +22,7 @@ export interface EmbedConfig {
22
22
  activationDtype?: 'f16' | 'f32';
23
23
  embeddingDtype?: 'f16' | 'f32';
24
24
  executionPolicies?: ExecutionV1PoliciesSchema | null;
25
+ probeStage?: string;
25
26
  inputHiddenSize?: number;
26
27
  hiddenOffset?: number;
27
28
  }
@@ -35,37 +36,32 @@ export interface ValidationResult {
35
36
  infCount: number;
36
37
  }
37
38
 
38
- /**
39
- * Record scale operation (batched, no submit)
40
- */
41
- export function recordScale(
42
- recorder: CommandRecorder,
43
- inputBuffer: GPUBuffer,
44
- scale: number,
45
- count: number,
46
- useF16?: boolean
47
- ): GPUBuffer;
48
-
49
- /**
50
- * Scale GPU buffer (standalone, with submit)
51
- * @deprecated Use recordScale with CommandRecorder instead
52
- */
53
- export function scaleGPUBuffer(
54
- inputBuffer: GPUBuffer,
55
- scale: number,
56
- count: number,
57
- useF16?: boolean
58
- ): Promise<GPUBuffer>;
59
-
60
39
  export function embed(
61
40
  tokenIds: number[] | Uint32Array | GPUBuffer,
62
41
  embedBuffer: GPUBuffer | Float32Array | CpuWeightBuffer,
63
42
  config: EmbedConfig
64
43
  ): Promise<Tensor>;
65
44
 
66
- export function validateEmbedding(
67
- buffer: GPUBuffer,
68
- label: string,
69
- numTokens: number,
45
+ /**
46
+ * True when `value` looks like a range-backed CPU embedding source
47
+ * (has a typed byte range that the loader can pull from).
48
+ */
49
+ export declare function isRangeBackedCpuEmbeddingSource(value: unknown): boolean;
50
+
51
+ /**
52
+ * Normalize a range bytes payload (typed array or buffer view) into a
53
+ * Uint8Array, throwing a labeled error when the shape is not usable.
54
+ */
55
+ export declare function normalizeRangeBytes(value: unknown, label: string): Uint8Array;
56
+
57
+ /**
58
+ * Decode a range-backed chunk into `output` at `dstOffset` using the
59
+ * declared `sourceDtype` and per-row `hiddenSize`.
60
+ */
61
+ export declare function decodeRangeChunkIntoOutput(
62
+ bytes: Uint8Array,
63
+ sourceDtype: string,
64
+ output: Float32Array,
65
+ dstOffset: number,
70
66
  hiddenSize: number
71
- ): Promise<ValidationResult | null>;
67
+ ): void;
@@ -84,8 +84,10 @@ export async function embed(tokenIds, embedBuffer, config) {
84
84
  activationDtype,
85
85
  embeddingDtype,
86
86
  operatorDiagnostics,
87
+ probeStage = 'embed_out',
87
88
  inputHiddenSize = hiddenSize,
88
89
  hiddenOffset = 0,
90
+ stats = null,
89
91
  } = config;
90
92
  const device = getDevice();
91
93
  const tokenBufferInput = isGpuBufferInstance(tokenIds);
@@ -274,7 +276,7 @@ export async function embed(tokenIds, embedBuffer, config) {
274
276
  } else {
275
277
  releaseBuffer(f32Buffer);
276
278
  }
277
- await runProbes('embed_out', f16Tensor.buffer, {
279
+ await runProbes(probeStage, f16Tensor.buffer, {
278
280
  numTokens,
279
281
  hiddenSize,
280
282
  probes: config.debugProbes,
@@ -290,7 +292,11 @@ export async function embed(tokenIds, embedBuffer, config) {
290
292
  ? preAllocatedOutput
291
293
  : acquireBuffer(outputBytes, undefined, 'embed_cpu_f32_out');
292
294
  device.queue.writeBuffer(outputBuffer, 0, output);
293
- await runProbes('embed_out', outputBuffer, {
295
+ if (stats) {
296
+ stats.pleWriteBufferCount = (stats.pleWriteBufferCount ?? 0) + 1;
297
+ stats.pleWriteBufferBytes = (stats.pleWriteBufferBytes ?? 0) + outputBytes;
298
+ }
299
+ await runProbes(probeStage, outputBuffer, {
294
300
  numTokens,
295
301
  hiddenSize,
296
302
  probes: config.debugProbes,
@@ -360,7 +366,7 @@ export async function embed(tokenIds, embedBuffer, config) {
360
366
  }
361
367
 
362
368
  if (!scaleEmbeddings) {
363
- await runProbes('embed_out', gatherOutput.buffer, {
369
+ await runProbes(probeStage, gatherOutput.buffer, {
364
370
  numTokens,
365
371
  hiddenSize,
366
372
  probes: config.debugProbes,
@@ -431,7 +437,7 @@ export async function embed(tokenIds, embedBuffer, config) {
431
437
  throw new Error('[Embed] Scaled embedding contains NaN/Inf');
432
438
  }
433
439
  }
434
- await runProbes('embed_out', scaledBuffer, {
440
+ await runProbes(probeStage, scaledBuffer, {
435
441
  numTokens,
436
442
  hiddenSize,
437
443
  probes: config.debugProbes,
@@ -442,42 +448,3 @@ export async function embed(tokenIds, embedBuffer, config) {
442
448
 
443
449
  return createTensor(scaledBuffer, dtype, [numTokens, hiddenSize], 'embed_output');
444
450
  }
445
-
446
-
447
- export async function validateEmbedding(buffer, label, numTokens, hiddenSize) {
448
- const device = getDevice();
449
- if (!device) return null;
450
-
451
- const sampleSize = Math.min(1024 * 4, buffer.size);
452
- const sample = await readBuffer(buffer, sampleSize);
453
- const f32 = new Float32Array(sample);
454
-
455
- let min = Infinity;
456
- let max = -Infinity;
457
- let sum = 0;
458
- let zeros = 0;
459
- let nanCount = 0;
460
- let infCount = 0;
461
-
462
- for (let i = 0; i < f32.length; i++) {
463
- const v = f32[i];
464
- if (Number.isNaN(v)) {
465
- nanCount++;
466
- } else if (!Number.isFinite(v)) {
467
- infCount++;
468
- } else {
469
- if (v === 0) zeros++;
470
- if (v < min) min = v;
471
- if (v > max) max = v;
472
- sum += v;
473
- }
474
- }
475
-
476
- const mean = sum / f32.length;
477
-
478
- trace.embed(`${label}: tokens=${numTokens}, hidden=${hiddenSize}`);
479
- trace.embed(`${label}: min=${min.toFixed(4)}, max=${max.toFixed(4)}, mean=${mean.toFixed(4)}`);
480
- trace.embed(`${label}: zeros=${zeros}/${f32.length}, NaN=${nanCount}, Inf=${infCount}`);
481
-
482
- return { min, max, mean, zeros, nanCount, infCount };
483
- }
@@ -2,9 +2,6 @@ import type { KernelPathSchema } from '../../../config/schema/index.js';
2
2
  import type { RuntimeConfigSchema } from '../../../config/schema/index.js';
3
3
  import type { KernelPathSource } from '../../../config/kernel-path-loader.js';
4
4
 
5
- export const PRIMARY_EXECUTION_PLAN_ID: 'primary';
6
- export const FINITENESS_FALLBACK_EXECUTION_PLAN_ID: 'finiteness_fallback';
7
-
8
5
  export interface StaticExecutionPlan {
9
6
  id: 'primary' | 'finiteness_fallback';
10
7
  source: string;
@@ -127,6 +124,11 @@ export declare function resolveMaxBatchDecodeTokens(config: {
127
124
  hasLinearAttentionLayers?: boolean;
128
125
  }): number | null;
129
126
 
127
+ export declare function resolvePrefillRecorderChunkLayers(config: {
128
+ hasGpuSplitPerLayerInputs?: boolean;
129
+ numTokens: number;
130
+ }): number;
131
+
130
132
  export declare function isDecodeRecorderEnabled(config: {
131
133
  hasDevice: boolean;
132
134
  debug: boolean;
@@ -6,8 +6,8 @@ import {
6
6
  } from './finiteness-policy.js';
7
7
  import { log } from '../../../debug/index.js';
8
8
 
9
- export const PRIMARY_EXECUTION_PLAN_ID = 'primary';
10
- export const FINITENESS_FALLBACK_EXECUTION_PLAN_ID = 'finiteness_fallback';
9
+ const PRIMARY_EXECUTION_PLAN_ID = 'primary';
10
+ const FINITENESS_FALLBACK_EXECUTION_PLAN_ID = 'finiteness_fallback';
11
11
  const DEFAULT_MAX_TOKENS = 256;
12
12
 
13
13
  function assertOptionalBoolean(value, label) {
@@ -444,6 +444,19 @@ export function resolveMaxBatchDecodeTokens(config) {
444
444
  return value;
445
445
  }
446
446
 
447
+ export function resolvePrefillRecorderChunkLayers(config) {
448
+ const value = selectRuleValue('inference', 'execution', 'prefillRecorderChunkLayers', {
449
+ hasGpuSplitPerLayerInputs: config.hasGpuSplitPerLayerInputs === true,
450
+ numTokens: config.numTokens,
451
+ });
452
+ if (!Number.isInteger(value) || value < 1) {
453
+ throw new Error(
454
+ `[ExecutionPlan] prefillRecorderChunkLayers must be a positive integer; got ${JSON.stringify(value)}.`
455
+ );
456
+ }
457
+ return value;
458
+ }
459
+
447
460
  export function isDecodeRecorderEnabled(config) {
448
461
  return selectRuleValue('inference', 'execution', 'decodeRecorderEnabled', {
449
462
  hasDevice: config.hasDevice === true,
@@ -8,10 +8,6 @@ export declare function requireSessionActivationDtype(
8
8
  label?: string
9
9
  ): 'f16' | 'f32';
10
10
 
11
- export declare function resolveFinitenessFallbackKernelPathId(
12
- kernelPathId: string | null | undefined
13
- ): string | null;
14
-
15
11
  export declare function buildInlineKernelPath(
16
12
  steps: readonly Record<string, unknown>[],
17
13
  session: Record<string, unknown> | null,
@@ -168,17 +168,6 @@ export function assertKernelPathSessionCompatibility(path, session) {
168
168
  }
169
169
  }
170
170
 
171
- export function resolveFinitenessFallbackKernelPathId(kernelPathId) {
172
- return kernelPathId
173
- ? selectRuleValue(
174
- 'inference',
175
- 'kernelPath',
176
- 'finitenessFallback',
177
- { kernelPathId }
178
- )
179
- : null;
180
- }
181
-
182
171
  export function buildInlineKernelPath(
183
172
  steps,
184
173
  session,
@@ -6,6 +6,23 @@ import type {
6
6
  } from '../../../config/schema/execution-v1.schema.js';
7
7
  import type { KernelPathSchema } from '../../../config/schema/index.js';
8
8
 
9
+ export interface ExecutionV1LaneIntegrity {
10
+ declared: {
11
+ activationDtype: string | null;
12
+ mathDtype: string | null;
13
+ accumDtype: string | null;
14
+ kvDtype: string | null;
15
+ };
16
+ executed: {
17
+ activationDtype: string | null;
18
+ mathDtype: string | null;
19
+ accumDtype: string | null;
20
+ kvDtype: string | null;
21
+ };
22
+ status: 'matches' | 'transformed';
23
+ transforms: string[];
24
+ }
25
+
9
26
  export interface ExecutionV1CompiledState {
10
27
  session: ExecutionV1SessionSchema;
11
28
  policies: ExecutionV1PoliciesSchema;
@@ -16,6 +33,7 @@ export interface ExecutionV1CompiledState {
16
33
  };
17
34
  runtimeInferencePatch: Record<string, unknown>;
18
35
  appliedTransforms?: string[];
36
+ laneIntegrity?: ExecutionV1LaneIntegrity;
19
37
  fallbackKernelPath?: KernelPathSchema | null;
20
38
  }
21
39