doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -5,6 +5,7 @@ import type { CommandRecorder } from '../../../gpu/command-recorder.js';
5
5
  import type { LinearNormMode } from '../../../config/schema/index.js';
6
6
  import type { ProbeConfigSchema } from '../../../config/schema/index.js';
7
7
  import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
8
+ import type { KernelPathSchema } from '../../../config/schema/kernel-path.schema.js';
8
9
 
9
10
  export interface LinearLayerRuntimeState {
10
11
  layerIdx: number;
@@ -61,7 +62,7 @@ export interface RunLinearAttentionLayerOptions {
61
62
  };
62
63
  currentSeqLen: number;
63
64
  activationDtype?: 'f16' | 'f32';
64
- kernelPath?: string | null;
65
+ kernelPath?: KernelPathSchema | null;
65
66
  executionPolicies?: ExecutionV1PoliciesSchema | null;
66
67
  linearRuntime?: LinearAttentionRuntime | null;
67
68
  getWeightBuffer: (
@@ -0,0 +1,21 @@
1
+ export interface WeightLoadErrorDetails {
2
+ [key: string]: unknown;
3
+ }
4
+
5
+ export type AnnotatableError = Error & {
6
+ details?: Record<string, unknown>;
7
+ code?: string;
8
+ };
9
+
10
+ export declare function annotateWeightLoadError<T>(
11
+ error: T,
12
+ details?: WeightLoadErrorDetails
13
+ ): T;
14
+
15
+ export declare function rewriteWeightLoadError(
16
+ error: unknown,
17
+ context?: {
18
+ modelId?: string | null;
19
+ deviceLossInfo?: Record<string, unknown> | null;
20
+ }
21
+ ): unknown;
@@ -0,0 +1,59 @@
1
+ import { getLastDeviceLossInfo } from '../../../gpu/device.js';
2
+ import { createDopplerError, ERROR_CODES } from '../../../errors/doppler-error.js';
3
+
4
+ const DEVICE_LIFECYCLE_PATTERNS = [
5
+ /device not initialized/i,
6
+ /gpu device not initialized/i,
7
+ /no gpu device available/i,
8
+ ];
9
+
10
+ function matchesDeviceLifecycleFailure(message) {
11
+ return DEVICE_LIFECYCLE_PATTERNS.some((pattern) => pattern.test(message));
12
+ }
13
+
14
+ export function annotateWeightLoadError(error, details = {}) {
15
+ if (!error || typeof error !== 'object') {
16
+ return error;
17
+ }
18
+ const existingDetails = error.details && typeof error.details === 'object'
19
+ ? error.details
20
+ : {};
21
+ error.details = {
22
+ ...existingDetails,
23
+ weightLoadFailure: {
24
+ ...(existingDetails.weightLoadFailure && typeof existingDetails.weightLoadFailure === 'object'
25
+ ? existingDetails.weightLoadFailure
26
+ : {}),
27
+ ...details,
28
+ },
29
+ };
30
+ return error;
31
+ }
32
+
33
+ export function rewriteWeightLoadError(error, context = {}) {
34
+ const message = error?.message || String(error);
35
+ if (!matchesDeviceLifecycleFailure(message)) {
36
+ return error;
37
+ }
38
+
39
+ const modelId = typeof context.modelId === 'string' && context.modelId.trim()
40
+ ? context.modelId.trim()
41
+ : 'unknown';
42
+ const lastDeviceLoss = context.deviceLossInfo ?? getLastDeviceLossInfo();
43
+ const wrapped = createDopplerError(
44
+ ERROR_CODES.GPU_DEVICE_FAILED,
45
+ `Weight load lost access to the WebGPU device for model "${modelId}". ` +
46
+ 'This is a device lifecycle failure during loadWeights, not proof of VRAM-capacity exhaustion. ' +
47
+ 'Check device initialization/loss logs and the first failing buffer allocation or upload.'
48
+ );
49
+ wrapped.cause = error;
50
+ wrapped.details = {
51
+ ...(error?.details && typeof error.details === 'object' ? error.details : {}),
52
+ loadPhase: 'loadWeights',
53
+ modelId,
54
+ lifecycleFailure: 'device_unavailable_during_weight_load',
55
+ originalMessage: message,
56
+ lastDeviceLoss: lastDeviceLoss ? { ...lastDeviceLoss } : null,
57
+ };
58
+ return wrapped;
59
+ }
@@ -1,5 +1,6 @@
1
+ import { f16ToF32 } from '../../../../loader/dtype-utils.js';
1
2
 
2
-
3
+ export { f16ToF32 };
3
4
 
4
5
  // Caller should pass eps from model config; inline default for safety
5
6
  export function rmsNormCPU(
@@ -25,25 +26,6 @@ export function rmsNormCPU(
25
26
  }
26
27
 
27
28
 
28
- export function f16ToF32(h) {
29
- const sign = (h >> 15) & 0x1;
30
- const exp = (h >> 10) & 0x1f;
31
- const mant = h & 0x3ff;
32
-
33
- if (exp === 0) {
34
- if (mant === 0) return sign ? -0 : 0;
35
- const f = mant / 1024 * Math.pow(2, -14);
36
- return sign ? -f : f;
37
- }
38
- if (exp === 31) {
39
- return mant ? NaN : (sign ? -Infinity : Infinity);
40
- }
41
-
42
- const f = (1 + mant / 1024) * Math.pow(2, exp - 15);
43
- return sign ? -f : f;
44
- }
45
-
46
-
47
29
  export function f16BufferToF32(data) {
48
30
  const u16 = new Uint16Array(data);
49
31
  const out = new Float32Array(u16.length);
@@ -554,6 +554,7 @@ export async function computeLogitsGPU(
554
554
  await runProbes('pre_final_norm', inputBuffer, {
555
555
  numTokens,
556
556
  hiddenSize,
557
+ probes: config.debugProbes ?? null,
557
558
  operatorDiagnostics,
558
559
  dtype: inputDtype,
559
560
  });
@@ -612,6 +613,7 @@ export async function computeLogitsGPU(
612
613
  await runProbes('final_norm', finalNormTensor.buffer, {
613
614
  numTokens,
614
615
  hiddenSize,
616
+ probes: config.debugProbes ?? null,
615
617
  operatorDiagnostics,
616
618
  dtype: finalNormTensor.dtype,
617
619
  });
@@ -23,7 +23,7 @@ export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } fro
23
23
  export { computeLogitsGPU, recordLogitsGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
24
24
 
25
25
  // Re-export utilities
26
- export { extractLastPositionLogits, finalizeLogits } from './utils.js';
26
+ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from './utils.js';
27
27
 
28
28
  export interface ComputeLogitsOptions {
29
29
  lastPositionOnly?: boolean;
@@ -48,12 +48,13 @@ export function activateKernelPathState(
48
48
  kernelPathState: KernelPathResolutionResult | null | undefined
49
49
  ): void;
50
50
 
51
- export function resolveAndActivateKernelPath(
52
- options: KernelPathResolutionOptions
53
- ): KernelPathResolutionResult;
54
-
55
51
  export function initTokenizerFromManifest(
56
52
  manifest: Manifest,
57
53
  baseUrl?: string | null,
58
54
  storageContext?: PipelineStorageContext | null
59
55
  ): Promise<Tokenizer>;
56
+
57
+ export function assertManifestComputeLaneBinding(options: {
58
+ manifest: Manifest;
59
+ runtimeConfig: RuntimeConfigSchema;
60
+ }): void;
@@ -208,7 +208,7 @@ export function applyModelBatchingRuntimeDefaults(runtimeConfig, manifest, model
208
208
  runtimeConfig,
209
209
  manifest,
210
210
  runtimeOverrides,
211
- arguments.length > 3
211
+ runtimeOverrides != null
212
212
  );
213
213
  if (!patch) {
214
214
  return runtimeConfig;
@@ -266,10 +266,29 @@ export function applyModelBatchingRuntimeDefaults(runtimeConfig, manifest, model
266
266
  dl.readbackMode = resolvedReadbackMode;
267
267
  patch.batching.readbackMode = resolvedReadbackMode;
268
268
 
269
+ // Promote the resolved session and manifest largeWeights into runtimeConfig
270
+ // so existing pipeline reads of getRuntimeConfig().inference.session.* and
271
+ // .largeWeights.* pick up per-model values. buildResolvedDecodeLoopRuntimePatch()
272
+ // already applied runtime-over-manifest session precedence field-by-field.
273
+ // For largeWeights, an explicit runtime array wins, including [] to disable
274
+ // manifest gpu-resident overrides on constrained verification hosts.
275
+ const manifestInf = manifest?.inference ?? {};
276
+ const manifestLargeWeights = manifestInf.largeWeights;
277
+ const runtimeLargeWeightsOverrides =
278
+ runtimeConfig?.inference?.largeWeights?.gpuResidentOverrides;
279
+ const hasRuntimeLargeWeightsOverride = Array.isArray(runtimeLargeWeightsOverrides);
280
+ const largeWeightsPatch = (!hasRuntimeLargeWeightsOverride
281
+ && manifestLargeWeights && typeof manifestLargeWeights === 'object'
282
+ && Array.isArray(manifestLargeWeights.gpuResidentOverrides)
283
+ && manifestLargeWeights.gpuResidentOverrides.length > 0)
284
+ ? { gpuResidentOverrides: manifestLargeWeights.gpuResidentOverrides }
285
+ : null;
286
+
269
287
  const nextRuntimeConfig = mergeRuntimeValues(runtimeConfig, {
270
288
  inference: {
271
289
  session: patch.session,
272
290
  batching: patch.batching,
291
+ ...(largeWeightsPatch ? { largeWeights: largeWeightsPatch } : {}),
273
292
  },
274
293
  });
275
294
  log.info(
@@ -425,8 +444,6 @@ function summarizeUnsupportedKernelUsages(usages) {
425
444
  }
426
445
 
427
446
  function assertKernelPathFeatureCompatibility(
428
- configuredKernelPathRef,
429
- effectiveKernelPathRef,
430
447
  resolvedKernelPath,
431
448
  kernelPathSource,
432
449
  capabilities,
@@ -439,16 +456,13 @@ function assertKernelPathFeatureCompatibility(
439
456
  const policyAllowsSource = kernelPathPolicy.mode === 'capability-aware'
440
457
  && sourceScope.includes(kernelPathSource);
441
458
  const remapRequested = policyAllowsSource && kernelPathPolicy.onIncompatible === 'remap';
442
- const remapApplied = typeof configuredKernelPathRef === 'string'
443
- && typeof effectiveKernelPathRef === 'string'
444
- && configuredKernelPathRef !== effectiveKernelPathRef;
445
459
  const summary = summarizeUnsupportedKernelUsages(unsupportedUsages);
446
460
 
447
- if (remapRequested && !remapApplied) {
461
+ if (remapRequested) {
448
462
  throw new Error(
449
463
  `KernelPath "${resolvedKernelPath?.id ?? 'unknown'}" requires unsupported GPU features (${summary}) ` +
450
- `and no explicit auto-select remap matched for source "${kernelPathSource}". ` +
451
- 'Add a kernel-path remap rule or choose a compatible kernelPath.'
464
+ `for source "${kernelPathSource}". String registry remaps are removed; use execution graph ` +
465
+ 'capability transforms or choose a compatible inline kernelPath.'
452
466
  );
453
467
  }
454
468
 
@@ -545,6 +559,57 @@ function describeExplicitRuntimeDtypeMismatch(contract, explicitRuntime) {
545
559
  return mismatches;
546
560
  }
547
561
 
562
+ /**
563
+ * Manifest-side lane binding: the manifest variant tag claims a compute lane
564
+ * via quantizationInfo.compute. Once all runtime resolution phases have run,
565
+ * the resolved session must dispatch that same lane. If they disagree, the
566
+ * operator picked the wrong manifest variant or the wrong runtime profile.
567
+ *
568
+ * The compute lane covers activation/math/accum dtypes only. KV cache dtype
569
+ * is an orthogonal axis — Gemma-family layouts pair f32 compute with f16 KV
570
+ * for memory savings, and that combination is supported by design — so kv
571
+ * dtype is intentionally excluded from the comparison.
572
+ *
573
+ * Throws on mismatch. Returns silently when the manifest does not declare
574
+ * a compute lane (legacy / vision-only manifests) or when no resolved values
575
+ * are available to compare against.
576
+ *
577
+ * @param {Object} options
578
+ * @param {Object} options.manifest
579
+ * @param {Object} options.runtimeConfig
580
+ * @returns {void}
581
+ */
582
+ export function assertManifestComputeLaneBinding({ manifest, runtimeConfig }) {
583
+ const declared = normalizeKernelDtype(manifest?.quantizationInfo?.compute);
584
+ if (!declared) return;
585
+
586
+ const session = runtimeConfig?.inference?.session ?? {};
587
+ const computeDefaults = session.compute?.defaults ?? {};
588
+ const candidates = [
589
+ ['session.compute.defaults.activationDtype', computeDefaults.activationDtype],
590
+ ['session.compute.defaults.mathDtype', computeDefaults.mathDtype],
591
+ ['session.compute.defaults.accumDtype', computeDefaults.accumDtype],
592
+ ];
593
+
594
+ const mismatches = [];
595
+ for (const [field, value] of candidates) {
596
+ const resolved = normalizeKernelDtype(value);
597
+ if (resolved && resolved !== declared) {
598
+ mismatches.push(`${field}=${value}`);
599
+ }
600
+ }
601
+ if (mismatches.length === 0) return;
602
+
603
+ throw new Error(
604
+ `Manifest "${manifest?.modelId ?? 'unknown'}" declares ` +
605
+ `quantizationInfo.compute=${declared} but runtime resolved ` +
606
+ `[${mismatches.join('; ')}]. ` +
607
+ 'The manifest variant tag is the lane identity — load the manifest variant ' +
608
+ 'whose compute lane matches the runtime profile, or pick a runtime profile ' +
609
+ 'whose dtype defaults match this manifest.'
610
+ );
611
+ }
612
+
548
613
  function assertManifestKernelPathDtypeCompatibility(manifest, resolvedKernelPath, kernelPathSource) {
549
614
  if (!resolvedKernelPath) return;
550
615
  if (kernelPathSource === 'config') return;
@@ -748,8 +813,6 @@ export function resolveKernelPathState(options) {
748
813
 
749
814
  if (capabilities) {
750
815
  assertKernelPathFeatureCompatibility(
751
- configuredKernelPathRef,
752
- configuredKernelPathRef,
753
816
  resolvedKernelPath,
754
817
  kernelPathSource,
755
818
  capabilities,
@@ -790,12 +853,6 @@ export function activateKernelPathState(kernelPathState) {
790
853
  );
791
854
  }
792
855
 
793
- export function resolveAndActivateKernelPath(options) {
794
- const state = resolveKernelPathState(options);
795
- activateKernelPathState(state);
796
- return state;
797
- }
798
-
799
856
  export async function initTokenizerFromManifest(manifest, baseUrl, storageContext = null) {
800
857
  return initTokenizer(manifest, {
801
858
  baseUrl: baseUrl ?? undefined,
@@ -21,8 +21,6 @@ import type {
21
21
  import type { LayerWeights } from './types.js';
22
22
  import type { LoRAAdapter } from './lora.js';
23
23
 
24
- export function isDecodeBuffer(decodeBuffers: DecodeBufferManager | null | undefined, buffer: GPUBuffer): boolean;
25
-
26
24
  export function releaseOrTrack(
27
25
  recorder: CommandRecorder | undefined,
28
26
  buffer: GPUBuffer,
@@ -179,3 +177,16 @@ export function doAttention(
179
177
  recorder?: CommandRecorder,
180
178
  lora?: LoRAAdapter | null
181
179
  ): Promise<AttentionResult>;
180
+
181
+ /**
182
+ * Initialize per-layer convolution state from the loaded weight buffer.
183
+ * Resolves kernel shape, dtype, and input projection bindings.
184
+ */
185
+ export declare function initConvLayerState(
186
+ convState: Record<string, unknown>,
187
+ convKernel: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | CpuWeightBuffer,
188
+ convInProj: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | CpuWeightBuffer | null,
189
+ hiddenSize: number,
190
+ label: string,
191
+ layerIdx: number
192
+ ): Promise<void>;
@@ -25,7 +25,7 @@ import { runLinearAttentionLayer } from './linear-attention.js';
25
25
  import { runGatedShortConvGPU } from '../../../gpu/kernels/gated-short-conv.js';
26
26
 
27
27
 
28
- export function isDecodeBuffer(decodeBuffers, buffer) {
28
+ function isDecodeBuffer(decodeBuffers, buffer) {
29
29
  return !!decodeBuffers?.ownsBuffer(buffer);
30
30
  }
31
31
 
@@ -19,7 +19,7 @@ import {
19
19
  runScale,
20
20
  } from '../../../gpu/kernel-selector.js';
21
21
  import { getNormWeightBuffer, getWeightBuffer } from './weights.js';
22
- import { doMatmul, doRMSNorm, releaseOrTrack } from './ops.js';
22
+ import { doCast, doMatmul, doRMSNorm, releaseOrTrack } from './ops.js';
23
23
  import { runProbes } from './probes.js';
24
24
  import { embed, isRangeBackedCpuEmbeddingSource, normalizeRangeBytes, decodeRangeChunkIntoOutput } from './embed.js';
25
25
  import { selectRuleValue } from '../../../rules/rule-registry.js';
@@ -1447,6 +1447,14 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1447
1447
  let fusedProjectionForScale = null;
1448
1448
  let scaledFused = null;
1449
1449
  try {
1450
+ // Per-layer-model projection: precision island. Source rows decoded
1451
+ // from int4_per_row produce activations whose magnitude before
1452
+ // RMSNorm-mediated scale-down can exceed f16 max (~65504). When the
1453
+ // matmul kernel does not internally widen its accumulator (RDNA3 is
1454
+ // strict, Metal happens to widen), the f16 output saturates to Inf
1455
+ // and then NaN. Compute matmul + QDQ + scale in f32 so the dynamic
1456
+ // range fits, then narrow to activationDtype after the scale step
1457
+ // brings magnitudes back to ~O(1).
1450
1458
  fusedProjection = await doMatmul(
1451
1459
  projectionInputTensor,
1452
1460
  projectionWeight,
@@ -1458,7 +1466,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1458
1466
  label: 'per_layer_fused_projection',
1459
1467
  kernelPath: context.kernelPath ?? null,
1460
1468
  role: 'per_layer_model_projection',
1461
- outputDtype: activationDtype,
1469
+ outputDtype: 'f32',
1462
1470
  },
1463
1471
  recorder
1464
1472
  );
@@ -1476,7 +1484,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1476
1484
  fusedProjection = null;
1477
1485
  }
1478
1486
 
1479
- scaledFused = recorder
1487
+ let scaledFusedF32 = recorder
1480
1488
  ? await recordScale(recorder, fusedProjectionForScale, projectionScale, {
1481
1489
  count: numTokens * totalPerLayerHiddenSize,
1482
1490
  })
@@ -1486,6 +1494,13 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1486
1494
  releaseOrTrack(recorder, fusedProjectionForScale.buffer, decodeBuffers);
1487
1495
  fusedProjectionForScale = null;
1488
1496
 
1497
+ scaledFused = scaledFusedF32.dtype === activationDtype
1498
+ ? scaledFusedF32
1499
+ : await doCast(scaledFusedF32, activationDtype, recorder);
1500
+ if (scaledFused !== scaledFusedF32) {
1501
+ releaseOrTrack(recorder, scaledFusedF32.buffer, decodeBuffers);
1502
+ }
1503
+
1489
1504
  // Step 4: Extract per-layer slices via GPU buffer copies (one encoder, one submit).
1490
1505
  // Reuse cached slice buffers when available to avoid per-step pool churn.
1491
1506
  const device = getDevice();
@@ -1608,6 +1623,10 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1608
1623
  });
1609
1624
  projectionWeightOffset = hiddenOffset * config.hiddenSize * projectionWeightBytes;
1610
1625
  }
1626
+ // Per-layer-model projection: precision island (see fused path
1627
+ // above for rationale). Force f32 through matmul + QDQ + scale,
1628
+ // then narrow to activationDtype after scale brings magnitudes
1629
+ // back to ~O(1).
1611
1630
  projectedTensor = await doMatmul(
1612
1631
  projectionInputTensor,
1613
1632
  projectionWeightForLayer,
@@ -1621,7 +1640,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1621
1640
  layerIdx,
1622
1641
  kernelPath: context.kernelPath ?? null,
1623
1642
  role: 'per_layer_model_projection',
1624
- outputDtype: activationDtype,
1643
+ outputDtype: 'f32',
1625
1644
  },
1626
1645
  recorder
1627
1646
  );
@@ -1632,7 +1651,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1632
1651
  probes: context.debugProbes,
1633
1652
  recorder,
1634
1653
  operatorDiagnostics: context.operatorDiagnostics,
1635
- dtype: activationDtype,
1654
+ dtype: 'f32',
1636
1655
  });
1637
1656
  projectionTensorForScale = pleActivationStaticQdq
1638
1657
  ? await applyPleActivationStaticQdq(
@@ -1647,7 +1666,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1647
1666
  releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
1648
1667
  projectedTensor = null;
1649
1668
  }
1650
- scaledProjectionTensor = recorder
1669
+ const scaledF32 = recorder
1651
1670
  ? await recordScale(recorder, projectionTensorForScale, projectionScale, {
1652
1671
  count: numTokens * hiddenSizePerLayerInput,
1653
1672
  })
@@ -1659,6 +1678,12 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1659
1678
  }
1660
1679
  releaseOrTrack(recorder, projectionTensorForScale.buffer, decodeBuffers);
1661
1680
  projectionTensorForScale = null;
1681
+ scaledProjectionTensor = scaledF32.dtype === activationDtype
1682
+ ? scaledF32
1683
+ : await doCast(scaledF32, activationDtype, recorder);
1684
+ if (scaledProjectionTensor !== scaledF32) {
1685
+ releaseOrTrack(recorder, scaledF32.buffer, decodeBuffers);
1686
+ }
1662
1687
  } finally {
1663
1688
  if (projectionTensorForScale && projectionTensorForScale !== projectedTensor) {
1664
1689
  releaseOrTrack(recorder, projectionTensorForScale.buffer, decodeBuffers);
@@ -20,12 +20,9 @@ export function runProbes(
20
20
  hiddenSize: number;
21
21
  probes?: ProbeConfigSchema[] | null;
22
22
  recorder?: CommandRecorder | null;
23
+ operatorDiagnostics?: Record<string, unknown> | null;
23
24
  dtype?: 'f16' | 'f32';
25
+ phase?: string | null;
26
+ tokenIndex?: number | null;
24
27
  }
25
28
  ): Promise<void>;
26
-
27
- export function hasProbeStage(
28
- probes: ProbeConfigSchema[] | null | undefined,
29
- stage: ProbeStage,
30
- layerIdx?: number
31
- ): boolean;